From 61a94bb6384801c3d355f625f509a0a60c52865c Mon Sep 17 00:00:00 2001 From: prashantbytesyntax Date: Sun, 9 Aug 2026 21:24:30 +0000 Subject: [PATCH 01/11] Add NVIDIA nsys GPU capture for adhoc flamegraphs Capture NVIDIA CUDA kernel activity via host-installed Nsight Systems (nsys) and upload the resulting flamegraph HTML through the existing adhoc path, so GPU profiles show up in Performance Studio's Adhoc Profiling view alongside CPU profiles. - nsys_profiler.py: host-detect nsys (PATH / NSYS_PATH / common install dirs), run a capture, export cuda_gpu_kern_sum (fallback cuda_api_sum), convert the stats CSV to collapsed stacks, and render flamegraph HTML (template when the full agent runtime is present, simple self-contained fallback otherwise). nsys is detected, never bundled. - main.py: --enable-nsys / --nsys-path / --nsys-workload CLI args; run the GPU capture on a background thread in parallel with CPU profilers and prefer the GPU HTML for the adhoc upload when a capture succeeds. - dynamic_profiling_management: honor enable_nsys / nsys_path / nsys_workload from combined_config (PerfSpect-shaped control plane) and tag perf_events with nsys-cuda so the Adhoc UI can show a GPU/nsys chip. - tests: unit coverage for CSV->collapsed and nsys discovery (no GPU/nsys required). - docs/NSYS_GPU_PROFILING.md: enabling, packaging, sandbox topology, and how this relates to a possible later Intel iaprof backend behind the same GPU-profiler control. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/NSYS_GPU_PROFILING.md | 70 +++ .../dynamic_profiling_management/__init__.py | 27 ++ gprofiler/main.py | 76 +++ gprofiler/nsys_profiler.py | 446 ++++++++++++++++++ tests/test_nsys_profiler.py | 106 +++++ 5 files changed, 725 insertions(+) create mode 100644 docs/NSYS_GPU_PROFILING.md create mode 100644 gprofiler/nsys_profiler.py create mode 100644 tests/test_nsys_profiler.py diff --git a/docs/NSYS_GPU_PROFILING.md b/docs/NSYS_GPU_PROFILING.md new file mode 100644 index 000000000..1d3dfe799 --- /dev/null +++ b/docs/NSYS_GPU_PROFILING.md @@ -0,0 +1,70 @@ +# NVIDIA Nsight Systems (nsys) GPU profiling with gProfiler + +## Summary + +gProfiler can capture **CUDA kernel** activity via host-installed **`nsys`** and +upload the resulting flamegraph HTML through the existing **adhoc** path +(Performance Studio → Adhoc Profiling view). + +This is **not** Intel [iaprof](https://github.com/intel/iaprof) (Xe / Battlemage +EU-stall GPU flame graphs). On NVIDIA hosts, nsys is the correct tool; iaprof +can be a later backend behind the same `enable_nsys`-style control-plane flag. + +## Packaging: host-detect, do not bundle + +Mirror PerfSpect: + +| | Behavior | +|---|---| +| Default | Search `NSYS_PATH`, `PATH`, `/usr/local/bin/nsys`, `/opt/nvidia/nsight-systems/*/bin/nsys` | +| Override | `--nsys-path` / heartbeat `combined_config.nsys_path` | +| Bundle | **No** — Nsight Systems is large and NVIDIA-licensed | + +## Enabling + +### CLI + +```bash +sudo ./gprofiler \ + --upload-results --token=... --service-name=k8s-sandbox \ + --enable-heartbeat-server --api-server=https://localhost:30443 \ + --enable-nsys \ + --nsys-workload='/path/to/cuda_burn 30' \ + --duration=30 --flamegraph +``` + +### Heartbeat / Dynamic Profiling console + +Studio checkbox **GPU (nsys)** sets `additional_args.enable_nsys=true`. +The backend merges that into top-level `combined_config`. Optional: + +- `nsys_path`: absolute path to `nsys` +- `nsys_workload`: command for nsys to wrap (strongly recommended) + +When `enable_nsys` succeeds, the uploaded `flamegraph_html` is the **GPU** +flamegraph (preferred over CPU). `run_arguments.perf_events` also gains +`nsys-cuda` so the Adhoc UI can show a **GPU / nsys** chip. + +## Kind / sandbox topology + +Kind runs the Studio control plane only. The agent that invokes nsys must run on +the **GPU host** (see `gprofiler-performance-studio/deploy/k8s-sandbox/gpu/`). + +Run that agent in its own container with `--gpus all` rather than as a bare host +process. `grab_gprofiler_mutex()` binds an abstract socket in the init network +namespace, so a bare host agent conflicts with any other privileged gProfiler on +the machine; a separate network namespace avoids the conflict while still giving +nsys direct GPU access. Mount the Nsight Systems install read-only — nsys is +detected, never bundled. + +## Collapsed stack shape + +``` +gpu;nsys;cuda_kernel;burn(float *, int, int) 28918 +``` + +Fallback if kern sum is empty: + +``` +gpu;nsys;cuda_api;cudaLaunchKernel 213 +``` diff --git a/gprofiler/dynamic_profiling_management/__init__.py b/gprofiler/dynamic_profiling_management/__init__.py index 8ec7e1995..b32886594 100644 --- a/gprofiler/dynamic_profiling_management/__init__.py +++ b/gprofiler/dynamic_profiling_management/__init__.py @@ -106,6 +106,24 @@ def create_profiler_args( logger.error(f"PerfSpect not found at {perfspect_path}, hardware metrics disabled") new_args.collect_hw_metrics = False + # NVIDIA nsys GPU capture (host-detect; not bundled — see docs/NSYS_GPU_PROFILING.md) + if combined_config.get("enable_nsys", False): + from gprofiler.nsys_profiler import find_nsys + + new_args.enable_nsys = True + nsys_path = combined_config.get("nsys_path") or None + found = find_nsys(nsys_path) + if found is not None: + new_args.nsys_path = str(found) + new_args.nsys_workload = combined_config.get("nsys_workload") + logger.info(f"enable_nsys: using nsys at {found}") + else: + logger.error( + "enable_nsys set but nsys not found on host " + "(install Nsight Systems or set NSYS_PATH / nsys_path); GPU capture disabled" + ) + new_args.enable_nsys = False + max_processes = combined_config.get("max_processes", 10) new_args.max_processes_per_profiler = max_processes @@ -113,6 +131,15 @@ def create_profiler_args( if profiler_configs: _apply_profiler_configs(new_args, profiler_configs) + # Tag after profiler_configs so Adhoc UI can show a GPU/nsys chip without + # being overwritten by perf event defaults. + if getattr(new_args, "enable_nsys", False): + existing = getattr(new_args, "perf_events", None) or "cycles" + events = [e.strip() for e in str(existing).split(",") if e.strip()] + if "nsys-cuda" not in events: + events.append("nsys-cuda") + new_args.perf_events = ",".join(events) + return new_args diff --git a/gprofiler/main.py b/gprofiler/main.py index e6b1ec704..e7cec5320 100644 --- a/gprofiler/main.py +++ b/gprofiler/main.py @@ -168,6 +168,12 @@ def __init__( self._collect_hw_metrics = collect_hw_metrics self._perfspect_path = perfspect_path self._perfspect_duration = perfspect_duration + # NVIDIA nsys GPU capture (host-detect; optional) + self._enable_nsys = bool(user_args.get("enable_nsys", False)) + self._nsys_path = user_args.get("nsys_path") + self._nsys_workload = user_args.get("nsys_workload") + self._nsys_thread: Optional[threading.Thread] = None + self._nsys_html: Optional[str] = None if self._collect_metadata: self._static_metadata = get_static_metadata(self._spawn_time, user_args, self._external_metadata_path) @@ -379,6 +385,40 @@ def start(self) -> None: logger.warning(f"Failed to start {prof.__class__.__name__}, continuing without it", exc_info=True) self.process_profilers.remove(cast(ProcessProfilerBase, prof)) + # Kick off NVIDIA nsys GPU capture in parallel with CPU profilers (adhoc). + if self._enable_nsys: + self._nsys_html = None + self._nsys_thread = threading.Thread( + target=self._run_nsys_capture_background, + name="nsys-capture", + daemon=True, + ) + self._nsys_thread.start() + logger.info("Started background nsys GPU capture thread") + + def _run_nsys_capture_background(self) -> None: + """Collect nsys GPU flamegraph HTML while CPU profilers run.""" + try: + from gprofiler.nsys_profiler import collect_nsys_adhoc_html + + def _gen(collapsed: str) -> Optional[str]: + # Approximate window; labels only. + end = datetime.datetime.utcnow() + start = end - datetime.timedelta(seconds=self._duration) + return self._generate_flamegraph_html(collapsed, start, end) + + html = collect_nsys_adhoc_html( + duration_sec=self._duration, + nsys_path=self._nsys_path, + workload=self._nsys_workload, + stop_event=self._profiler_state.stop_event, + generate_html_fn=_gen, + ) + self._nsys_html = html + except Exception: + logger.exception("Background nsys GPU capture failed") + self._nsys_html = None + def stop(self) -> None: logger.info("Stopping ...") self._profiler_state.stop_event.set() @@ -520,6 +560,17 @@ def _snapshot(self) -> None: if flamegraph_html: logger.info("Generated flamegraph HTML for profile data") + # Prefer nsys GPU HTML for Adhoc when enable_nsys produced a capture. + if self._enable_nsys: + if self._nsys_thread is not None and self._nsys_thread.is_alive(): + logger.info("Waiting for background nsys GPU capture to finish...") + self._nsys_thread.join(timeout=max(60, self._duration + 120)) + if self._nsys_html: + logger.info("Using nsys GPU flamegraph HTML for upload (preferred over CPU)") + flamegraph_html = self._nsys_html + else: + logger.warning("enable_nsys was set but no GPU HTML was produced; keeping CPU flamegraph if any") + if NoopProfiler.is_noop_profiler(self.system_profiler): assert system_result == {}, system_result # should be empty! merged_result = concatenate_profiles( @@ -1264,6 +1315,31 @@ def parse_cmd_args() -> configargparse.Namespace: help="The default perfspect tool collection time is 60 second.", ) + nsys_options = parser.add_argument_group("NVIDIA nsys GPU profiling") + nsys_options.add_argument( + "--enable-nsys", + action="store_true", + default=False, + dest="enable_nsys", + help="Enable NVIDIA Nsight Systems (nsys) GPU capture for adhoc flamegraphs. " + "Requires nsys installed on the host (not bundled).", + ) + nsys_options.add_argument( + "--nsys-path", + type=str, + dest="nsys_path", + default=None, + help="Path to nsys binary (default: search PATH / NSYS_PATH / common install dirs).", + ) + nsys_options.add_argument( + "--nsys-workload", + type=str, + dest="nsys_workload", + default=None, + help="Command line for nsys to wrap (e.g. '/path/to/cuda_burn 30'). " + "Recommended for useful CUDA kernel frames.", + ) + args = parser.parse_args() args.perf_inject = args.nodejs_mode == "perf" diff --git a/gprofiler/nsys_profiler.py b/gprofiler/nsys_profiler.py new file mode 100644 index 000000000..bb5643e09 --- /dev/null +++ b/gprofiler/nsys_profiler.py @@ -0,0 +1,446 @@ +# +# Copyright (C) 2026 Intel Corporation / Pinterest +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""NVIDIA Nsight Systems (nsys) GPU capture helpers for adhoc flamegraphs. + +Host-detect only — nsys is NOT bundled with the agent (same packaging model as +PerfSpect). See docs/NSYS_GPU_PROFILING.md. +""" + +from __future__ import annotations + +import csv +import glob +import io +import logging +import os +import shlex +import shutil +import subprocess # nosec B404 +from pathlib import Path +from typing import Callable, List, Optional, Sequence + +logger = logging.getLogger(__name__) + +NSYS_DATA_DIRECTORY = "/tmp/gprofiler_nsys" +DEFAULT_NSYS_CANDIDATES = ( + "/usr/local/bin/nsys", + "/opt/nvidia/nsight-systems/2025.6.1/bin/nsys", + "/opt/nvidia/nsight-systems/2025.5.1/bin/nsys", + "/opt/nvidia/nsight-systems/2024.6.2/bin/nsys", +) + + +def find_nsys(explicit_path: Optional[str] = None) -> Optional[Path]: + """Locate an executable nsys binary. + + Search order: explicit_path → NSYS_PATH env → PATH → common install paths. + """ + candidates: List[str] = [] + if explicit_path: + candidates.append(explicit_path) + env_path = os.environ.get("NSYS_PATH") + if env_path: + candidates.append(env_path) + which = shutil.which("nsys") + if which: + candidates.append(which) + candidates.extend(DEFAULT_NSYS_CANDIDATES) + # Glob any versioned install. Some packages ship only the target-linux-* tree + # without the bin/ symlink, so check both layouts. + candidates.extend(sorted(glob.glob("/opt/nvidia/nsight-systems/*/bin/nsys"), reverse=True)) + candidates.extend(sorted(glob.glob("/opt/nvidia/nsight-systems/*/target-linux-*/nsys"), reverse=True)) + + seen = set() + for raw in candidates: + if not raw or raw in seen: + continue + seen.add(raw) + path = Path(raw) + try: + if path.is_file() and os.access(path, os.X_OK): + # Resolve symlinks for logging, but return the usable path. + return path.resolve() if path.exists() else path + except OSError: + continue + return None + + +def _parse_workload(workload: Optional[str | Sequence[str]]) -> Optional[List[str]]: + if workload is None or workload == "": + return None + if isinstance(workload, (list, tuple)): + return [str(x) for x in workload] + return shlex.split(str(workload)) + + +def run_nsys_capture( + nsys: Path, + output_prefix: Path, + duration_sec: int, + workload_cmd: Optional[Sequence[str]] = None, + stop_event=None, +) -> Optional[Path]: + """Run `nsys profile` and return the path to the `.nsys-rep` file, or None.""" + output_prefix.parent.mkdir(parents=True, exist_ok=True) + duration_sec = max(1, int(duration_sec)) + + cmd: List[str] = [ + str(nsys), + "profile", + "-o", + str(output_prefix), + "--force-overwrite=true", + # CUDA-focused, skip heavy CPU sampling / slow symbol waits where possible. + "-t", + "cuda,nvtx", + "-s", + "none", + ] + + workload = list(workload_cmd) if workload_cmd else None + if workload: + cmd.extend(workload) + else: + # Duration-bounded no-op session: sleep so nsys still produces a report. + # Prefer a real CUDA workload via nsys_workload for useful kern sums. + cmd.extend(["--duration", str(duration_sec), "sleep", str(duration_sec)]) + + logger.info("Running nsys capture: %s", " ".join(cmd)) + timeout = duration_sec + 120 + try: + # nosec B603 — nsys path validated; workload from operator config + proc = subprocess.run( # nosec B603 + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + if proc.stdout: + logger.info("nsys profile output (tail): %s", proc.stdout.decode("utf-8", "replace")[-2000:]) + if proc.returncode != 0: + logger.error("nsys profile exited with code %s", proc.returncode) + return None + except subprocess.TimeoutExpired: + logger.error("nsys profile timed out after %ss", timeout) + return None + except OSError as exc: + logger.error("Failed to execute nsys: %s", exc) + return None + + if stop_event is not None and getattr(stop_event, "is_set", lambda: False)(): + logger.info("Stop event set during nsys capture; discarding result") + return None + + rep = Path(str(output_prefix) + ".nsys-rep") + if not rep.is_file(): + # Some nsys versions may write without forcing the suffix identically. + matches = list(output_prefix.parent.glob(output_prefix.name + "*.nsys-rep")) + if matches: + rep = matches[0] + if not rep.is_file(): + logger.error("nsys profile completed but .nsys-rep not found at %s", rep) + return None + return rep + + +def _csv_to_collapsed(csv_text: str, frame_prefix: str) -> Optional[str]: + """Convert an nsys stats CSV into collapsed stacks. + + Uses Total Time (ns) (or the first numeric time-like column) as weight. + """ + reader = csv.DictReader(io.StringIO(csv_text)) + if not reader.fieldnames: + return None + + # Normalize header keys (strip BOM / whitespace) + fieldnames = [f.strip().lstrip("\ufeff") for f in reader.fieldnames] + reader.fieldnames = fieldnames + + name_key = next((f for f in fieldnames if f.lower() in ("name", "kernel name", "kernel")), None) + time_key = next( + ( + f + for f in fieldnames + if f.lower() in ("total time (ns)", "total time(ns)", "total (ns)", "time (ns)") + ), + None, + ) + if name_key is None: + # Fall back to last column as name (nsys puts Name last) + name_key = fieldnames[-1] + if time_key is None: + # Prefer any column containing 'total' and 'ns' + time_key = next( + (f for f in fieldnames if "total" in f.lower() and "ns" in f.lower()), + None, + ) + if time_key is None: + time_key = next((f for f in fieldnames if "time" in f.lower()), None) + + lines: List[str] = [] + for row in reader: + # DictReader may still use original keys; build a stripped map. + cleaned = { (k or "").strip().lstrip("\ufeff"): (v or "").strip() for k, v in row.items() } + name = cleaned.get(name_key, "").strip().strip('"') + if not name: + continue + raw_time = cleaned.get(time_key or "", "0") or "0" + try: + # Total Time may be float ns + weight = max(1, int(float(raw_time))) + except ValueError: + weight = 1 + # Scale down huge ns values to keep flamegraph samples manageable + # (1 sample ≈ 1µs of GPU time). Keep at least 1. + samples = max(1, weight // 1000) + safe_name = name.replace(";", ",") + lines.append(f"{frame_prefix};{safe_name} {samples}") + + return "\n".join(lines) if lines else None + + +def nsys_stats_to_collapsed(nsys: Path, nsys_rep: Path, work_dir: Path) -> Optional[str]: + """Export cuda_gpu_kern_sum (fallback cuda_api_sum) and convert to collapsed stacks.""" + work_dir.mkdir(parents=True, exist_ok=True) + + def _export(report: str, out_prefix: Path) -> Optional[str]: + cmd = [ + str(nsys), + "stats", + f"--report={report}", + "--format=csv", + "--force-export=true", + "-o", + str(out_prefix), + str(nsys_rep), + ] + try: + proc = subprocess.run( # nosec B603 + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=120, + check=False, + ) + if proc.returncode != 0: + logger.warning( + "nsys stats %s failed (rc=%s): %s", + report, + proc.returncode, + (proc.stdout or b"")[-1000:].decode("utf-8", "replace"), + ) + except (subprocess.TimeoutExpired, OSError) as exc: + logger.warning("nsys stats %s error: %s", report, exc) + return None + + matches = sorted(work_dir.glob(out_prefix.name + "*.csv")) + # Also check CWD-relative names nsys sometimes writes + matches += sorted(Path(".").glob(out_prefix.name + "*.csv")) + matches += sorted(nsys_rep.parent.glob(out_prefix.name + "*.csv")) + # Dedup + seen = set() + unique = [] + for m in matches: + rp = str(m.resolve()) if m.exists() else str(m) + if rp not in seen and m.is_file() and m.stat().st_size > 0: + seen.add(rp) + unique.append(m) + if not unique: + logger.info("nsys stats %s produced no non-empty CSV", report) + return None + return unique[0].read_text(encoding="utf-8", errors="replace") + + kern_csv = _export("cuda_gpu_kern_sum", work_dir / "kern") + if kern_csv: + collapsed = _csv_to_collapsed(kern_csv, "gpu;nsys;cuda_kernel") + if collapsed: + logger.info("Built collapsed stacks from cuda_gpu_kern_sum (%d lines)", collapsed.count("\n") + 1) + return collapsed + + api_csv = _export("cuda_api_sum", work_dir / "api") + if api_csv: + collapsed = _csv_to_collapsed(api_csv, "gpu;nsys;cuda_api") + if collapsed: + logger.info("Built collapsed stacks from cuda_api_sum fallback (%d lines)", collapsed.count("\n") + 1) + return collapsed + + logger.error("No usable nsys stats CSV (cuda_gpu_kern_sum / cuda_api_sum)") + return None + + +def _simple_gpu_flamegraph_html(collapsed: str, title: str = "nsys GPU profile") -> str: + """Minimal self-contained HTML for Adhoc iframe when burn/template unavailable.""" + rows = [] + total = 0 + parsed = [] + for line in collapsed.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + try: + stack, count_s = line.rsplit(" ", 1) + count = int(count_s) + except ValueError: + continue + parsed.append((stack, count)) + total += count + total = total or 1 + # Sort widest first (flamegraph convention) + parsed.sort(key=lambda x: -x[1]) + bars = [] + for stack, count in parsed: + pct = 100.0 * count / total + leaf = stack.split(";")[-1] + bars.append( + f'
' + f"{leaf}{pct:.1f}%
" + ) + bars_html = "\n".join(bars) if bars else "

No GPU samples

" + return f""" +{title} + +

{title}

+

Collapsed stacks from NVIDIA nsys (cuda_gpu_kern_sum / cuda_api_sum). +Total weight: {total}. Inspired by GPU flame-graph views; host-detect nsys (not bundled).

+
{bars_html}
+ +""" + + +def generate_nsys_flamegraph_html( + collapsed: str, + start_iso: str, + end_iso: str, + generate_html_fn: Optional[Callable[..., Optional[str]]] = None, +) -> Optional[str]: + """Turn collapsed stacks into flamegraph HTML via callback or simple fallback.""" + if generate_html_fn is not None: + try: + html = generate_html_fn(collapsed) + if html: + return html + except Exception as exc: + logger.warning("generate_html_fn failed (%s); using simple GPU HTML fallback", exc) + + # Prefer burn+template when the full agent runtime is available. + try: + from gprofiler.utils import resource_path, run_process, get_iso8601_format_time + from datetime import datetime as _dt + + start = _dt.fromisoformat(start_iso.replace("Z", "+00:00")) if start_iso else _dt.utcnow() + end = _dt.fromisoformat(end_iso.replace("Z", "+00:00")) if end_iso else _dt.utcnow() + + start_ts = get_iso8601_format_time(start) + end_ts = get_iso8601_format_time(end) + html = ( + Path(resource_path("flamegraph/flamegraph_template.html")) + .read_bytes() + .replace( + b"{{{JSON_DATA}}}", + run_process( + [resource_path("burn"), "convert", "--type=folded"], + suppress_log=True, + stdin=collapsed.encode(), + stop_event=None, + timeout=30, + ).stdout, + ) + .replace(b"{{{START_TIME}}}", start_ts.encode()) + .replace(b"{{{END_TIME}}}", end_ts.encode()) + ) + return html.decode("utf-8") + except Exception as exc: + logger.info("burn/template HTML unavailable (%s); using simple GPU HTML fallback", exc) + return _simple_gpu_flamegraph_html(collapsed) + + +def collect_nsys_adhoc_html( + *, + duration_sec: int, + nsys_path: Optional[str] = None, + workload: Optional[str | Sequence[str]] = None, + work_dir: Optional[str] = None, + stop_event=None, + generate_html_fn: Optional[Callable[[str], Optional[str]]] = None, +) -> Optional[str]: + """End-to-end: find nsys → capture → collapsed → HTML. Returns HTML or None.""" + nsys = find_nsys(nsys_path) + if nsys is None: + logger.error( + "enable_nsys set but nsys not found (install Nsight Systems on the host " + "or set NSYS_PATH / --nsys-path). nsys is not bundled with gProfiler." + ) + return None + + logger.info("Using nsys at %s", nsys) + base = Path(work_dir or NSYS_DATA_DIRECTORY) + base.mkdir(parents=True, exist_ok=True) + prefix = base / "gprofiler_nsys_capture" + workload_cmd = _parse_workload(workload) + + if workload_cmd is None: + logger.warning( + "No nsys_workload configured; capturing a duration-bounded sleep session. " + "Set nsys_workload (e.g. path to cuda_burn) for real CUDA kernel frames." + ) + + rep = run_nsys_capture( + nsys, + prefix, + duration_sec=duration_sec, + workload_cmd=workload_cmd, + stop_event=stop_event, + ) + if rep is None: + return None + + collapsed = nsys_stats_to_collapsed(nsys, rep, base) + if not collapsed: + return None + + from datetime import datetime, timezone + + now = datetime.now(timezone.utc) + start = now # approximate; fine for adhoc labeling + end = now + + def _gen(collapsed_data: str) -> Optional[str]: + if generate_html_fn is not None: + return generate_html_fn(collapsed_data) + return generate_nsys_flamegraph_html( + collapsed_data, + start.isoformat(), + end.isoformat(), + generate_html_fn=None, + ) + + html = _gen(collapsed) + if html: + logger.info("Generated nsys GPU flamegraph HTML (%d bytes)", len(html)) + else: + logger.error("Failed to generate nsys flamegraph HTML from collapsed stacks") + return html diff --git a/tests/test_nsys_profiler.py b/tests/test_nsys_profiler.py new file mode 100644 index 000000000..ce0118c41 --- /dev/null +++ b/tests/test_nsys_profiler.py @@ -0,0 +1,106 @@ +# +# Copyright (C) 2026 Intel Corporation / Pinterest +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Unit tests for gprofiler.nsys_profiler (no GPU / nsys required).""" + +from __future__ import annotations + +import os +from pathlib import Path +from unittest import mock + +import pytest + +from gprofiler.nsys_profiler import ( + _csv_to_collapsed, + find_nsys, + nsys_stats_to_collapsed, +) + + +KERN_CSV = """Time (%),Total Time (ns),Instances,Avg (ns),Med (ns),Min (ns),Max (ns),StdDev (ns),Name +100.0,28918591,5,5783718.2,5780070.0,5777126,5797542,8313.1,"burn(float *, int, int)" +""" + +API_CSV = """Time (%),Total Time (ns),Num Calls,Avg (ns),Med (ns),Min (ns),Max (ns),StdDev (ns),Name +81.2,131012864,1,131012864.0,131012864.0,131012864,131012864,0.0,cudaMalloc +17.9,28943134,5,5788626.8,5785451.0,5783190,5801941,7724.7,cudaDeviceSynchronize +""" + + +def test_csv_to_collapsed_kern(): + collapsed = _csv_to_collapsed(KERN_CSV, "gpu;nsys;cuda_kernel") + assert collapsed is not None + assert "gpu;nsys;cuda_kernel;burn(float *, int, int)" in collapsed + # weight = 28918591 // 1000 + assert collapsed.split()[-1] == "28918" + + +def test_csv_to_collapsed_api(): + collapsed = _csv_to_collapsed(API_CSV, "gpu;nsys;cuda_api") + assert collapsed is not None + lines = collapsed.splitlines() + assert len(lines) == 2 + assert "cudaMalloc" in lines[0] + assert "cudaDeviceSynchronize" in lines[1] + + +def test_find_nsys_explicit(tmp_path: Path): + fake = tmp_path / "nsys" + fake.write_text("#!/bin/sh\n") + fake.chmod(0o755) + found = find_nsys(str(fake)) + assert found is not None + assert found.name == "nsys" + + +def test_find_nsys_missing_falls_through_to_path(monkeypatch, tmp_path: Path): + # Invalid explicit path should not block discovery of a real nsys on PATH. + monkeypatch.setenv("PATH", str(tmp_path)) # empty PATH dir + monkeypatch.delenv("NSYS_PATH", raising=False) + # With no PATH hit and a nonsense explicit path, still may find /opt installs — + # assert only that an explicitly *valid* missing path doesn't crash. + result = find_nsys("/nonexistent/nsys-binary-xyz") + # Either None (no system nsys) or a real install — never raises. + assert result is None or result.name == "nsys" + + +def test_find_nsys_prefers_explicit(tmp_path: Path): + fake = tmp_path / "nsys" + fake.write_text("#!/bin/sh\n") + fake.chmod(0o755) + found = find_nsys(str(fake)) + assert found is not None + assert found == fake.resolve() + + +def test_nsys_stats_to_collapsed_uses_kern(tmp_path: Path, monkeypatch): + nsys = tmp_path / "nsys" + nsys.write_text("#!/bin/sh\n") + nsys.chmod(0o755) + rep = tmp_path / "cap.nsys-rep" + rep.write_bytes(b"fake") + + def fake_run(cmd, **kwargs): + # Write kern CSV where nsys_stats_to_collapsed looks + out_prefix = Path(cmd[cmd.index("-o") + 1]) + csv_path = Path(str(out_prefix) + "_cuda_gpu_kern_sum.csv") + csv_path.write_text(KERN_CSV) + return mock.Mock(returncode=0, stdout=b"ok") + + monkeypatch.setattr("gprofiler.nsys_profiler.subprocess.run", fake_run) + collapsed = nsys_stats_to_collapsed(nsys, rep, tmp_path / "work") + assert collapsed is not None + assert "cuda_kernel;burn" in collapsed From 1c61fc85e7f0f9d3cab5df5d161cb39b812b0a51 Mon Sep 17 00:00:00 2001 From: prashantbytesyntax Date: Mon, 10 Aug 2026 01:06:27 +0000 Subject: [PATCH 02/11] Fix nsys workload env so dynamically-linked workloads can launch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent runs as a PyInstaller bundle that prepends its own lib dir (/tmp/_MEIxxxx) to LD_LIBRARY_PATH. A spawned nsys workload inherited that and picked up the bundle's older libstdc++, so a dynamically-linked target (e.g. PyTorch) failed to import (CXXABI_1.3.8 not found) and nsys captured nothing — the adhoc flamegraph fell back to an empty CPU profile. _workload_env() now builds the child environment from the pre-bundle values: prefer PyInstaller's saved LD_LIBRARY_PATH_ORIG / LD_PRELOAD_ORIG, and if absent, strip any _MEI bundle path so the child resolves system libraries. Statically-linked workloads (e.g. cuda_burn) were unaffected and still are. Co-Authored-By: Claude Opus 4.8 (1M context) --- gprofiler/nsys_profiler.py | 30 ++++++++++++++++++++++++++ tests/test_nsys_profiler.py | 43 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/gprofiler/nsys_profiler.py b/gprofiler/nsys_profiler.py index bb5643e09..8de0988e5 100644 --- a/gprofiler/nsys_profiler.py +++ b/gprofiler/nsys_profiler.py @@ -43,6 +43,35 @@ ) +def _workload_env() -> dict: + """Environment for the nsys-wrapped workload. + + gProfiler runs as a PyInstaller bundle that prepends its own lib dir + (/tmp/_MEIxxxx) to LD_LIBRARY_PATH; a spawned workload (e.g. dynamically-linked + PyTorch) would otherwise pick up the bundle's older libstdc++ and fail to + import (CXXABI_1.3.8 not found). Prefer PyInstaller's saved *_ORIG value; if it + is absent, strip any _MEI bundle path from LD_LIBRARY_PATH / LD_PRELOAD so the + child resolves the system libraries. + """ + env = os.environ.copy() + for var in ("LD_LIBRARY_PATH", "LD_PRELOAD"): + orig = env.pop(f"{var}_ORIG", None) + if orig is not None: + if orig: + env[var] = orig + else: + env.pop(var, None) + continue + current = env.get(var) + if current: + cleaned = os.pathsep.join(p for p in current.split(os.pathsep) if p and "/_MEI" not in p) + if cleaned: + env[var] = cleaned + else: + env.pop(var, None) + return env + + def find_nsys(explicit_path: Optional[str] = None) -> Optional[Path]: """Locate an executable nsys binary. @@ -128,6 +157,7 @@ def run_nsys_capture( stderr=subprocess.STDOUT, timeout=timeout, check=False, + env=_workload_env(), ) if proc.stdout: logger.info("nsys profile output (tail): %s", proc.stdout.decode("utf-8", "replace")[-2000:]) diff --git a/tests/test_nsys_profiler.py b/tests/test_nsys_profiler.py index ce0118c41..f05a13d64 100644 --- a/tests/test_nsys_profiler.py +++ b/tests/test_nsys_profiler.py @@ -25,6 +25,7 @@ from gprofiler.nsys_profiler import ( _csv_to_collapsed, + _workload_env, find_nsys, nsys_stats_to_collapsed, ) @@ -86,6 +87,48 @@ def test_find_nsys_prefers_explicit(tmp_path: Path): assert found == fake.resolve() +def test_workload_env_restores_original_ld_library_path(monkeypatch): + # PyInstaller bundle sets LD_LIBRARY_PATH to its own libs and saves the + # pre-launch value in *_ORIG. The spawned workload must get the original. + monkeypatch.setenv("LD_LIBRARY_PATH", "/tmp/_MEIxxxx") + monkeypatch.setenv("LD_LIBRARY_PATH_ORIG", "/usr/lib/x86_64-linux-gnu") + env = _workload_env() + assert env["LD_LIBRARY_PATH"] == "/usr/lib/x86_64-linux-gnu" + assert "LD_LIBRARY_PATH_ORIG" not in env + + +def test_workload_env_empty_orig_unsets_var(monkeypatch): + # When *_ORIG is empty, the var was unset before the bundle ran: unset it. + monkeypatch.setenv("LD_LIBRARY_PATH", "/tmp/_MEIxxxx") + monkeypatch.setenv("LD_LIBRARY_PATH_ORIG", "") + env = _workload_env() + assert "LD_LIBRARY_PATH" not in env + + +def test_workload_env_noop_without_orig(monkeypatch): + # Not running under PyInstaller (no *_ORIG, no _MEI): leave the env as-is. + monkeypatch.setenv("LD_LIBRARY_PATH", "/keep/this") + monkeypatch.delenv("LD_LIBRARY_PATH_ORIG", raising=False) + env = _workload_env() + assert env["LD_LIBRARY_PATH"] == "/keep/this" + + +def test_workload_env_strips_mei_when_no_orig(monkeypatch): + # No *_ORIG saved, but LD_LIBRARY_PATH carries a PyInstaller _MEI bundle dir: + # drop the bundle path, keep the rest so the child finds system libs. + monkeypatch.delenv("LD_LIBRARY_PATH_ORIG", raising=False) + monkeypatch.setenv("LD_LIBRARY_PATH", "/tmp/_MEIabc123:/usr/lib/x86_64-linux-gnu") + env = _workload_env() + assert env["LD_LIBRARY_PATH"] == "/usr/lib/x86_64-linux-gnu" + + +def test_workload_env_unsets_when_only_mei(monkeypatch): + monkeypatch.delenv("LD_LIBRARY_PATH_ORIG", raising=False) + monkeypatch.setenv("LD_LIBRARY_PATH", "/tmp/_MEIabc123") + env = _workload_env() + assert "LD_LIBRARY_PATH" not in env + + def test_nsys_stats_to_collapsed_uses_kern(tmp_path: Path, monkeypatch): nsys = tmp_path / "nsys" nsys.write_text("#!/bin/sh\n") From 26565c7b7dfc02178668a48716db7a13510f34f1 Mon Sep 17 00:00:00 2001 From: prashantbytesyntax Date: Mon, 10 Aug 2026 08:09:24 +0000 Subject: [PATCH 03/11] Document flamegraph semantics and PyInstaller workload caveat - Reading the flamegraph: the prefix is a category not a call stack, weights are GPU time (CUPTI tracing) not samples or PMU counters, nsys-cuda is a capture-type tag, and real ML workloads surface library-dispatched kernels (cutlass GEMM with fused epilogues). - Workload environment: why dynamically-linked workloads failed under the PyInstaller bundle's LD_LIBRARY_PATH and how _workload_env() prevents it, plus the bare-root symptom to look for. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/NSYS_GPU_PROFILING.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/NSYS_GPU_PROFILING.md b/docs/NSYS_GPU_PROFILING.md index 1d3dfe799..b1e4a0bec 100644 --- a/docs/NSYS_GPU_PROFILING.md +++ b/docs/NSYS_GPU_PROFILING.md @@ -68,3 +68,29 @@ Fallback if kern sum is empty: ``` gpu;nsys;cuda_api;cudaLaunchKernel 213 ``` + +## Reading the flamegraph + +- The `gpu;nsys;cuda_kernel` prefix is a constant category, **not a call stack**; + `cuda_gpu_kern_sum` is a flat per-kernel summary, so each kernel is a leaf and + the only meaningful axis is width. +- Weights are **GPU time in microseconds** from CUPTI activity tracing (total ns + per kernel across all launches, divided by 1000) — not samples, despite the + renderer's "samples" label, and not PMU counters. +- The `nsys-cuda` entry in `perf_events` is a capture-type tag for the Adhoc UI + chip, not a hardware event. +- For a real ML workload the frames are library kernels the framework dispatched + (e.g. `cutlass_80_tensorop_*gemm_*` for a PyTorch fp16 matmul on Tensor Cores; + a `_relu` infix means the activation was fused into the GEMM epilogue). + +## Workload environment (PyInstaller caveat) + +gProfiler runs as a PyInstaller bundle that prepends its unpack dir +(`/tmp/_MEIxxxx`) to `LD_LIBRARY_PATH`. The nsys workload is spawned with a +cleaned environment (`_workload_env()`): PyInstaller's saved `*_ORIG` values are +restored, and any `_MEI` path is stripped otherwise. Without this, a +**dynamically-linked** workload (python/torch, most real binaries) loads the +bundle's older `libstdc++` and fails to start (`CXXABI_... not found`), yielding +an empty capture; statically-linked workloads are unaffected. If `enable_nsys` +produces a bare `root` flamegraph, check the agent log for +`ImportError`/`CXXABI` from the workload. From 2ebb52fe8bcb2ef75ea34a6f31fed62484471f02 Mon Sep 17 00:00:00 2001 From: prashantbytesyntax Date: Mon, 10 Aug 2026 08:29:05 +0000 Subject: [PATCH 04/11] Add end-to-end architecture section to the nsys spec Documents the full pipeline with PyTorch as the worked example: control plane vs GPU agent container topology, the five agent-side steps from profile_request to the /api/profiles upload, where the flamegraph HTML is generated vs rendered, and the scope boundary against torch.profiler and Nsight Compute (with the NVTX emit_nvtx bridge as a possible follow-up). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/NSYS_GPU_PROFILING.md | 49 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/NSYS_GPU_PROFILING.md b/docs/NSYS_GPU_PROFILING.md index b1e4a0bec..6b5f2f692 100644 --- a/docs/NSYS_GPU_PROFILING.md +++ b/docs/NSYS_GPU_PROFILING.md @@ -57,6 +57,55 @@ the machine; a separate network namespace avoids the conflict while still giving nsys direct GPU access. Mount the Nsight Systems install read-only — nsys is detected, never bundled. +## Architecture: end-to-end pipeline (PyTorch example) + +The nsys workload is just a command string, so profiling a real ML framework is +the same pipeline as `cuda_burn` — the only requirement is that the agent's +container can execute the command. For PyTorch that means an agent image with +`python3` + `torch` layered on (see `deploy/k8s-sandbox/gpu/agent-torch.Dockerfile` +in gprofiler-performance-studio). + +``` +┌────────────────────────── GPU host (e.g. g5.4xlarge / A10G) ─────────────────────────┐ +│ │ +│ ┌── Kind cluster (Studio control plane — no GPU) ─────────────────────────────────┐ │ +│ │ webapp ─ indexer ─ Postgres ─ ClickHouse ─ LocalStack("S3") │ │ +│ │ ▲ │ ▲ │ │ +│ │ │ └── writes flamegraph HTML ─────────┘ │ │ +│ └─────┼────────────────────────────────────────────────────────────────────────--┘ │ +│ │ kubectl port-forward (bound on all addresses) │ +│ ┌─────┴── GPU agent container (--gpus all, own network namespace) ──────────────┐ │ +│ │ gProfiler agent (PyInstaller exe) │ │ +│ │ ① heartbeat receives adhoc profile_request │ │ +│ │ { enable_nsys, nsys_workload: "python3 /gpu/torch_workload.py 30" } │ │ +│ │ ② spawns (with _workload_env() cleaned environment): │ │ +│ │ nsys profile -t cuda,nvtx -s none │ │ +│ │ └─ torch dispatches cuBLAS/cutlass + ATen kernels on the GPU │ │ +│ │ ③ nsys stats cuda_gpu_kern_sum → CSV → collapsed stacks │ │ +│ │ ④ collapsed → flamegraph HTML; perf_events += "nsys-cuda" │ │ +│ │ ⑤ POST /api/profiles with flamegraph_html in the profile header JSON │ │ +│ │ mounts: workload dir (ro) · Nsight Systems install (ro) │ │ +│ └───────────────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────────────────┘ + Browser → Studio Adhoc view: lists HTML files from S3, shows the GPU / nsys + chip (from nsys-cuda), renders the agent-generated HTML in an iframe. +``` + +Notes: + +- The flamegraph is **generated by the agent** (step ④) and stored as-is; the UI + replays it in an iframe rather than computing it. +- The agent container reaches Studio via the host gateway (`host.docker.internal`) + because it deliberately does not share the host network namespace (mutex, see + topology above). +- The indexer marks the upload adhoc and records `perf_events` metadata + (`callstacks.go`), which is what drives the Adhoc UI chip. +- Scope: this is kernel-level triage. Framework-side attribution (which module / + line launched a kernel) is `torch.profiler` territory; per-kernel hardware + counters are Nsight Compute. A future improvement could surface NVTX ranges + (already traced via `-t cuda,nvtx`) emitted by `torch.autograd.profiler.emit_nvtx()` + to add op-level frames above the kernel leaves. + ## Collapsed stack shape ``` From 57981820b77a83c57c68674ff3d9d0cc90172891 Mon Sep 17 00:00:00 2001 From: prashantbytesyntax Date: Fri, 14 Aug 2026 06:09:20 +0000 Subject: [PATCH 05/11] Add CPU/GPU timeline view for nsys adhoc captures With --nsys-timeline (CLI) or combined_config.nsys_timeline (heartbeat), export cuda_gpu_trace + cuda_api_trace from the same capture and upload a self-contained timeline HTML instead of the GPU flamegraph: swim lanes per CPU thread and GPU device/stream, wheel-zoom/drag-pan, and click-to-highlight CorrID linking each cudaLaunchKernel to its kernel. Falls back to the flamegraph when the trace export yields no events; captures over 20k events keep the longest ones to bound upload size. perf_events gains nsys-timeline so the UI can distinguish the view. Co-Authored-By: Claude Fable 5 --- docs/NSYS_GPU_PROFILING.md | 20 + .../dynamic_profiling_management/__init__.py | 3 + gprofiler/main.py | 13 +- gprofiler/nsys_profiler.py | 402 ++++++++++++++++-- tests/test_nsys_profiler.py | 100 +++++ 5 files changed, 491 insertions(+), 47 deletions(-) diff --git a/docs/NSYS_GPU_PROFILING.md b/docs/NSYS_GPU_PROFILING.md index 6b5f2f692..2ca3e5962 100644 --- a/docs/NSYS_GPU_PROFILING.md +++ b/docs/NSYS_GPU_PROFILING.md @@ -45,6 +45,26 @@ When `enable_nsys` succeeds, the uploaded `flamegraph_html` is the **GPU** flamegraph (preferred over CPU). `run_arguments.perf_events` also gains `nsys-cuda` so the Adhoc UI can show a **GPU / nsys** chip. +## CPU/GPU timeline mode (`--nsys-timeline` / `nsys_timeline`) + +The same capture (`-t cuda,nvtx -s none`) already records both sides with +timestamps: `cuda_gpu_trace` (per-kernel Start/Duration/Device/Stream) and +`cuda_api_trace` (per-call Start/Duration/Pid/Tid — `-s none` only disables CPU +*stack sampling*, CUDA API tracing stays on). Both share one nsys clock and a +`CorrID` linking each CPU-side launch to the GPU kernel it produced. + +With `--nsys-timeline` (CLI) or `combined_config.nsys_timeline: true` +(heartbeat, alongside `enable_nsys`), the agent exports those two trace reports +instead of the kern/api summaries and uploads a **self-contained timeline HTML** +through the same adhoc path: swim lanes per CPU thread issuing CUDA calls and +per GPU device/stream, wheel-zoom / drag-pan, and click-to-highlight CorrID so a +`cudaLaunchKernel` and its kernel light up together. If the trace export yields +no events, the agent falls back to the GPU flamegraph. `perf_events` also gains +`nsys-timeline` (next to `nsys-cuda`) so the UI can distinguish the view. + +Captures with more than 20,000 events keep the longest ones (the HTML notes +"showing N of M") to bound upload size. + ## Kind / sandbox topology Kind runs the Studio control plane only. The agent that invokes nsys must run on diff --git a/gprofiler/dynamic_profiling_management/__init__.py b/gprofiler/dynamic_profiling_management/__init__.py index b32886594..5f9bb6ea3 100644 --- a/gprofiler/dynamic_profiling_management/__init__.py +++ b/gprofiler/dynamic_profiling_management/__init__.py @@ -116,6 +116,7 @@ def create_profiler_args( if found is not None: new_args.nsys_path = str(found) new_args.nsys_workload = combined_config.get("nsys_workload") + new_args.nsys_timeline = bool(combined_config.get("nsys_timeline", False)) logger.info(f"enable_nsys: using nsys at {found}") else: logger.error( @@ -138,6 +139,8 @@ def create_profiler_args( events = [e.strip() for e in str(existing).split(",") if e.strip()] if "nsys-cuda" not in events: events.append("nsys-cuda") + if getattr(new_args, "nsys_timeline", False) and "nsys-timeline" not in events: + events.append("nsys-timeline") new_args.perf_events = ",".join(events) return new_args diff --git a/gprofiler/main.py b/gprofiler/main.py index e7cec5320..faa56fa06 100644 --- a/gprofiler/main.py +++ b/gprofiler/main.py @@ -172,6 +172,7 @@ def __init__( self._enable_nsys = bool(user_args.get("enable_nsys", False)) self._nsys_path = user_args.get("nsys_path") self._nsys_workload = user_args.get("nsys_workload") + self._nsys_timeline = bool(user_args.get("nsys_timeline", False)) self._nsys_thread: Optional[threading.Thread] = None self._nsys_html: Optional[str] = None if self._collect_metadata: @@ -413,6 +414,7 @@ def _gen(collapsed: str) -> Optional[str]: workload=self._nsys_workload, stop_event=self._profiler_state.stop_event, generate_html_fn=_gen, + timeline=self._nsys_timeline, ) self._nsys_html = html except Exception: @@ -566,7 +568,8 @@ def _snapshot(self) -> None: logger.info("Waiting for background nsys GPU capture to finish...") self._nsys_thread.join(timeout=max(60, self._duration + 120)) if self._nsys_html: - logger.info("Using nsys GPU flamegraph HTML for upload (preferred over CPU)") + kind = "CPU/GPU timeline" if self._nsys_timeline else "GPU flamegraph" + logger.info(f"Using nsys {kind} HTML for upload (preferred over CPU)") flamegraph_html = self._nsys_html else: logger.warning("enable_nsys was set but no GPU HTML was produced; keeping CPU flamegraph if any") @@ -1339,6 +1342,14 @@ def parse_cmd_args() -> configargparse.Namespace: help="Command line for nsys to wrap (e.g. '/path/to/cuda_burn 30'). " "Recommended for useful CUDA kernel frames.", ) + nsys_options.add_argument( + "--nsys-timeline", + action="store_true", + default=False, + dest="nsys_timeline", + help="With --enable-nsys, upload a CPU/GPU timeline (cuda_gpu_trace + " + "cuda_api_trace swim lanes, CorrID-linked) instead of the GPU flamegraph.", + ) args = parser.parse_args() diff --git a/gprofiler/nsys_profiler.py b/gprofiler/nsys_profiler.py index 8de0988e5..720c9c7f4 100644 --- a/gprofiler/nsys_profiler.py +++ b/gprofiler/nsys_profiler.py @@ -243,56 +243,61 @@ def _csv_to_collapsed(csv_text: str, frame_prefix: str) -> Optional[str]: return "\n".join(lines) if lines else None +def _export_stats_csv(nsys: Path, nsys_rep: Path, work_dir: Path, report: str, out_prefix: Path) -> Optional[str]: + """Run `nsys stats --report= --format=csv` and return the CSV text, or None.""" + cmd = [ + str(nsys), + "stats", + f"--report={report}", + "--format=csv", + "--force-export=true", + "-o", + str(out_prefix), + str(nsys_rep), + ] + try: + proc = subprocess.run( # nosec B603 + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=120, + check=False, + ) + if proc.returncode != 0: + logger.warning( + "nsys stats %s failed (rc=%s): %s", + report, + proc.returncode, + (proc.stdout or b"")[-1000:].decode("utf-8", "replace"), + ) + except (subprocess.TimeoutExpired, OSError) as exc: + logger.warning("nsys stats %s error: %s", report, exc) + return None + + matches = sorted(work_dir.glob(out_prefix.name + "*.csv")) + # Also check CWD-relative names nsys sometimes writes + matches += sorted(Path(".").glob(out_prefix.name + "*.csv")) + matches += sorted(nsys_rep.parent.glob(out_prefix.name + "*.csv")) + # Dedup + seen = set() + unique = [] + for m in matches: + rp = str(m.resolve()) if m.exists() else str(m) + if rp not in seen and m.is_file() and m.stat().st_size > 0: + seen.add(rp) + unique.append(m) + if not unique: + logger.info("nsys stats %s produced no non-empty CSV", report) + return None + return unique[0].read_text(encoding="utf-8", errors="replace") + + def nsys_stats_to_collapsed(nsys: Path, nsys_rep: Path, work_dir: Path) -> Optional[str]: """Export cuda_gpu_kern_sum (fallback cuda_api_sum) and convert to collapsed stacks.""" work_dir.mkdir(parents=True, exist_ok=True) def _export(report: str, out_prefix: Path) -> Optional[str]: - cmd = [ - str(nsys), - "stats", - f"--report={report}", - "--format=csv", - "--force-export=true", - "-o", - str(out_prefix), - str(nsys_rep), - ] - try: - proc = subprocess.run( # nosec B603 - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=120, - check=False, - ) - if proc.returncode != 0: - logger.warning( - "nsys stats %s failed (rc=%s): %s", - report, - proc.returncode, - (proc.stdout or b"")[-1000:].decode("utf-8", "replace"), - ) - except (subprocess.TimeoutExpired, OSError) as exc: - logger.warning("nsys stats %s error: %s", report, exc) - return None - - matches = sorted(work_dir.glob(out_prefix.name + "*.csv")) - # Also check CWD-relative names nsys sometimes writes - matches += sorted(Path(".").glob(out_prefix.name + "*.csv")) - matches += sorted(nsys_rep.parent.glob(out_prefix.name + "*.csv")) - # Dedup - seen = set() - unique = [] - for m in matches: - rp = str(m.resolve()) if m.exists() else str(m) - if rp not in seen and m.is_file() and m.stat().st_size > 0: - seen.add(rp) - unique.append(m) - if not unique: - logger.info("nsys stats %s produced no non-empty CSV", report) - return None - return unique[0].read_text(encoding="utf-8", errors="replace") + return _export_stats_csv(nsys, nsys_rep, work_dir, report, out_prefix) kern_csv = _export("cuda_gpu_kern_sum", work_dir / "kern") if kern_csv: @@ -312,6 +317,296 @@ def _export(report: str, out_prefix: Path) -> Optional[str]: return None +# --- CPU/GPU timeline (cuda_gpu_trace + cuda_api_trace) --------------------- + +# Self-contained timeline HTML gets large fast; keep the longest events if the +# capture has more than this many. +MAX_TIMELINE_EVENTS = 20000 + + +def _find_column(fieldnames: List[str], *needles: str) -> Optional[str]: + """First column whose lowercase name contains any needle (checked in order).""" + lowered = [(f, f.lower()) for f in fieldnames] + for needle in needles: + for original, low in lowered: + if needle in low: + return original + return None + + +def _parse_trace_csv(csv_text: str, kind: str) -> List[dict]: + """Parse an nsys cuda_gpu_trace / cuda_api_trace CSV into timeline events. + + kind is "gpu" or "api". Returns events with keys: + start (ns), dur (ns), name, corr (int|None), lane (str). + GPU lanes group by device+stream; API lanes group by pid/tid. + """ + reader = csv.DictReader(io.StringIO(csv_text)) + if not reader.fieldnames: + return [] + fieldnames = [f.strip().lstrip("\ufeff") for f in reader.fieldnames] + reader.fieldnames = fieldnames + + start_key = _find_column(fieldnames, "start (ns)", "start") + dur_key = _find_column(fieldnames, "duration (ns)", "duration") + name_key = next((f for f in fieldnames if f.lower() in ("name", "kernel name", "kernel")), None) + corr_key = _find_column(fieldnames, "corrid") + if start_key is None or dur_key is None or name_key is None: + logger.warning("nsys %s trace CSV missing start/duration/name columns: %s", kind, fieldnames) + return [] + + device_key = stream_key = pid_key = tid_key = None + if kind == "gpu": + device_key = _find_column(fieldnames, "device") + stream_key = _find_column(fieldnames, "strm", "stream") + else: + pid_key = _find_column(fieldnames, "pid") + tid_key = _find_column(fieldnames, "tid") + + events: List[dict] = [] + for row in reader: + cleaned = {(k or "").strip().lstrip("\ufeff"): (v or "").strip() for k, v in row.items()} + name = cleaned.get(name_key, "").strip().strip('"') + if not name: + continue + try: + start = int(float(cleaned.get(start_key, ""))) + dur = max(0, int(float(cleaned.get(dur_key, "")))) + except ValueError: + continue + corr: Optional[int] = None + if corr_key: + try: + corr = int(float(cleaned.get(corr_key, ""))) + except ValueError: + corr = None + if kind == "gpu": + device = cleaned.get(device_key, "") if device_key else "" + stream = cleaned.get(stream_key, "") if stream_key else "" + lane = f"GPU {device or '?'} stream {stream or '?'}" + else: + pid = cleaned.get(pid_key, "") if pid_key else "" + tid = cleaned.get(tid_key, "") if tid_key else "" + lane = f"CPU pid {pid or '?'} tid {tid or '?'}" + events.append({"start": start, "dur": dur, "name": name, "corr": corr, "lane": lane}) + return events + + +def nsys_trace_to_timeline_events(nsys: Path, nsys_rep: Path, work_dir: Path) -> Optional[dict]: + """Export cuda_gpu_trace + cuda_api_trace and parse them into timeline events. + + Returns {"gpu": [...], "api": [...]} (either list may be empty), or None if + neither trace produced events. + """ + work_dir.mkdir(parents=True, exist_ok=True) + + gpu_events: List[dict] = [] + api_events: List[dict] = [] + + gpu_csv = _export_stats_csv(nsys, nsys_rep, work_dir, "cuda_gpu_trace", work_dir / "gpu_trace") + if gpu_csv: + gpu_events = _parse_trace_csv(gpu_csv, "gpu") + api_csv = _export_stats_csv(nsys, nsys_rep, work_dir, "cuda_api_trace", work_dir / "api_trace") + if api_csv: + api_events = _parse_trace_csv(api_csv, "api") + + if not gpu_events and not api_events: + logger.error("No usable nsys trace CSV (cuda_gpu_trace / cuda_api_trace)") + return None + logger.info( + "Parsed nsys timeline traces: %d GPU events, %d CUDA API events", len(gpu_events), len(api_events) + ) + return {"gpu": gpu_events, "api": api_events} + + +def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timeline") -> Optional[str]: + """Render timeline events as a self-contained HTML swim-lane view. + + Lanes: one per CPU thread issuing CUDA API calls, one per GPU device/stream. + CorrID links a CPU-side launch to the GPU kernel it produced (click to + highlight). No external assets; suitable for the Studio Adhoc iframe. + """ + import json + + all_events: List[dict] = [] + for kind in ("api", "gpu"): + for ev in events.get(kind, []): + all_events.append({**ev, "kind": kind}) + if not all_events: + return None + + total_events = len(all_events) + truncated = total_events > MAX_TIMELINE_EVENTS + if truncated: + all_events.sort(key=lambda e: -e["dur"]) + all_events = all_events[:MAX_TIMELINE_EVENTS] + + t0 = min(e["start"] for e in all_events) + span = max(1, max(e["start"] + e["dur"] for e in all_events) - t0) + all_events.sort(key=lambda e: e["start"]) + + # CPU lanes first, then GPU lanes, each sorted by name for stable order. + lane_names = sorted({e["lane"] for e in all_events if e["kind"] == "api"}) + sorted( + {e["lane"] for e in all_events if e["kind"] == "gpu"} + ) + lane_index = {name: i for i, name in enumerate(lane_names)} + + name_table: List[str] = [] + name_index: dict = {} + packed = [] + for e in all_events: + idx = name_index.get(e["name"]) + if idx is None: + idx = len(name_table) + name_index[e["name"]] = idx + name_table.append(e["name"]) + corr = e["corr"] if e["corr"] is not None else -1 + packed.append([e["start"] - t0, e["dur"], lane_index[e["lane"]], corr, idx]) + + data = { + "lanes": lane_names, + "cpuLanes": sum(1 for n in lane_names if n.startswith("CPU")), + "names": name_table, + "events": packed, + "span": span, + "total": total_events, + "shown": len(all_events), + } + data_json = json.dumps(data, separators=(",", ":")) + + template = """ +__TITLE__ + +

__TITLE__

+

+
+ + +""" + return template.replace("__TITLE__", title).replace("__DATA__", data_json) + + def _simple_gpu_flamegraph_html(collapsed: str, title: str = "nsys GPU profile") -> str: """Minimal self-contained HTML for Adhoc iframe when burn/template unavailable.""" rows = [] @@ -416,8 +711,14 @@ def collect_nsys_adhoc_html( work_dir: Optional[str] = None, stop_event=None, generate_html_fn: Optional[Callable[[str], Optional[str]]] = None, + timeline: bool = False, ) -> Optional[str]: - """End-to-end: find nsys → capture → collapsed → HTML. Returns HTML or None.""" + """End-to-end: find nsys → capture → collapsed → HTML. Returns HTML or None. + + With timeline=True the same capture is exported as cuda_gpu_trace + + cuda_api_trace and rendered as a CPU/GPU timeline instead of a flamegraph + (falling back to the flamegraph if the trace export yields no events). + """ nsys = find_nsys(nsys_path) if nsys is None: logger.error( @@ -448,6 +749,15 @@ def collect_nsys_adhoc_html( if rep is None: return None + if timeline: + events = nsys_trace_to_timeline_events(nsys, rep, base) + if events: + html = generate_nsys_timeline_html(events) + if html: + logger.info("Generated nsys CPU/GPU timeline HTML (%d bytes)", len(html)) + return html + logger.warning("nsys timeline requested but no trace events; falling back to GPU flamegraph") + collapsed = nsys_stats_to_collapsed(nsys, rep, base) if not collapsed: return None diff --git a/tests/test_nsys_profiler.py b/tests/test_nsys_profiler.py index f05a13d64..89d190457 100644 --- a/tests/test_nsys_profiler.py +++ b/tests/test_nsys_profiler.py @@ -25,9 +25,12 @@ from gprofiler.nsys_profiler import ( _csv_to_collapsed, + _parse_trace_csv, _workload_env, find_nsys, + generate_nsys_timeline_html, nsys_stats_to_collapsed, + nsys_trace_to_timeline_events, ) @@ -40,6 +43,19 @@ 17.9,28943134,5,5788626.8,5785451.0,5783190,5801941,7724.7,cudaDeviceSynchronize """ +GPU_TRACE_CSV = ( + "Start (ns),Duration (ns),CorrId,GrdX,GrdY,GrdZ,BlkX,BlkY,BlkZ,Reg/Trd," + "StcSMem (MB),DymSMem (MB),Bytes (MB),Throughput (MBps),SrcMemKd,DstMemKd,Device,Ctx,Strm,Name\n" + '1000,5000,101,160,1,1,256,1,1,32,0.000,0.000,,,,,NVIDIA A10G (0),1,7,"burn(float *, int, int)"\n' + '9000,4000,102,160,1,1,256,1,1,32,0.000,0.000,,,,,NVIDIA A10G (0),1,7,"burn(float *, int, int)"\n' +) + +API_TRACE_CSV = """Start (ns),Duration (ns),Name,Result,CorrID,Pid,Tid,T-Pri,Thread Name +500,300,cudaLaunchKernel,0,101,4242,4242,20,python3 +8600,250,cudaLaunchKernel,0,102,4242,4242,20,python3 +14000,2000,cudaDeviceSynchronize,0,103,4242,4242,20,python3 +""" + def test_csv_to_collapsed_kern(): collapsed = _csv_to_collapsed(KERN_CSV, "gpu;nsys;cuda_kernel") @@ -147,3 +163,87 @@ def fake_run(cmd, **kwargs): collapsed = nsys_stats_to_collapsed(nsys, rep, tmp_path / "work") assert collapsed is not None assert "cuda_kernel;burn" in collapsed + + +def test_parse_trace_csv_gpu(): + events = _parse_trace_csv(GPU_TRACE_CSV, "gpu") + assert len(events) == 2 + first = events[0] + assert first["start"] == 1000 + assert first["dur"] == 5000 + assert first["corr"] == 101 + assert first["name"] == "burn(float *, int, int)" + assert first["lane"] == "GPU NVIDIA A10G (0) stream 7" + + +def test_parse_trace_csv_api(): + events = _parse_trace_csv(API_TRACE_CSV, "api") + assert len(events) == 3 + launch = events[0] + assert launch["name"] == "cudaLaunchKernel" + assert launch["corr"] == 101 + assert launch["lane"] == "CPU pid 4242 tid 4242" + + +def test_parse_trace_csv_missing_columns(): + assert _parse_trace_csv("Foo,Bar\n1,2\n", "gpu") == [] + assert _parse_trace_csv("", "api") == [] + + +def test_generate_timeline_html_links_corrid(): + events = { + "gpu": _parse_trace_csv(GPU_TRACE_CSV, "gpu"), + "api": _parse_trace_csv(API_TRACE_CSV, "api"), + } + html = generate_nsys_timeline_html(events) + assert html is not None + assert html.startswith("") + # CPU lanes listed before GPU lanes; both present + assert "CPU pid 4242 tid 4242" in html + assert "GPU NVIDIA A10G (0) stream 7" in html + # kernel + API names present exactly once each (name table dedup) + assert html.count("burn(float *, int, int)") == 1 + assert html.count("cudaLaunchKernel") == 1 + # correlation ids embedded for CPU<->GPU linking + assert "101" in html and "102" in html + # no external assets — must render standalone in the Studio iframe + assert "http://" not in html and "https://" not in html + + +def test_generate_timeline_html_empty(): + assert generate_nsys_timeline_html({"gpu": [], "api": []}) is None + + +def test_nsys_trace_to_timeline_events(tmp_path: Path, monkeypatch): + nsys = tmp_path / "nsys" + nsys.write_text("#!/bin/sh\n") + nsys.chmod(0o755) + rep = tmp_path / "cap.nsys-rep" + rep.write_bytes(b"fake") + + def fake_run(cmd, **kwargs): + out_prefix = Path(cmd[cmd.index("-o") + 1]) + report = next(a for a in cmd if a.startswith("--report=")).split("=", 1)[1] + csv_text = GPU_TRACE_CSV if report == "cuda_gpu_trace" else API_TRACE_CSV + Path(str(out_prefix) + f"_{report}.csv").write_text(csv_text) + return mock.Mock(returncode=0, stdout=b"ok") + + monkeypatch.setattr("gprofiler.nsys_profiler.subprocess.run", fake_run) + events = nsys_trace_to_timeline_events(nsys, rep, tmp_path / "work") + assert events is not None + assert len(events["gpu"]) == 2 + assert len(events["api"]) == 3 + + +def test_nsys_trace_to_timeline_events_none_when_empty(tmp_path: Path, monkeypatch): + nsys = tmp_path / "nsys" + nsys.write_text("#!/bin/sh\n") + nsys.chmod(0o755) + rep = tmp_path / "cap.nsys-rep" + rep.write_bytes(b"fake") + + def fake_run(cmd, **kwargs): + return mock.Mock(returncode=1, stdout=b"no data") + + monkeypatch.setattr("gprofiler.nsys_profiler.subprocess.run", fake_run) + assert nsys_trace_to_timeline_events(nsys, rep, tmp_path / "work") is None From e9ee026bfa7855789e4742a819c869a689bebecd Mon Sep 17 00:00:00 2001 From: prashantbytesyntax Date: Fri, 14 Aug 2026 07:58:47 +0000 Subject: [PATCH 06/11] Make the timeline readable at full zoom-out At full span most captures collapsed into solid bars: every sub-pixel event was drawn at a 1px minimum, so a fully-packed lane and a half-idle lane looked identical. Two changes: - Open auto-zoomed to a window where the median event is a few pixels wide, centered mid-capture, with a "Full span" button to reset. - At low zoom, sub-pixel events accumulate per-pixel occupancy and shade the lane by how busy it actually is, instead of tiling opaque 1px bars. Verified in headless Chromium: a lane busy for half the capture now renders visibly brighter on the busy half (mean red 224 vs 24) at full span. Co-Authored-By: Claude Fable 5 --- docs/NSYS_GPU_PROFILING.md | 5 ++++ gprofiler/nsys_profiler.py | 55 ++++++++++++++++++++++++++++++++----- tests/test_nsys_profiler.py | 4 +++ 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/docs/NSYS_GPU_PROFILING.md b/docs/NSYS_GPU_PROFILING.md index 2ca3e5962..2c67856eb 100644 --- a/docs/NSYS_GPU_PROFILING.md +++ b/docs/NSYS_GPU_PROFILING.md @@ -65,6 +65,11 @@ no events, the agent falls back to the GPU flamegraph. `perf_events` also gains Captures with more than 20,000 events keep the longest ones (the HTML notes "showing N of M") to bound upload size. +The view opens auto-zoomed to a window where the median event is a few pixels +wide (a "Full span" button resets), and at low zoom sub-pixel events shade the +lane by occupancy instead of tiling solid 1px bars — so a fully-packed lane +looks different from a half-idle one even when zoomed out. + ## Kind / sandbox topology Kind runs the Studio control plane only. The agent that invokes nsys must run on diff --git a/gprofiler/nsys_profiler.py b/gprofiler/nsys_profiler.py index 720c9c7f4..30d4bf55d 100644 --- a/gprofiler/nsys_profiler.py +++ b/gprofiler/nsys_profiler.py @@ -501,11 +501,32 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin if (ns >= 1e3) return (ns / 1e3).toFixed(1) + ' us'; return ns + ' ns'; } -document.getElementById('meta').textContent = +// Open auto-zoomed so the median event is a few pixels wide; at full span +// most captures collapse into a solid smear. "0" or the button resets. +let autoZoomed = false; +function initView() { + const durs = DATA.events.map((e) => e[1]).filter((d) => d > 0).sort((a, b) => a - b); + if (!durs.length) return; + const med = durs[Math.floor(durs.length / 2)]; + const plotW = (canvas.clientWidth || 800) - LABEL_W; + const target = Math.max(1000, med * plotW / 6); + if (target < DATA.span * 0.9) { + viewSpan = target; + viewStart = Math.max(0, (DATA.span - viewSpan) / 2); + autoZoomed = true; + } +} +function fullSpan() { viewStart = 0; viewSpan = DATA.span; draw(); } +initView(); +document.getElementById('meta').innerHTML = 'CUDA API calls (CPU threads) + GPU kernels/memops from nsys cuda_api_trace / cuda_gpu_trace. ' + 'Span ' + fmtNs(DATA.span) + '. Showing ' + DATA.shown + ' of ' + DATA.total + ' events' + (DATA.shown < DATA.total ? ' (longest kept)' : '') + - '. Wheel: zoom - drag: pan - click: highlight CorrID (CPU launch <-> GPU kernel).'; + '. Wheel: zoom - drag: pan - click: highlight CorrID (CPU launch <-> GPU kernel)' + + (autoZoomed ? '. Auto-zoomed to the middle of the capture; sub-pixel events fade by lane occupancy at low zoom. ' : '. ') + + ''; +document.getElementById('fit').addEventListener('click', fullSpan); function draw() { const cssW = canvas.clientWidth || 800; const dpr = window.devicePixelRatio || 1; @@ -533,15 +554,35 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin ctx.fillText(fmtNs(t), Math.min(x, cssW - 60), 12); } // events + // Sub-pixel events don't get a solid 1px bar each (thousands of them tile + // into a misleading solid strip); instead they accumulate per-pixel + // occupancy and the column is shaded by how busy the lane actually is. + const pxT = viewSpan / plotW; // time units per pixel + const density = DATA.lanes.map(() => new Float32Array(Math.max(1, Math.ceil(plotW)))); for (const [s, d, lane, corr, nameIdx] of DATA.events) { if (s + d < viewStart || s > viewStart + viewSpan) continue; const x = LABEL_W + (s - viewStart) / viewSpan * plotW; - const w = Math.max(1, d / viewSpan * plotW); + const w = d / viewSpan * plotW; const y = AXIS_H + lane * LANE_H + 3; - const isCpu = lane < DATA.cpuLanes; - if (selCorr >= 0 && corr === selCorr) ctx.fillStyle = '#f5e663'; - else ctx.fillStyle = isCpu ? '#5c8ae6' : '#e6a15c'; - ctx.fillRect(x, y, w, LANE_H - 6); + const isSel = selCorr >= 0 && corr === selCorr; + if (w < 1 && !isSel) { + const col = Math.min(density[lane].length - 1, Math.max(0, Math.floor(x - LABEL_W))); + density[lane][col] = Math.min(1, density[lane][col] + Math.max(0.05, w)); + continue; + } + if (isSel) ctx.fillStyle = '#f5e663'; + else ctx.fillStyle = lane < DATA.cpuLanes ? '#5c8ae6' : '#e6a15c'; + ctx.fillRect(x, y, Math.max(1, w), LANE_H - 6); + } + for (let lane = 0; lane < density.length; lane++) { + const col = density[lane]; + const y = AXIS_H + lane * LANE_H + 3; + const rgb = lane < DATA.cpuLanes ? '92,138,230' : '230,161,92'; + for (let i = 0; i < col.length; i++) { + if (col[i] <= 0) continue; + ctx.fillStyle = 'rgba(' + rgb + ',' + (0.25 + 0.75 * col[i]).toFixed(2) + ')'; + ctx.fillRect(LABEL_W + i, y, 1, LANE_H - 6); + } } } function hit(mx, my) { diff --git a/tests/test_nsys_profiler.py b/tests/test_nsys_profiler.py index 89d190457..fb68c18e0 100644 --- a/tests/test_nsys_profiler.py +++ b/tests/test_nsys_profiler.py @@ -208,6 +208,10 @@ def test_generate_timeline_html_links_corrid(): assert "101" in html and "102" in html # no external assets — must render standalone in the Studio iframe assert "http://" not in html and "https://" not in html + # auto-zoom on load + density shading for sub-pixel events + assert "initView()" in html + assert "Full span" in html + assert "density" in html def test_generate_timeline_html_empty(): From ba99b14a958a08cec682463c326c10430e04bea6 Mon Sep 17 00:00:00 2001 From: prashantbytesyntax Date: Fri, 14 Aug 2026 20:08:34 +0000 Subject: [PATCH 07/11] Add click-for-stack CPU backtraces to the nsys timeline --nsys-timeline-stacks / combined_config.nsys_timeline_stacks captures with --cudabacktrace=kernel -s process-tree -b dwarf (opt-in; heavier than the default -s none timeline), exports the report to SQLite, and joins CUPTI_ACTIVITY_KIND_RUNTIME.callchainId -> CUDA_CALLCHAINS -> StringIds into a deduped stacks table in the timeline HTML. Clicking an event opens a panel with the launching CPU backtrace; GPU kernels resolve their launch stack through CorrID. Timelines without callchains render unchanged. Co-Authored-By: Claude Fable 5 --- docs/NSYS_GPU_PROFILING.md | 22 ++ .../dynamic_profiling_management/__init__.py | 3 + gprofiler/main.py | 11 + gprofiler/nsys_profiler.py | 212 +++++++++++++++++- tests/test_nsys_profiler.py | 140 ++++++++++++ 5 files changed, 376 insertions(+), 12 deletions(-) diff --git a/docs/NSYS_GPU_PROFILING.md b/docs/NSYS_GPU_PROFILING.md index 2c67856eb..87960e912 100644 --- a/docs/NSYS_GPU_PROFILING.md +++ b/docs/NSYS_GPU_PROFILING.md @@ -70,6 +70,28 @@ wide (a "Full span" button resets), and at low zoom sub-pixel events shade the lane by occupancy instead of tiling solid 1px bars — so a fully-packed lane looks different from a half-idle one even when zoomed out. +### Click-for-stack mode (`--nsys-timeline-stacks` / `nsys_timeline_stacks`) + +With `--nsys-timeline-stacks` (CLI) or `combined_config.nsys_timeline_stacks: +true` (heartbeat, alongside `nsys_timeline`), the capture adds +`--cudabacktrace=kernel -s process-tree -b dwarf` so nsys records a CPU +backtrace at each kernel-launching CUDA API call (`--cudabacktrace` requires +CPU sampling, so this mode gives up `-s none`; NVIDIA warns of significant +runtime overhead — keep it for deep dives, not the default timeline). Launches +shorter than the nsys default 1µs threshold get no backtrace. + +The backtraces are not in any `nsys stats` report; the agent additionally runs +`nsys export --type sqlite` and joins `CUPTI_ACTIVITY_KIND_RUNTIME.callchainId` +→ `CUDA_CALLCHAINS` → `StringIds` (stdlib `sqlite3`, no new dependency). +Identical callchains are deduped into a `stacks` table in the HTML payload, so +thousands of events typically add only a few hundred distinct stacks. Clicking +an event opens a panel under the timeline with the launch's stack, innermost +frame first; clicking a **GPU kernel** shows the stack of the CPU call that +launched it (resolved through CorrID). Frames are native C++ symbols +(libtorch/aten/libcudart), not Python lines. `perf_events` gains `nsys-stacks`. +If the SQLite export has no callchains, the timeline still renders — clicks +just highlight CorrIDs as before. + ## Kind / sandbox topology Kind runs the Studio control plane only. The agent that invokes nsys must run on diff --git a/gprofiler/dynamic_profiling_management/__init__.py b/gprofiler/dynamic_profiling_management/__init__.py index 5f9bb6ea3..9f276ea8e 100644 --- a/gprofiler/dynamic_profiling_management/__init__.py +++ b/gprofiler/dynamic_profiling_management/__init__.py @@ -117,6 +117,7 @@ def create_profiler_args( new_args.nsys_path = str(found) new_args.nsys_workload = combined_config.get("nsys_workload") new_args.nsys_timeline = bool(combined_config.get("nsys_timeline", False)) + new_args.nsys_timeline_stacks = bool(combined_config.get("nsys_timeline_stacks", False)) logger.info(f"enable_nsys: using nsys at {found}") else: logger.error( @@ -141,6 +142,8 @@ def create_profiler_args( events.append("nsys-cuda") if getattr(new_args, "nsys_timeline", False) and "nsys-timeline" not in events: events.append("nsys-timeline") + if getattr(new_args, "nsys_timeline_stacks", False) and "nsys-stacks" not in events: + events.append("nsys-stacks") new_args.perf_events = ",".join(events) return new_args diff --git a/gprofiler/main.py b/gprofiler/main.py index faa56fa06..dde35ade4 100644 --- a/gprofiler/main.py +++ b/gprofiler/main.py @@ -173,6 +173,7 @@ def __init__( self._nsys_path = user_args.get("nsys_path") self._nsys_workload = user_args.get("nsys_workload") self._nsys_timeline = bool(user_args.get("nsys_timeline", False)) + self._nsys_timeline_stacks = bool(user_args.get("nsys_timeline_stacks", False)) self._nsys_thread: Optional[threading.Thread] = None self._nsys_html: Optional[str] = None if self._collect_metadata: @@ -415,6 +416,7 @@ def _gen(collapsed: str) -> Optional[str]: stop_event=self._profiler_state.stop_event, generate_html_fn=_gen, timeline=self._nsys_timeline, + timeline_stacks=self._nsys_timeline_stacks, ) self._nsys_html = html except Exception: @@ -1350,6 +1352,15 @@ def parse_cmd_args() -> configargparse.Namespace: help="With --enable-nsys, upload a CPU/GPU timeline (cuda_gpu_trace + " "cuda_api_trace swim lanes, CorrID-linked) instead of the GPU flamegraph.", ) + nsys_options.add_argument( + "--nsys-timeline-stacks", + action="store_true", + default=False, + dest="nsys_timeline_stacks", + help="With --nsys-timeline, also record a CPU backtrace per kernel launch " + "(--cudabacktrace=kernel; enables CPU sampling — noticeably heavier) and " + "show it when an event is clicked in the timeline.", + ) args = parser.parse_args() diff --git a/gprofiler/nsys_profiler.py b/gprofiler/nsys_profiler.py index 30d4bf55d..5db6c96c4 100644 --- a/gprofiler/nsys_profiler.py +++ b/gprofiler/nsys_profiler.py @@ -121,8 +121,14 @@ def run_nsys_capture( duration_sec: int, workload_cmd: Optional[Sequence[str]] = None, stop_event=None, + backtraces: bool = False, ) -> Optional[Path]: - """Run `nsys profile` and return the path to the `.nsys-rep` file, or None.""" + """Run `nsys profile` and return the path to the `.nsys-rep` file, or None. + + backtraces=True records a CPU backtrace per kernel-launching CUDA API call + (--cudabacktrace needs CPU sampling on, so it swaps -s none for + process-tree sampling — measurably heavier; keep it opt-in). + """ output_prefix.parent.mkdir(parents=True, exist_ok=True) duration_sec = max(1, int(duration_sec)) @@ -132,12 +138,14 @@ def run_nsys_capture( "-o", str(output_prefix), "--force-overwrite=true", - # CUDA-focused, skip heavy CPU sampling / slow symbol waits where possible. "-t", "cuda,nvtx", - "-s", - "none", ] + if backtraces: + cmd.extend(["-s", "process-tree", "-b", "dwarf", "--cudabacktrace=kernel"]) + else: + # CUDA-focused, skip heavy CPU sampling / slow symbol waits where possible. + cmd.extend(["-s", "none"]) workload = list(workload_cmd) if workload_cmd else None if workload: @@ -392,11 +400,125 @@ def _parse_trace_csv(csv_text: str, kind: str) -> List[dict]: return events -def nsys_trace_to_timeline_events(nsys: Path, nsys_rep: Path, work_dir: Path) -> Optional[dict]: +def nsys_export_sqlite(nsys: Path, nsys_rep: Path, work_dir: Path) -> Optional[Path]: + """Run `nsys export --type sqlite` and return the .sqlite path, or None.""" + out = work_dir / (nsys_rep.stem + ".sqlite") + cmd = [ + str(nsys), + "export", + "--type", + "sqlite", + "--force-overwrite", + "true", + "--output", + str(out), + str(nsys_rep), + ] + try: + proc = subprocess.run( # nosec B603 + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=300, + check=False, + ) + if proc.returncode != 0: + logger.warning( + "nsys export sqlite failed (rc=%s): %s", + proc.returncode, + (proc.stdout or b"")[-1000:].decode("utf-8", "replace"), + ) + return None + except (subprocess.TimeoutExpired, OSError) as exc: + logger.warning("nsys export sqlite error: %s", exc) + return None + if not out.is_file() or out.stat().st_size == 0: + logger.warning("nsys export sqlite produced no file at %s", out) + return None + return out + + +def load_callchains_from_sqlite(sqlite_path: Path) -> tuple: + """Read CUDA API backtraces from an nsys SQLite export. + + Returns (corr_to_stack, stacks): correlationId → index into stacks, where + each stack is a list of "symbol (module)" frames, innermost first. Stacks + are deduped — CUDA_CALLCHAINS rows are shared across API calls already, + and identical symbol sequences from different callchain ids collapse too. + Empty results (no --cudabacktrace in the capture, or old schema) are not + an error: ({}, []). + """ + import sqlite3 + + corr_to_stack: dict = {} + stacks: List[List[str]] = [] + try: + conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + try: + tables = { + row[0] + for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'") + } + required = {"CUPTI_ACTIVITY_KIND_RUNTIME", "CUDA_CALLCHAINS", "StringIds"} + if not required.issubset(tables): + logger.info( + "nsys sqlite export lacks callchain tables (missing %s); no stacks", + ",".join(sorted(required - tables)), + ) + return {}, [] + corr_to_chain = dict( + conn.execute( + "SELECT correlationId, callchainId FROM CUPTI_ACTIVITY_KIND_RUNTIME " + "WHERE callchainId IS NOT NULL" + ).fetchall() + ) + frame_rows = conn.execute( + "SELECT c.id, c.stackDepth, COALESCE(s.value, '?'), COALESCE(m.value, '') " + "FROM CUDA_CALLCHAINS c " + "LEFT JOIN StringIds s ON s.id = c.symbol " + "LEFT JOIN StringIds m ON m.id = c.module " + "ORDER BY c.id, c.stackDepth" + ).fetchall() + finally: + conn.close() + except sqlite3.Error as exc: + logger.warning("Failed to read callchains from %s: %s", sqlite_path, exc) + return {}, [] + + chain_frames: dict = {} + for chain_id, depth, symbol, module in frame_rows: + frames = chain_frames.setdefault(chain_id, []) + frames.append((depth, f"{symbol} ({module})" if module else str(symbol))) + + stack_index: dict = {} + chain_to_stack: dict = {} + for chain_id, frames in chain_frames.items(): + ordered = [f for _, f in sorted(frames, key=lambda x: x[0])] + key = tuple(ordered) + idx = stack_index.get(key) + if idx is None: + idx = len(stacks) + stack_index[key] = idx + stacks.append(ordered) + chain_to_stack[chain_id] = idx + + for corr, chain_id in corr_to_chain.items(): + idx = chain_to_stack.get(chain_id) + if idx is not None: + corr_to_stack[int(corr)] = idx + return corr_to_stack, stacks + + +def nsys_trace_to_timeline_events( + nsys: Path, nsys_rep: Path, work_dir: Path, with_stacks: bool = False +) -> Optional[dict]: """Export cuda_gpu_trace + cuda_api_trace and parse them into timeline events. - Returns {"gpu": [...], "api": [...]} (either list may be empty), or None if - neither trace produced events. + Returns {"gpu": [...], "api": [...], "stacks": [...]} (lists may be empty), + or None if neither trace produced events. With with_stacks=True the report + is also exported to SQLite and each event whose CorrID has a recorded CPU + backtrace gets a "stack" index into the "stacks" table (GPU kernels resolve + through the launching API call's CorrID). """ work_dir.mkdir(parents=True, exist_ok=True) @@ -413,10 +535,31 @@ def nsys_trace_to_timeline_events(nsys: Path, nsys_rep: Path, work_dir: Path) -> if not gpu_events and not api_events: logger.error("No usable nsys trace CSV (cuda_gpu_trace / cuda_api_trace)") return None + + stacks: List[List[str]] = [] + if with_stacks: + sqlite_path = nsys_export_sqlite(nsys, nsys_rep, work_dir) + if sqlite_path is not None: + corr_to_stack, stacks = load_callchains_from_sqlite(sqlite_path) + if corr_to_stack: + for ev in api_events + gpu_events: + if ev["corr"] is not None: + ev["stack"] = corr_to_stack.get(ev["corr"], -1) + logger.info( + "Attached CPU backtraces: %d distinct stacks over %d CorrIDs", + len(stacks), + len(corr_to_stack), + ) + if not stacks: + logger.warning( + "nsys timeline stacks requested but the SQLite export has no callchains " + "(capture without --cudabacktrace, or unwinding produced nothing)" + ) + logger.info( "Parsed nsys timeline traces: %d GPU events, %d CUDA API events", len(gpu_events), len(api_events) ) - return {"gpu": gpu_events, "api": api_events} + return {"gpu": gpu_events, "api": api_events, "stacks": stacks} def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timeline") -> Optional[str]: @@ -424,7 +567,10 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin Lanes: one per CPU thread issuing CUDA API calls, one per GPU device/stream. CorrID links a CPU-side launch to the GPU kernel it produced (click to - highlight). No external assets; suitable for the Studio Adhoc iframe. + highlight). When events carry a "stack" index into events["stacks"] + (--cudabacktrace capture), clicking also opens a panel with the CPU + backtrace that issued the launch. No external assets; suitable for the + Studio Adhoc iframe. """ import json @@ -461,13 +607,15 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin name_index[e["name"]] = idx name_table.append(e["name"]) corr = e["corr"] if e["corr"] is not None else -1 - packed.append([e["start"] - t0, e["dur"], lane_index[e["lane"]], corr, idx]) + stack = e.get("stack", -1) + packed.append([e["start"] - t0, e["dur"], lane_index[e["lane"]], corr, idx, stack]) data = { "lanes": lane_names, "cpuLanes": sum(1 for n in lane_names if n.startswith("CPU")), "names": name_table, "events": packed, + "stacks": events.get("stacks") or [], "span": span, "total": total_events, "shown": len(all_events), @@ -485,10 +633,17 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin #tip { position: absolute; display: none; pointer-events: none; background: #222b45; color: #e8ecf5; border: 1px solid #3a4568; border-radius: 4px; padding: 6px 8px; font-size: 0.75rem; max-width: 480px; white-space: pre-wrap; word-break: break-all; z-index: 10; } +#stack { display: none; background: #151b2e; border-radius: 8px; padding: 10px 12px; margin-top: 8px; + font-size: 0.75rem; } +#stack h2 { font-size: 0.8rem; font-weight: 600; margin: 0 0 6px 0; word-break: break-all; } +#stack ol { margin: 0; padding-left: 22px; font-family: ui-monospace, monospace; } +#stack li { color: #c5cde0; word-break: break-all; padding: 1px 0; } +#stack .none { color: #9aa3b5; }

__TITLE__

+
diff --git a/tests/test_nsys_profiler.py b/tests/test_nsys_profiler.py index b9e913d80..5baf248aa 100644 --- a/tests/test_nsys_profiler.py +++ b/tests/test_nsys_profiler.py @@ -369,6 +369,20 @@ def test_generate_timeline_html_with_stacks(): assert ",0]" in html and ",-1]" in html +def test_generate_timeline_html_with_stacks_has_flamegraph(): + events = { + "gpu": _parse_trace_csv(GPU_TRACE_CSV, "gpu"), + "api": _parse_trace_csv(API_TRACE_CSV, "api"), + "stacks": [["cudaLaunchKernel (libcudart.so)", "main (python3)"]], + } + for ev in events["api"] + events["gpu"]: + ev["stack"] = 0 if ev["corr"] in (101, 102) else -1 + html = generate_nsys_timeline_html(events) + assert html is not None + assert "Launch-stack flamegraph" in html + assert "fgDraw" in html + + def test_generate_timeline_html_without_stacks_still_renders(): events = { "gpu": _parse_trace_csv(GPU_TRACE_CSV, "gpu"), From 87b9d907c6b85923b43c29e3bfca05da31dbd7b1 Mon Sep 17 00:00:00 2001 From: prashantbytesyntax Date: Mon, 17 Aug 2026 00:27:45 +0000 Subject: [PATCH 10/11] Make the timeline navigable: overview minimap, keyboard, legend User feedback from the GPU-host e2e: the timeline is hard to navigate once auto-zoomed (no sense of where you are in the capture) and the CorrID mechanics were unexplained. - Overview strip above the lanes: full-capture CPU/GPU density with a draggable viewport box, so you can jump anywhere without zooming out. - Keyboard: arrows pan, +/- zoom, n/p select next/previous event in view (driving the CorrID highlight and stack panel), 0 resets to full span. - The meta line now explains the lanes and correlation IDs in plain terms with color swatches instead of assuming nsys vocabulary. Co-Authored-By: Claude Fable 5 --- docs/NSYS_GPU_PROFILING.md | 6 ++ gprofiler/nsys_profiler.py | 117 +++++++++++++++++++++++++++++++++--- tests/test_nsys_profiler.py | 12 ++++ 3 files changed, 128 insertions(+), 7 deletions(-) diff --git a/docs/NSYS_GPU_PROFILING.md b/docs/NSYS_GPU_PROFILING.md index fdba63eaa..9b5169e72 100644 --- a/docs/NSYS_GPU_PROFILING.md +++ b/docs/NSYS_GPU_PROFILING.md @@ -70,6 +70,12 @@ wide (a "Full span" button resets), and at low zoom sub-pixel events shade the lane by occupancy instead of tiling solid 1px bars — so a fully-packed lane looks different from a half-idle one even when zoomed out. +Navigation: an overview strip above the lanes shows the full capture (CPU +density on top, GPU below) with a highlighted viewport box — drag on it to +jump anywhere without zooming out first. Wheel or `+`/`-` zoom, drag or +arrow keys pan, `n`/`p` select the next/previous event in view (highlighting +its CorrID pair and stack panel), `0` resets to full span. + ### Click-for-stack mode (`--nsys-timeline-stacks` / `nsys_timeline_stacks`) With `--nsys-timeline-stacks` (CLI) or `combined_config.nsys_timeline_stacks: diff --git a/gprofiler/nsys_profiler.py b/gprofiler/nsys_profiler.py index f581f1050..fe0f773cd 100644 --- a/gprofiler/nsys_profiler.py +++ b/gprofiler/nsys_profiler.py @@ -636,7 +636,9 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin h1 { font-size: 1.1rem; font-weight: 600; margin: 0 0 4px 0; } .meta { color: #9aa3b5; font-size: 0.8rem; margin-bottom: 10px; } #wrap { position: relative; background: #151b2e; border-radius: 8px; padding: 8px; } +#mini { display: block; width: 100%; height: 36px; cursor: grab; margin-bottom: 6px; } #tl { display: block; width: 100%; cursor: crosshair; } +.lg { display: inline-block; width: 10px; height: 10px; border-radius: 2px; margin: 0 3px -1px 8px; } #tip, #fgtip { position: absolute; display: none; pointer-events: none; background: #222b45; color: #e8ecf5; border: 1px solid #3a4568; border-radius: 4px; padding: 6px 8px; font-size: 0.75rem; max-width: 480px; white-space: pre-wrap; word-break: break-all; z-index: 10; } @@ -652,7 +654,7 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin

__TITLE__

-
+

Launch-stack flamegraph

@@ -689,14 +691,18 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin function fullSpan() { viewStart = 0; viewSpan = DATA.span; draw(); } initView(); document.getElementById('meta').innerHTML = - 'CUDA API calls (CPU threads) + GPU kernels/memops from nsys cuda_api_trace / cuda_gpu_trace. ' + - 'Span ' + fmtNs(DATA.span) + '. Showing ' + DATA.shown + ' of ' + DATA.total + ' events' + + 'CPU lanes: CUDA API calls per thread (a launch is the ' + + 'CPU asking for work).GPU lanes: kernels/memcpys per ' + + 'stream (the work itself, usually later and longer). Each launch and its kernel share a correlation ID, ' + + 'so clicking either highlights both' + + (DATA.stacks.length ? ' and shows the CPU backtrace of the launch' : '') + + '.
Span ' + fmtNs(DATA.span) + ', ' + DATA.shown + ' of ' + DATA.total + ' events' + (DATA.shown < DATA.total ? ' (longest kept)' : '') + - '. Wheel: zoom - drag: pan - click: highlight CorrID (CPU launch <-> GPU kernel)' + - (DATA.stacks.length ? ' and show the CPU backtrace of the launch' : '') + - (autoZoomed ? '. Auto-zoomed to the middle of the capture; sub-pixel events fade by lane occupancy at low zoom. ' : '. ') + + '. Drag on the overview strip to jump anywhere. Wheel or +/- keys: zoom' + + ' - drag or arrow keys: pan - n/p keys: select next/previous event in view - 0: ' + ''; + ' border-radius: 4px; cursor: pointer; padding: 1px 8px;">Full span' + + (autoZoomed ? '. Opened auto-zoomed to the middle of the capture; sub-pixel events fade by lane occupancy.' : '.'); document.getElementById('fit').addEventListener('click', fullSpan); function draw() { const cssW = canvas.clientWidth || 800; @@ -755,7 +761,66 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin ctx.fillRect(LABEL_W + i, y, 1, LANE_H - 6); } } + drawMini(); } +// Overview strip: full-capture density with a draggable viewport window, so +// you always see where you are and can jump without scroll-zooming out first. +const mini = document.getElementById('mini'); +const MINI_H = 36; +let miniDensity = null; // per-pixel [cpu, gpu] occupancy over the full span, cached per width +function drawMini() { + const cssW = mini.clientWidth || 800; + const dpr = window.devicePixelRatio || 1; + mini.width = cssW * dpr; mini.height = MINI_H * dpr; + mini.style.height = MINI_H + 'px'; + const ctx = mini.getContext('2d'); + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.fillStyle = '#10152a'; + ctx.fillRect(0, 0, cssW, MINI_H); + if (!miniDensity || miniDensity.cpu.length !== cssW) { + miniDensity = { cpu: new Float32Array(cssW), gpu: new Float32Array(cssW) }; + for (const [s, d, lane] of DATA.events) { + const a = Math.max(0, Math.floor(s / DATA.span * cssW)); + const b = Math.min(cssW - 1, Math.floor((s + d) / DATA.span * cssW)); + const arr = lane < DATA.cpuLanes ? miniDensity.cpu : miniDensity.gpu; + for (let i = a; i <= b; i++) arr[i] = Math.min(1, arr[i] + 0.15); + } + } + const half = (MINI_H - 6) / 2; + for (let i = 0; i < cssW; i++) { + if (miniDensity.cpu[i] > 0) { + ctx.fillStyle = 'rgba(92,138,230,' + (0.3 + 0.7 * miniDensity.cpu[i]).toFixed(2) + ')'; + ctx.fillRect(i, 3, 1, half); + } + if (miniDensity.gpu[i] > 0) { + ctx.fillStyle = 'rgba(230,161,92,' + (0.3 + 0.7 * miniDensity.gpu[i]).toFixed(2) + ')'; + ctx.fillRect(i, 3 + half, 1, half); + } + } + const vx = viewStart / DATA.span * cssW; + const vw = Math.max(3, viewSpan / DATA.span * cssW); + ctx.strokeStyle = '#f5e663'; + ctx.lineWidth = 1.5; + ctx.strokeRect(vx + 0.75, 1, vw - 1.5, MINI_H - 2); + ctx.fillStyle = 'rgba(245,230,99,0.12)'; + ctx.fillRect(vx, 1, vw, MINI_H - 2); +} +function miniJump(clientX) { + const r = mini.getBoundingClientRect(); + const frac = Math.min(1, Math.max(0, (clientX - r.left) / r.width)); + viewStart = Math.min(Math.max(0, frac * DATA.span - viewSpan / 2), DATA.span - viewSpan); + draw(); +} +let miniDrag = false; +mini.addEventListener('mousedown', (e) => { miniDrag = true; miniJump(e.clientX); }); +window.addEventListener('mousemove', (e) => { if (miniDrag) miniJump(e.clientX); }); +window.addEventListener('mouseup', () => { miniDrag = false; }); +mini.addEventListener('wheel', (e) => { + e.preventDefault(); + viewSpan = Math.min(DATA.span, Math.max(1000, viewSpan * (e.deltaY > 0 ? 1.25 : 0.8))); + viewStart = Math.min(Math.max(0, viewStart), DATA.span - viewSpan); + draw(); +}, { passive: false }); function hit(mx, my) { const plotW = (canvas.clientWidth || 800) - LABEL_W; if (mx < LABEL_W || my < AXIS_H) return null; @@ -952,6 +1017,44 @@ def generate_nsys_timeline_html(events: dict, title: str = "nsys CPU/GPU timelin window.addEventListener('resize', fgDraw); fgDraw(); } +// Keyboard: arrows pan, +/- zoom, n/p step through events in view (selecting +// each so its CorrID pair lights up and the stack panel follows), 0 resets. +let stepIdx = -1; +function selectEvent(ev) { + selCorr = ev[3] >= 0 ? ev[3] : -1; + showStack(ev); + // keep the stepped event in view + if (ev[0] < viewStart || ev[0] > viewStart + viewSpan) { + viewStart = Math.min(Math.max(0, ev[0] - viewSpan / 2), DATA.span - viewSpan); + } + draw(); +} +function step(dir) { + const inView = []; + for (let i = 0; i < DATA.events.length; i++) { + const e = DATA.events[i]; + if (e[0] + e[1] >= viewStart && e[0] <= viewStart + viewSpan) inView.push(i); + } + if (!inView.length) return; + const pos = inView.indexOf(stepIdx); + stepIdx = inView[(pos + dir + inView.length) % inView.length]; + selectEvent(DATA.events[stepIdx]); +} +window.addEventListener('keydown', (e) => { + if (e.target && /INPUT|TEXTAREA|SELECT/.test(e.target.tagName)) return; + const panBy = viewSpan * 0.15; + if (e.key === 'ArrowLeft') viewStart = Math.max(0, viewStart - panBy); + else if (e.key === 'ArrowRight') viewStart = Math.min(DATA.span - viewSpan, viewStart + panBy); + else if (e.key === '+' || e.key === '=') viewSpan = Math.max(1000, viewSpan * 0.7); + else if (e.key === '-' || e.key === '_') viewSpan = Math.min(DATA.span, viewSpan * 1.4); + else if (e.key === '0') { fullSpan(); return; } + else if (e.key === 'n') { step(1); return; } + else if (e.key === 'p') { step(-1); return; } + else return; + e.preventDefault(); + viewStart = Math.min(Math.max(0, viewStart), Math.max(0, DATA.span - viewSpan)); + draw(); +}); window.addEventListener('resize', draw); draw(); diff --git a/tests/test_nsys_profiler.py b/tests/test_nsys_profiler.py index 5baf248aa..46a263cf7 100644 --- a/tests/test_nsys_profiler.py +++ b/tests/test_nsys_profiler.py @@ -393,6 +393,18 @@ def test_generate_timeline_html_without_stacks_still_renders(): assert '"stacks":[]' in html +def test_generate_timeline_html_has_minimap_and_keys(): + events = { + "gpu": _parse_trace_csv(GPU_TRACE_CSV, "gpu"), + "api": _parse_trace_csv(API_TRACE_CSV, "api"), + } + html = generate_nsys_timeline_html(events) + assert html is not None + assert 'id="mini"' in html + assert "drawMini" in html + assert "keydown" in html + + def test_nsys_trace_to_timeline_events_none_when_empty(tmp_path: Path, monkeypatch): nsys = tmp_path / "nsys" nsys.write_text("#!/bin/sh\n") From 43dd09626ef12df488e293566c5e12044e65a3d7 Mon Sep 17 00:00:00 2001 From: prashantbytesyntax Date: Thu, 20 Aug 2026 06:24:47 +0000 Subject: [PATCH 11/11] Add opt-in raw .nsys-rep upload for Nsight Systems download --nsys-upload-rep / nsys_upload_rep heartbeat key: after a successful nsys capture and profile upload, POST the raw .nsys-rep to the server as an octet-stream (new ProfilerAPIClient.submit_nsys_rep), tagged with the same start_time + hostname so the server pairs it with the adhoc flamegraph. Off by default since reps can be hundreds of MB. A failing rep upload or on_rep callback never breaks HTML generation or the profile upload. Co-Authored-By: Claude Fable 5 --- gprofiler/client.py | 30 +++++++++++++ .../dynamic_profiling_management/__init__.py | 1 + gprofiler/main.py | 28 ++++++++++++ gprofiler/nsys_profiler.py | 9 ++++ tests/test_nsys_profiler.py | 43 +++++++++++++++++++ 5 files changed, 111 insertions(+) diff --git a/gprofiler/client.py b/gprofiler/client.py index 4cce440a8..f23fd64ce 100644 --- a/gprofiler/client.py +++ b/gprofiler/client.py @@ -298,3 +298,33 @@ def submit_profile( api_version="v2" if profile_api_version is None else profile_api_version, params={"version": __version__}, ) + + def submit_nsys_rep(self, start_time: datetime.datetime, rep_path: str) -> Dict: + """Upload a raw .nsys-rep capture as an octet-stream body. + + Bypasses the JSON+gzip encoding of _request_url: reps are binary and + already compressed, so they are streamed from disk as-is. start_time + must match the profile's start_time so the server can pair the rep + with its adhoc flamegraph entry (keyed by start_time + hostname). + """ + url = "{}/nsys_rep".format(self.get_base_url("v2")) + params = self._get_query_params() + [ + ("version", __version__), + ("start_time", get_iso8601_format_time(start_time)), + ] + with open(rep_path, "rb") as rep_file: + resp = self._session.post( + url, + data=rep_file, + headers={"Content-Type": "application/octet-stream"}, + params=params, + timeout=max(self._upload_timeout, 600), + ) + if 400 <= resp.status_code < 500: + try: + response_data = resp.json() + raise APIError(response_data.get("message", "(no message in response)"), response_data) + except ValueError: + raise APIError(resp.text) + resp.raise_for_status() + return cast(dict, resp.json()) diff --git a/gprofiler/dynamic_profiling_management/__init__.py b/gprofiler/dynamic_profiling_management/__init__.py index 9f276ea8e..bece29f54 100644 --- a/gprofiler/dynamic_profiling_management/__init__.py +++ b/gprofiler/dynamic_profiling_management/__init__.py @@ -118,6 +118,7 @@ def create_profiler_args( new_args.nsys_workload = combined_config.get("nsys_workload") new_args.nsys_timeline = bool(combined_config.get("nsys_timeline", False)) new_args.nsys_timeline_stacks = bool(combined_config.get("nsys_timeline_stacks", False)) + new_args.nsys_upload_rep = bool(combined_config.get("nsys_upload_rep", False)) logger.info(f"enable_nsys: using nsys at {found}") else: logger.error( diff --git a/gprofiler/main.py b/gprofiler/main.py index dde35ade4..ac253d40f 100644 --- a/gprofiler/main.py +++ b/gprofiler/main.py @@ -174,8 +174,10 @@ def __init__( self._nsys_workload = user_args.get("nsys_workload") self._nsys_timeline = bool(user_args.get("nsys_timeline", False)) self._nsys_timeline_stacks = bool(user_args.get("nsys_timeline_stacks", False)) + self._nsys_upload_rep = bool(user_args.get("nsys_upload_rep", False)) self._nsys_thread: Optional[threading.Thread] = None self._nsys_html: Optional[str] = None + self._nsys_rep_path: Optional[Path] = None if self._collect_metadata: self._static_metadata = get_static_metadata(self._spawn_time, user_args, self._external_metadata_path) @@ -409,6 +411,9 @@ def _gen(collapsed: str) -> Optional[str]: start = end - datetime.timedelta(seconds=self._duration) return self._generate_flamegraph_html(collapsed, start, end) + def _keep_rep(rep: Path) -> None: + self._nsys_rep_path = rep + html = collect_nsys_adhoc_html( duration_sec=self._duration, nsys_path=self._nsys_path, @@ -417,12 +422,24 @@ def _gen(collapsed: str) -> Optional[str]: generate_html_fn=_gen, timeline=self._nsys_timeline, timeline_stacks=self._nsys_timeline_stacks, + on_rep=_keep_rep if self._nsys_upload_rep else None, ) self._nsys_html = html except Exception: logger.exception("Background nsys GPU capture failed") self._nsys_html = None + def _upload_nsys_rep(self, start_time: datetime.datetime) -> None: + rep_path = self._nsys_rep_path + self._nsys_rep_path = None + try: + size_mb = os.path.getsize(rep_path) / (1024 * 1024) + logger.info(f"Uploading nsys rep {rep_path} ({size_mb:.1f}MB) to the server") + self._profiler_api_client.submit_nsys_rep(start_time, str(rep_path)) + logger.info("Successfully uploaded nsys rep to the server") + except Exception: + logger.exception("Failed to upload nsys rep to the server") + def stop(self) -> None: logger.info("Stopping ...") self._profiler_state.stop_event.set() @@ -615,6 +632,8 @@ def _snapshot(self) -> None: metrics, self._gpid, ) + if self._nsys_upload_rep and self._nsys_rep_path is not None: + self._upload_nsys_rep(local_start_time) if time.monotonic() - self._last_diagnostics > DIAGNOSTICS_INTERVAL_S: self._last_diagnostics = time.monotonic() log_diagnostics() @@ -1361,6 +1380,15 @@ def parse_cmd_args() -> configargparse.Namespace: "(--cudabacktrace=kernel; enables CPU sampling — noticeably heavier) and " "show it when an event is clicked in the timeline.", ) + nsys_options.add_argument( + "--nsys-upload-rep", + action="store_true", + default=False, + dest="nsys_upload_rep", + help="With --enable-nsys and --upload-results, also upload the raw .nsys-rep " + "capture to the Performance Studio so it can be downloaded and opened in " + "NVIDIA Nsight Systems. Reports can be large (tens to hundreds of MB).", + ) args = parser.parse_args() diff --git a/gprofiler/nsys_profiler.py b/gprofiler/nsys_profiler.py index fe0f773cd..bac4a680d 100644 --- a/gprofiler/nsys_profiler.py +++ b/gprofiler/nsys_profiler.py @@ -1169,6 +1169,7 @@ def collect_nsys_adhoc_html( generate_html_fn: Optional[Callable[[str], Optional[str]]] = None, timeline: bool = False, timeline_stacks: bool = False, + on_rep: Optional[Callable[[Path], None]] = None, ) -> Optional[str]: """End-to-end: find nsys → capture → collapsed → HTML. Returns HTML or None. @@ -1177,6 +1178,8 @@ def collect_nsys_adhoc_html( (falling back to the flamegraph if the trace export yields no events). timeline_stacks=True additionally captures CPU backtraces per kernel launch (--cudabacktrace; heavier) and shows them on click in the timeline. + on_rep is called with the .nsys-rep path right after a successful capture, + so callers can keep/upload the raw report alongside the rendered HTML. """ nsys = find_nsys(nsys_path) if nsys is None: @@ -1209,6 +1212,12 @@ def collect_nsys_adhoc_html( if rep is None: return None + if on_rep is not None: + try: + on_rep(rep) + except Exception: + logger.exception("on_rep callback failed; continuing with HTML generation") + if timeline: events = nsys_trace_to_timeline_events(nsys, rep, base, with_stacks=timeline_stacks) if events: diff --git a/tests/test_nsys_profiler.py b/tests/test_nsys_profiler.py index 46a263cf7..3644406b8 100644 --- a/tests/test_nsys_profiler.py +++ b/tests/test_nsys_profiler.py @@ -27,6 +27,7 @@ _csv_to_collapsed, _parse_trace_csv, _workload_env, + collect_nsys_adhoc_html, find_nsys, generate_nsys_timeline_html, load_callchains_from_sqlite, @@ -417,3 +418,45 @@ def fake_run(cmd, **kwargs): monkeypatch.setattr("gprofiler.nsys_profiler.subprocess.run", fake_run) assert nsys_trace_to_timeline_events(nsys, rep, tmp_path / "work") is None + + +def test_collect_adhoc_html_on_rep_callback(tmp_path: Path, monkeypatch): + nsys = tmp_path / "nsys" + nsys.write_text("#!/bin/sh\n") + nsys.chmod(0o755) + + def fake_run(cmd, **kwargs): + if cmd[1] == "profile": + Path(str(cmd[cmd.index("-o") + 1]) + ".nsys-rep").write_bytes(b"fake") + return mock.Mock(returncode=0, stdout=b"ok") + # nsys stats: write the kern CSV where _export_stats_csv looks for it + out_prefix = Path(cmd[cmd.index("-o") + 1]) + out_prefix.parent.mkdir(parents=True, exist_ok=True) + (out_prefix.parent / (out_prefix.name + "_cuda_gpu_kern_sum.csv")).write_text(KERN_CSV) + return mock.Mock(returncode=0, stdout=b"ok") + + monkeypatch.setattr("gprofiler.nsys_profiler.subprocess.run", fake_run) + monkeypatch.setattr("gprofiler.nsys_profiler.find_nsys", lambda explicit_path=None: nsys) + + seen_reps = [] + html = collect_nsys_adhoc_html( + duration_sec=1, + work_dir=str(tmp_path / "work"), + generate_html_fn=lambda collapsed: "ok", + on_rep=seen_reps.append, + ) + assert html == "ok" + assert len(seen_reps) == 1 + assert seen_reps[0].name.endswith(".nsys-rep") and seen_reps[0].is_file() + + # a failing callback must not break HTML generation + def boom(rep: Path) -> None: + raise RuntimeError("boom") + + html = collect_nsys_adhoc_html( + duration_sec=1, + work_dir=str(tmp_path / "work2"), + generate_html_fn=lambda collapsed: "ok", + on_rep=boom, + ) + assert html == "ok"