From aed7817c9fcd53ec15fe26a288b24c6b0034a4a5 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 18 Aug 2025 21:43:37 +0000 Subject: [PATCH 1/5] Added scan to available PIDs in every available languade and send the list to backend through heartbeat call Added readme about the post profiling filtering Fix example code to run heartbeat mode profiling --- docs/HEARTBEAT_SYSTEM_README.md | 3 +- docs/POST_PROFILING_FILTERING.md | 216 +++++++++++++++++++++++++++++++ gprofiler/heartbeat.py | 77 ++++++++++- 3 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 docs/POST_PROFILING_FILTERING.md diff --git a/docs/HEARTBEAT_SYSTEM_README.md b/docs/HEARTBEAT_SYSTEM_README.md index 18c0b8253..4474580f7 100644 --- a/docs/HEARTBEAT_SYSTEM_README.md +++ b/docs/HEARTBEAT_SYSTEM_README.md @@ -412,7 +412,8 @@ sudo ./build/x86_64/gprofiler \ --upload-results \ --token=$GPROFILER_TOKEN \ --service-name=$GPROFILER_SERVICE \ - --api-server $GPROFILER_SERVER \ + --api-server=$GPROFILER_SERVER \ + --server-host=$GPROFILER_SERVER \ --heartbeat-interval 30 \ --output-dir /tmp/profiles \ --dont-send-logs \ diff --git a/docs/POST_PROFILING_FILTERING.md b/docs/POST_PROFILING_FILTERING.md new file mode 100644 index 000000000..d0a2ad9ad --- /dev/null +++ b/docs/POST_PROFILING_FILTERING.md @@ -0,0 +1,216 @@ +# gProfiler Post-Profiling Filtering Behavior + +## Overview + +When using **heartbeat command control** or **specific PID targeting** in gProfiler, you may notice that profilers for other languages still run and attempt to profile processes, even though you're targeting a specific process. This is **expected behavior** due to gProfiler's post-profiling filtering architecture. + +## What Happens During Profiling + +### Timeline of Events + +1. **Start Phase**: All enabled profilers start up and begin scanning/profiling system-wide +2. **Profiling Phase**: Profilers collect data from ALL processes they can detect (for the full duration) +3. **Filtering Phase**: Only at the end, during result processing, do profilers filter out unwanted processes + +### This means: + +- **Profiling work is done system-wide** regardless of PID targeting +- **Resource usage occurs** for all detected processes of each language type +- **Filtering happens post-collection** when assembling final results + +## Affected Profilers + +This post-profiling filtering pattern affects **multiple profilers**, not just Python: + +### 🐍 **Python eBPF Profiler** +**Location**: `gprofiler/profilers/python_ebpf.py:355-357` + +```python +# PyPerf profiles ALL Python processes system-wide for full duration +def snapshot(self) -> ProcessToProfileData: + collapsed_path = self._dump() # ← All Python processes profiled + parsed = parse_many_collapsed(collapsed_text) # ← Parse all results + + for pid in parsed: # ← Iterate through ALL profiled PIDs + process = Process(pid) + # Filtering happens HERE - after profiling is complete + if self._profiler_state.processes_to_profile is not None: + if process not in self._profiler_state.processes_to_profile: + continue # ← Skip from final output only +``` + +**Impact**: PyPerf runs for full duration, attempts to profile all Python processes, then filters results. + +### 🐘 **PHP Profiler** +**Location**: `gprofiler/profilers/php.py:211-213` + +```python +# phpspy profiles detected PHP processes, then filters output +def _parse_phpspy_output(self, output: str, profiler_state: ProfilerState) -> ProcessToProfileData: + # ... profiling work already done ... + for pid in results: + # Post-profiling filtering + if profiler_state.processes_to_profile is not None: + if pid not in [process.pid for process in profiler_state.processes_to_profile]: + continue # ← Skip from results +``` + +**Impact**: phpspy runs and profiles PHP processes, then filters output. + +### ⚡ **System Profiler (perf)** +**Location**: `gprofiler/profilers/perf.py:200,216,232` + +```python +# perf receives processes_to_profile but still runs system-wide collection +def __init__(self, ...): + self._perf_fp = PerfProcess( + # ... + processes_to_profile=self._profiler_state.processes_to_profile, # ← Passed to perf + ) +``` + +**Impact**: `perf record` may use `--pid` flag to focus collection, but still runs system-wide monitoring. + +### 💎 **Ruby Profiler** +**Location**: `gprofiler/profilers/ruby.py` - Uses base class filtering + +```python +# Ruby uses the base class pre-profiling filter (better design) +def snapshot(self) -> ProcessToProfileData: + processes_to_profile = self._select_processes_to_profile() # ← Find Ruby processes + if self._profiler_state.processes_to_profile is not None: + processes_to_profile = [ + process for process in processes_to_profile + if process in self._profiler_state.processes_to_profile # ← Filter BEFORE profiling + ] + # Only profile filtered processes +``` + +**Impact**: Ruby profiler filters BEFORE profiling (more efficient). + +### ☕ **Java Profiler** +**Location**: `gprofiler/profilers/java.py` - Uses base class filtering + +**Impact**: Java profiler also filters BEFORE profiling (more efficient). + +## Why This Design Exists + +### **System-Wide Profilers** (Python eBPF, PHP, System/perf) +- **Efficiency**: eBPF/kernel-level tools are more efficient when monitoring system-wide +- **Process Discovery**: Some processes may spawn during profiling +- **Technical Constraints**: Harder to filter at kernel/eBPF level + +### **Process-Specific Profilers** (Java, Ruby, .NET) +- **Targeted Tools**: These tools naturally profile one process at a time +- **Early Filtering**: Can efficiently skip processes before starting profiling work + +## Impact on Performance + +### **Wasted Resources** +When targeting a specific non-Python process via heartbeat: + +``` +Example: Targeting Java PID 964466 via heartbeat +✅ Java Profiler: Profiles only PID 964466 (efficient) +❌ Python eBPF: Profiles ALL Python processes for 60s, then discards results +❌ System Profiler: Runs system-wide perf collection +❌ PHP Profiler: Scans for and profiles PHP processes, then discards +``` + +### **Resource Usage** +- **CPU**: System-wide profiling overhead +- **Memory**: Buffers for all processes +- **I/O**: Writing/reading profile data for unwanted processes +- **Time**: Full profiling duration spent on irrelevant processes + +## Example From Your Logs + +In your case, targeting Java PID 964466: + +``` +[2025-08-15 21:17:37,908] INFO: gprofiler.profilers.java: Profiling process 964466 with async-profiler +# ✅ Java profiler correctly targets only PID 964466 + +[2025-08-15 21:18:37,943] DEBUG: gprofiler.profilers.python_ebpf: PyPerf dump output +# ❌ Python eBPF profiler spent 60 seconds profiling ALL Python processes +# Then failed on Bazel processes with deleted libraries +# Finally discarded all results since none matched PID 964466 +``` + +## Solutions & Workarounds + +### **Option 1: Disable Unwanted Profilers** +```bash +gprofiler \ + --enable-heartbeat-server \ + --python-mode disabled \ + --php-mode disabled \ + --ruby-mode disabled \ + --dotnet-mode disabled \ + --perf-mode none \ + # Only Java profiler will run +``` + +### **Option 2: Use Direct PID Targeting** +```bash +gprofiler --processes-to-profile 964466 --java-mode ap +# More efficient than heartbeat for single-process scenarios +``` + +### **Option 3: Accept the Overhead** +- Current behavior ensures comprehensive system coverage +- Final results are correctly filtered +- Useful when you want context from multiple process types + +## Architecture Improvement Opportunities + +### **For System-Wide Profilers** +- **Early PID Filtering**: Check `processes_to_profile` before starting profiling +- **Conditional Startup**: Don't start profiler if no target processes match language type +- **Resource Optimization**: Reduce buffer sizes when targeting specific PIDs + +### **Example Improvement** +```python +def start(self) -> None: + # Check if we should even start + if self._profiler_state.processes_to_profile is not None: + target_pids = [p.pid for p in self._profiler_state.processes_to_profile] + if not any(self._is_python_process(pid) for pid in target_pids): + logger.info("No Python processes in target list, skipping Python profiler") + return + + # Proceed with profiling + logger.info("Starting profiling of Python processes with PyPerf") + # ... +``` + +## Related Issues + +- **GitHub Issue #764**: Python eBPF post-filtering (referenced in code) +- **GitHub Issue #763**: PHP post-filtering (referenced in code) + +## Profiler Filtering Summary + +### **POST-Profiling Filtering** (Less Efficient - Profiles All, Then Filters) +| Profiler | Location | Behavior | +|----------|----------|----------| +| **🐍 Python eBPF** | `python_ebpf.py:355-357` | PyPerf profiles ALL Python processes system-wide, then filters results | +| **🐘 PHP** | `php.py:211-213` | phpspy profiles detected PHP processes, then filters output | +| **⚡ System/perf** | `perf.py:200,216,232` | perf runs system-wide collection, may use some targeting | + +### **PRE-Profiling Filtering** (More Efficient - Filters First, Then Profiles) +| Profiler | Location | Behavior | +|----------|----------|----------| +| **☕ Java** | Uses `profiler_base.py:210-217` | Filters target processes BEFORE starting async-profiler | +| **💎 Ruby** | Uses `profiler_base.py:210-217` | Filters target processes BEFORE starting rbspy | +| **🔷 .NET** | Uses `profiler_base.py:210-217` | Filters target processes BEFORE starting dotnet-trace | + +### **Resource Impact When Targeting Specific PIDs** +- **✅ Efficient**: Java, Ruby, .NET profilers only work on target processes +- **❌ Wasteful**: Python eBPF, PHP, System profilers do unnecessary work then discard results + +## Summary + +**Post-profiling filtering is expected behavior** that affects multiple profilers (Python eBPF, PHP, System/perf). While this ensures comprehensive system coverage and correct final results, it can waste resources when targeting specific processes. Understanding this behavior helps explain why you see profiler activity for languages you're not interested in when using heartbeat command control. + +The most efficient approach for single-process profiling is to either disable unwanted profilers or use direct PID targeting instead of heartbeat control. diff --git a/gprofiler/heartbeat.py b/gprofiler/heartbeat.py index 7ab308e89..c78a48aa1 100644 --- a/gprofiler/heartbeat.py +++ b/gprofiler/heartbeat.py @@ -39,10 +39,11 @@ from gprofiler.profiler_state import ProfilerState from gprofiler.profilers.factory import get_profilers from gprofiler.profilers.profiler_base import NoopProfiler +from gprofiler.profilers.registry import get_profilers_registry from gprofiler.state import State, init_state, get_state from gprofiler.system_metrics import NoopSystemMetricsMonitor, SystemMetricsMonitor, SystemMetricsMonitorBase from gprofiler.usage_loggers import NoopUsageLogger -from gprofiler.utils import TEMPORARY_STORAGE_PATH +from gprofiler.utils import TEMPORARY_STORAGE_PATH, pgrep_maps, pgrep_exe from gprofiler.hw_metrics import HWMetricsMonitor, HWMetricsMonitorBase, NoopHWMetricsMonitor from gprofiler.exceptions import NoProfilersEnabledError @@ -85,7 +86,10 @@ def _get_local_ip(self) -> str: except Exception: return "127.0.0.1" - def send_heartbeat(self) -> Optional[Dict[str, Any]]: + def send_heartbeat( + self, + available_pids: Optional[List[int]] = None + ) -> Optional[Dict[str, Any]]: """Send heartbeat to server and return any profiling commands""" try: heartbeat_data = { @@ -97,6 +101,10 @@ def send_heartbeat(self) -> Optional[Dict[str, Any]]: "timestamp": datetime.datetime.now().isoformat() } + # Include available PIDs if provided + if available_pids is not None: + heartbeat_data["available_pids"] = available_pids + url = f"{self.api_server}/api/metrics/heartbeat" response = self.session.post( url, @@ -232,14 +240,77 @@ def __init__(self, base_args: configargparse.Namespace, heartbeat_client: Heartb self.stop_event = threading.Event() self.heartbeat_interval = 30 # seconds + def scan_available_pids(self) -> List[int]: + """Scan the host for all available PIDs across all supported languages""" + all_pids = [] + + try: + # Import regex patterns from granulate_utils + from granulate_utils.java import DETECTED_JAVA_PROCESSES_REGEX + from granulate_utils.python import DETECTED_PYTHON_PROCESSES_REGEX + + # Define profiler patterns + profiler_patterns = { + 'java': DETECTED_JAVA_PROCESSES_REGEX, + 'python': DETECTED_PYTHON_PROCESSES_REGEX, + 'ruby': r"^.+/libruby", # Ruby pattern + 'dotnet': r"^.+/libcoreclr\.so", # .NET pattern + 'php': r"^.+/(lib)?php[^/]*$", # PHP pattern + } + + # Scan for each language + for language, pattern in profiler_patterns.items(): + try: + if language == 'python': + # Python uses both pgrep_maps and pgrep_exe on Windows + from gprofiler.platform import is_windows + if is_windows(): + processes = pgrep_exe("python") + else: + processes = pgrep_maps(pattern) + else: + # Other languages use pgrep_maps + processes = pgrep_maps(pattern) + + pids = [p.pid for p in processes] + + if pids: + all_pids.extend(pids) + logger.debug(f"Found {len(pids)} {language} processes: {pids}") + + except Exception as e: + # Don't let individual profiler errors break the entire scan + logger.debug(f"Error scanning {language} processes: {e}") + continue + + # Remove duplicates and sort for consistent output + all_pids = sorted(list(set(all_pids))) + + if all_pids: + logger.debug(f"Total available PIDs found: {len(all_pids)} processes: {all_pids}") + else: + logger.debug("No available PIDs found for any supported languages") + + except Exception as e: + logger.debug(f"Error scanning available PIDs: {e}") + + return all_pids + def start_heartbeat_loop(self): """Start the main heartbeat loop""" logger.info("Starting heartbeat loop...") while not self.stop_event.is_set(): try: + # Scan all available PIDs + available_pids = self.scan_available_pids() + if available_pids: + logger.debug(f"Sending heartbeat with {len(available_pids)} available PIDs: {available_pids}") + # Send heartbeat and check for commands - command_response = self.heartbeat_client.send_heartbeat() + command_response = self.heartbeat_client.send_heartbeat( + available_pids=available_pids + ) if command_response and command_response.get("profiling_command"): profiling_command = command_response["profiling_command"] From 30affb4ce20a54c330ee65e748d26890bf1a2393 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 25 Aug 2025 15:12:29 +0000 Subject: [PATCH 2/5] Update pid scan to send pids linked with language. --- gprofiler/heartbeat.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/gprofiler/heartbeat.py b/gprofiler/heartbeat.py index c78a48aa1..df9beabda 100644 --- a/gprofiler/heartbeat.py +++ b/gprofiler/heartbeat.py @@ -88,7 +88,7 @@ def _get_local_ip(self) -> str: def send_heartbeat( self, - available_pids: Optional[List[int]] = None + available_pids: Optional[Dict[str, List[int]]] = None ) -> Optional[Dict[str, Any]]: """Send heartbeat to server and return any profiling commands""" try: @@ -240,9 +240,9 @@ def __init__(self, base_args: configargparse.Namespace, heartbeat_client: Heartb self.stop_event = threading.Event() self.heartbeat_interval = 30 # seconds - def scan_available_pids(self) -> List[int]: - """Scan the host for all available PIDs across all supported languages""" - all_pids = [] + def scan_available_pids(self) -> Dict[str, List[int]]: + """Scan the host for all available PIDs organized by language/profiler type""" + available_pids = {} try: # Import regex patterns from granulate_utils @@ -274,27 +274,29 @@ def scan_available_pids(self) -> List[int]: pids = [p.pid for p in processes] + # Only include languages that have running processes if pids: - all_pids.extend(pids) - logger.debug(f"Found {len(pids)} {language} processes: {pids}") + available_pids[language] = sorted(pids) + logger.debug(f"Found {len(pids)} {language} processes") except Exception as e: # Don't let individual profiler errors break the entire scan logger.debug(f"Error scanning {language} processes: {e}") continue - # Remove duplicates and sort for consistent output - all_pids = sorted(list(set(all_pids))) - - if all_pids: - logger.debug(f"Total available PIDs found: {len(all_pids)} processes: {all_pids}") + if available_pids: + total_pids = sum(len(pids) for pids in available_pids.values()) + logger.debug( + f"Total available PIDs found: {total_pids} processes across " + f"{len(available_pids)} languages" + ) else: logger.debug("No available PIDs found for any supported languages") except Exception as e: logger.debug(f"Error scanning available PIDs: {e}") - return all_pids + return available_pids def start_heartbeat_loop(self): """Start the main heartbeat loop""" @@ -302,10 +304,14 @@ def start_heartbeat_loop(self): while not self.stop_event.is_set(): try: - # Scan all available PIDs + # Scan all available PIDs by language available_pids = self.scan_available_pids() if available_pids: - logger.debug(f"Sending heartbeat with {len(available_pids)} available PIDs: {available_pids}") + total_pids = sum(len(pids) for pids in available_pids.values()) + logger.debug( + f"Sending heartbeat with {total_pids} PIDs across " + f"{len(available_pids)} languages" + ) # Send heartbeat and check for commands command_response = self.heartbeat_client.send_heartbeat( From c43eef41f094b0f467ec19df847178c167b45274 Mon Sep 17 00:00:00 2001 From: Lucas Date: Wed, 17 Sep 2025 17:54:38 +0000 Subject: [PATCH 3/5] Updated process filter behavior document --- docs/POST_PROFILING_FILTERING.md | 179 ++++++++++++++++++++----------- 1 file changed, 118 insertions(+), 61 deletions(-) diff --git a/docs/POST_PROFILING_FILTERING.md b/docs/POST_PROFILING_FILTERING.md index d0a2ad9ad..5908356ec 100644 --- a/docs/POST_PROFILING_FILTERING.md +++ b/docs/POST_PROFILING_FILTERING.md @@ -1,29 +1,40 @@ -# gProfiler Post-Profiling Filtering Behavior +# gProfiler Process Filtering Behavior ## Overview -When using **heartbeat command control** or **specific PID targeting** in gProfiler, you may notice that profilers for other languages still run and attempt to profile processes, even though you're targeting a specific process. This is **expected behavior** due to gProfiler's post-profiling filtering architecture. +When using **heartbeat command control** or **specific PID targeting** in gProfiler, you may notice that some profilers for other languages still run and attempt to profile processes, even though you're targeting a specific process. This behavior varies by profiler type due to gProfiler's mixed filtering architecture: some profilers now filter **before** profiling (efficient), while others still filter **after** profiling (legacy post-filtering behavior). ## What Happens During Profiling ### Timeline of Events -1. **Start Phase**: All enabled profilers start up and begin scanning/profiling system-wide +The filtering behavior depends on the profiler type: + +#### **Individual Process Profilers** (Java, Ruby, .NET, Python py-spy) +1. **Start Phase**: Profiler starts and scans for target processes +2. **Filtering Phase**: **BEFORE profiling begins**, filter processes based on `processes_to_profile` +3. **Profiling Phase**: Profile only the filtered set of processes + +#### **System-Wide Profilers** (Python eBPF, PHP, System/perf) +1. **Start Phase**: Profiler starts and begins scanning/profiling system-wide 2. **Profiling Phase**: Profilers collect data from ALL processes they can detect (for the full duration) -3. **Filtering Phase**: Only at the end, during result processing, do profilers filter out unwanted processes +3. **Filtering Phase**: **AFTER profiling**, filter out unwanted processes from results ### This means: -- **Profiling work is done system-wide** regardless of PID targeting -- **Resource usage occurs** for all detected processes of each language type -- **Filtering happens post-collection** when assembling final results +- **Individual process profilers** (Java, Ruby, .NET, py-spy): **Efficient** - only profile target processes +- **System-wide profilers** (Python eBPF, PHP, System/perf): **Less efficient** - profile all processes, then filter results + +## Profiler Filtering Behavior -## Affected Profilers +gProfiler profilers use two different filtering approaches: -This post-profiling filtering pattern affects **multiple profilers**, not just Python: +### **POST-Profiling Filtering** (System-Wide Profilers) -### 🐍 **Python eBPF Profiler** -**Location**: `gprofiler/profilers/python_ebpf.py:355-357` +These profilers collect data from all processes, then filter results afterwards: + +#### 🐍 **Python eBPF Profiler** +**Location**: `gprofiler/profilers/python_ebpf.py:373-375` ```python # PyPerf profiles ALL Python processes system-wide for full duration @@ -39,9 +50,9 @@ def snapshot(self) -> ProcessToProfileData: continue # ← Skip from final output only ``` -**Impact**: PyPerf runs for full duration, attempts to profile all Python processes, then filters results. +**Impact**: PyPerf runs for full duration, profiles all Python processes, then filters results. -### 🐘 **PHP Profiler** +#### 🐘 **PHP Profiler** **Location**: `gprofiler/profilers/php.py:211-213` ```python @@ -57,12 +68,12 @@ def _parse_phpspy_output(self, output: str, profiler_state: ProfilerState) -> Pr **Impact**: phpspy runs and profiles PHP processes, then filters output. -### ⚡ **System Profiler (perf)** -**Location**: `gprofiler/profilers/perf.py:200,216,232` +#### ⚡ **System Profiler (perf)** +**Location**: `gprofiler/profilers/perf.py:269` ```python # perf receives processes_to_profile but still runs system-wide collection -def __init__(self, ...): +def start(self) -> None: self._perf_fp = PerfProcess( # ... processes_to_profile=self._profiler_state.processes_to_profile, # ← Passed to perf @@ -71,13 +82,17 @@ def __init__(self, ...): **Impact**: `perf record` may use `--pid` flag to focus collection, but still runs system-wide monitoring. -### 💎 **Ruby Profiler** -**Location**: `gprofiler/profilers/ruby.py` - Uses base class filtering +### **PRE-Profiling Filtering** (Individual Process Profilers) + +These profilers filter target processes BEFORE starting profiling work: + +#### 💎 **Ruby Profiler** +**Location**: `gprofiler/profilers/profiler_base.py:269-273` (base class filtering) ```python -# Ruby uses the base class pre-profiling filter (better design) +# All ProcessProfilerBase profilers now filter BEFORE profiling def snapshot(self) -> ProcessToProfileData: - processes_to_profile = self._select_processes_to_profile() # ← Find Ruby processes + processes_to_profile = self._select_processes_to_profile() # ← Find target processes if self._profiler_state.processes_to_profile is not None: processes_to_profile = [ process for process in processes_to_profile @@ -86,12 +101,22 @@ def snapshot(self) -> ProcessToProfileData: # Only profile filtered processes ``` -**Impact**: Ruby profiler filters BEFORE profiling (more efficient). +**Impact**: Ruby profiler filters BEFORE profiling (efficient). + +#### ☕ **Java Profiler** +**Location**: `gprofiler/profilers/profiler_base.py:269-273` (base class filtering) + +**Impact**: Java profiler filters BEFORE profiling (efficient). -### ☕ **Java Profiler** -**Location**: `gprofiler/profilers/java.py` - Uses base class filtering +#### 🔷 **.NET Profiler** +**Location**: `gprofiler/profilers/profiler_base.py:269-273` (base class filtering) -**Impact**: Java profiler also filters BEFORE profiling (more efficient). +**Impact**: .NET profiler filters BEFORE profiling (efficient). + +#### 🐍 **Python py-spy Profiler** +**Location**: `gprofiler/profilers/profiler_base.py:269-273` (base class filtering) + +**Impact**: py-spy profiler now filters BEFORE profiling (efficient). ## Why This Design Exists @@ -99,68 +124,84 @@ def snapshot(self) -> ProcessToProfileData: - **Efficiency**: eBPF/kernel-level tools are more efficient when monitoring system-wide - **Process Discovery**: Some processes may spawn during profiling - **Technical Constraints**: Harder to filter at kernel/eBPF level +- **Architecture**: These inherit directly from `ProfilerBase`, not `ProcessProfilerBase` -### **Process-Specific Profilers** (Java, Ruby, .NET) +### **Individual Process Profilers** (Java, Ruby, .NET, Python py-spy) - **Targeted Tools**: These tools naturally profile one process at a time - **Early Filtering**: Can efficiently skip processes before starting profiling work +- **Architecture**: These inherit from `ProcessProfilerBase` or `SpawningProcessProfilerBase`, which provides automatic pre-filtering ## Impact on Performance -### **Wasted Resources** -When targeting a specific non-Python process via heartbeat: +### **Efficient vs Wasteful Resource Usage** +When targeting a specific process via heartbeat or PID targeting: +#### **✅ Efficient Profilers** (Individual Process Profilers) ``` Example: Targeting Java PID 964466 via heartbeat -✅ Java Profiler: Profiles only PID 964466 (efficient) +✅ Java Profiler: Profiles only PID 964466 (efficient - PRE-filtering) +✅ Ruby Profiler: Skips all Ruby processes (efficient - PRE-filtering) +✅ .NET Profiler: Skips all .NET processes (efficient - PRE-filtering) +✅ Python py-spy: Skips all Python processes (efficient - PRE-filtering) +``` + +#### **❌ Wasteful Profilers** (System-Wide Profilers) +``` +Example: Targeting Java PID 964466 via heartbeat ❌ Python eBPF: Profiles ALL Python processes for 60s, then discards results -❌ System Profiler: Runs system-wide perf collection +❌ System Profiler: Runs system-wide perf collection, then filters ❌ PHP Profiler: Scans for and profiles PHP processes, then discards ``` -### **Resource Usage** -- **CPU**: System-wide profiling overhead -- **Memory**: Buffers for all processes -- **I/O**: Writing/reading profile data for unwanted processes +### **Resource Usage by System-Wide Profilers** +When system-wide profilers run unnecessarily: +- **CPU**: System-wide profiling overhead for unwanted processes +- **Memory**: Buffers for all processes of that language type +- **I/O**: Writing/reading profile data for processes that will be discarded - **Time**: Full profiling duration spent on irrelevant processes -## Example From Your Logs +## Example Behavior When Targeting Specific PIDs -In your case, targeting Java PID 964466: +When targeting Java PID 964466 via heartbeat or `--processes-to-profile`: ``` [2025-08-15 21:17:37,908] INFO: gprofiler.profilers.java: Profiling process 964466 with async-profiler -# ✅ Java profiler correctly targets only PID 964466 +# ✅ Java profiler (ProcessProfilerBase) correctly targets only PID 964466 + +[2025-08-15 21:17:37,910] DEBUG: gprofiler.profilers.python: Selected 0 processes to profile +[2025-08-15 21:17:37,910] DEBUG: gprofiler.profilers.python: processes left after filtering: 0 +# ✅ Python py-spy (ProcessProfilerBase) efficiently skips all processes - no work done [2025-08-15 21:18:37,943] DEBUG: gprofiler.profilers.python_ebpf: PyPerf dump output -# ❌ Python eBPF profiler spent 60 seconds profiling ALL Python processes -# Then failed on Bazel processes with deleted libraries +# ❌ Python eBPF profiler (ProfilerBase) spent 60 seconds profiling ALL Python processes +# Then failed on Bazel processes with deleted libraries # Finally discarded all results since none matched PID 964466 ``` ## Solutions & Workarounds -### **Option 1: Disable Unwanted Profilers** +### **Option 1: Disable Wasteful System-Wide Profilers** ```bash gprofiler \ --enable-heartbeat-server \ - --python-mode disabled \ - --php-mode disabled \ - --ruby-mode disabled \ - --dotnet-mode disabled \ - --perf-mode none \ - # Only Java profiler will run + --python-mode pyspy \ # Use py-spy instead of eBPF (efficient pre-filtering) + --php-mode disabled \ # Disable PHP (system-wide profiler) + --perf-mode disabled \ # Disable perf (system-wide profiler) + # Java, Ruby, .NET profilers will efficiently target specific processes ``` ### **Option 2: Use Direct PID Targeting** ```bash gprofiler --processes-to-profile 964466 --java-mode ap -# More efficient than heartbeat for single-process scenarios +# All ProcessProfilerBase profilers automatically filter efficiently +# Only system-wide profilers (eBPF, PHP, perf) waste resources ``` -### **Option 3: Accept the Overhead** +### **Option 3: Accept the Overhead from System-Wide Profilers** - Current behavior ensures comprehensive system coverage - Final results are correctly filtered -- Useful when you want context from multiple process types +- Individual process profilers are now efficient (pre-filtering) +- Only system-wide profilers waste resources (post-filtering) ## Architecture Improvement Opportunities @@ -192,25 +233,41 @@ def start(self) -> None: ## Profiler Filtering Summary ### **POST-Profiling Filtering** (Less Efficient - Profiles All, Then Filters) -| Profiler | Location | Behavior | -|----------|----------|----------| -| **🐍 Python eBPF** | `python_ebpf.py:355-357` | PyPerf profiles ALL Python processes system-wide, then filters results | -| **🐘 PHP** | `php.py:211-213` | phpspy profiles detected PHP processes, then filters output | -| **⚡ System/perf** | `perf.py:200,216,232` | perf runs system-wide collection, may use some targeting | +| Profiler | Class Hierarchy | Location | Behavior | +|----------|----------------|----------|----------| +| **🐍 Python eBPF** | `ProfilerBase` | `python_ebpf.py:373-375` | PyPerf profiles ALL Python processes system-wide, then filters results | +| **🐘 PHP** | `ProfilerBase` | `php.py:211-213` | phpspy profiles detected PHP processes, then filters output | +| **⚡ System/perf** | `ProfilerBase` | `perf.py:269` | perf runs system-wide collection, then filters results | ### **PRE-Profiling Filtering** (More Efficient - Filters First, Then Profiles) -| Profiler | Location | Behavior | -|----------|----------|----------| -| **☕ Java** | Uses `profiler_base.py:210-217` | Filters target processes BEFORE starting async-profiler | -| **💎 Ruby** | Uses `profiler_base.py:210-217` | Filters target processes BEFORE starting rbspy | -| **🔷 .NET** | Uses `profiler_base.py:210-217` | Filters target processes BEFORE starting dotnet-trace | +| Profiler | Class Hierarchy | Location | Behavior | +|----------|----------------|----------|----------| +| **☕ Java** | `SpawningProcessProfilerBase` | `profiler_base.py:269-273` | Filters target processes BEFORE starting async-profiler | +| **💎 Ruby** | `SpawningProcessProfilerBase` | `profiler_base.py:269-273` | Filters target processes BEFORE starting rbspy | +| **🔷 .NET** | `ProcessProfilerBase` | `profiler_base.py:269-273` | Filters target processes BEFORE starting dotnet-trace | +| **🐍 Python py-spy** | `SpawningProcessProfilerBase` | `profiler_base.py:269-273` | Filters target processes BEFORE starting py-spy | ### **Resource Impact When Targeting Specific PIDs** -- **✅ Efficient**: Java, Ruby, .NET profilers only work on target processes +- **✅ Efficient**: Java, Ruby, .NET, Python py-spy profilers only work on target processes - **❌ Wasteful**: Python eBPF, PHP, System profilers do unnecessary work then discard results ## Summary -**Post-profiling filtering is expected behavior** that affects multiple profilers (Python eBPF, PHP, System/perf). While this ensures comprehensive system coverage and correct final results, it can waste resources when targeting specific processes. Understanding this behavior helps explain why you see profiler activity for languages you're not interested in when using heartbeat command control. +**gProfiler now uses two different filtering approaches** depending on the profiler architecture: + +### **✅ Efficient PRE-Profiling Filtering** +All profilers inheriting from `ProcessProfilerBase` or `SpawningProcessProfilerBase` now automatically filter target processes **BEFORE** starting profiling work: +- **Java, Ruby, .NET, Python py-spy**: Only profile processes that match `processes_to_profile` +- **Resource Impact**: Minimal overhead when targeting specific PIDs + +### **❌ Legacy POST-Profiling Filtering** +Profilers inheriting directly from `ProfilerBase` still collect data from all processes, then filter results afterwards: +- **Python eBPF, PHP, System/perf**: Profile all processes, then discard unwanted results +- **Resource Impact**: Significant overhead when targeting specific PIDs + +### **Recommendations** +- **For efficiency**: Use `--python-mode pyspy` instead of `pyperf` when targeting specific processes +- **Disable wasteful profilers**: Use `--php-mode disabled --perf-mode disabled` when not needed +- **Direct PID targeting**: Use `--processes-to-profile PID` for maximum efficiency -The most efficient approach for single-process profiling is to either disable unwanted profilers or use direct PID targeting instead of heartbeat control. +The gap between efficient and wasteful profilers has **significantly improved** with the base class pre-filtering implementation. From b854975b946f399e5461c1a91e79527223eb9535 Mon Sep 17 00:00:00 2001 From: Lucas Date: Wed, 17 Sep 2025 18:21:36 +0000 Subject: [PATCH 4/5] Updated the pid discovery to return top x process based in the max-processes param --- gprofiler/heartbeat.py | 75 ++++++++++++++++++++++++++++++++---------- gprofiler/main.py | 2 +- 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/gprofiler/heartbeat.py b/gprofiler/heartbeat.py index 0b8d634ac..f9cd947f2 100644 --- a/gprofiler/heartbeat.py +++ b/gprofiler/heartbeat.py @@ -209,8 +209,8 @@ def __init__(self, base_args: configargparse.Namespace, heartbeat_client: Heartb self.heartbeat_interval = 30 # seconds def scan_available_pids(self) -> Dict[str, List[int]]: - """Scan the host for all available PIDs organized by language/profiler type""" - available_pids = {} + """Scan the host and return top-N PIDs by CPU, grouped by language.""" + available_pids: Dict[str, List[int]] = {} try: # Import regex patterns from granulate_utils @@ -226,31 +226,70 @@ def scan_available_pids(self) -> Dict[str, List[int]]: 'php': r"^.+/(lib)?php[^/]*$", # PHP pattern } - # Scan for each language + # First, discover processes per language + processes_by_language: Dict[str, List[Process]] = {} for language, pattern in profiler_patterns.items(): try: if language == 'python': - # Python uses both pgrep_maps and pgrep_exe on Windows from gprofiler.platform import is_windows - if is_windows(): - processes = pgrep_exe("python") - else: - processes = pgrep_maps(pattern) + processes = pgrep_exe("python") if is_windows() else pgrep_maps(pattern) else: - # Other languages use pgrep_maps processes = pgrep_maps(pattern) - - pids = [p.pid for p in processes] - - # Only include languages that have running processes - if pids: - available_pids[language] = sorted(pids) - logger.debug(f"Found {len(pids)} {language} processes") - + + if processes: + processes_by_language[language] = processes + logger.debug(f"Found {len(processes)} {language} processes") except Exception as e: - # Don't let individual profiler errors break the entire scan logger.debug(f"Error scanning {language} processes: {e}") continue + + # Flatten and deduplicate by PID across languages + pid_to_process: Dict[int, Process] = {} + pid_to_language: Dict[int, str] = {} + for language, processes in processes_by_language.items(): + for proc in processes: + pid = getattr(proc, "pid", None) + if pid is None: + continue + if pid in pid_to_process: + continue + pid_to_process[pid] = proc + pid_to_language[pid] = language + + # Determine selection limit (0 means unlimited) + max_processes = getattr(self.base_args, "max_processes_per_profiler", 50) + + if max_processes and max_processes > 0: + # Measure CPU percent for each unique process (short interval) + pid_cpu_pairs = [] + for pid, proc in pid_to_process.items(): + try: + cpu_percent = proc.cpu_percent(interval=0.1) + except Exception: + cpu_percent = 0.0 + pid_cpu_pairs.append((pid, cpu_percent)) + + # Select top-N by CPU + pid_cpu_pairs.sort(key=lambda x: x[1], reverse=True) + selected = pid_cpu_pairs[:max_processes] + selected_pids = {pid for pid, _ in selected} + pid_to_cpu = {pid: cpu for pid, cpu in selected} + + # Group back by language, sorted by CPU within language + for pid in selected_pids: + language = pid_to_language.get(pid) + if language is None: + continue + available_pids.setdefault(language, []).append(pid) + + for language, pids in available_pids.items(): + pids.sort(key=lambda p: pid_to_cpu.get(p, 0.0), reverse=True) + else: + # Unlimited: return all PIDs discovered per language, sorted + for language, processes in processes_by_language.items(): + pids = sorted([p.pid for p in processes]) + if pids: + available_pids[language] = pids if available_pids: total_pids = sum(len(pids) for pids in available_pids.values()) diff --git a/gprofiler/main.py b/gprofiler/main.py index 88bc7d06a..41372cdd3 100644 --- a/gprofiler/main.py +++ b/gprofiler/main.py @@ -682,7 +682,7 @@ def parse_cmd_args() -> configargparse.Namespace: "--max-processes", dest="max_processes_per_profiler", type=positive_integer, - default=0, + default=50, help="Maximum number of processes to profile per runtime profiler (0=unlimited). " "When exceeded, profiles only the top N processes by CPU usage. " "Does not affect system-wide profilers (perf, eBPF). Default: %(default)s", From dad397524938334214a099de3ca1a84aff1ed61a Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 18 Sep 2025 19:23:40 +0000 Subject: [PATCH 5/5] Improve pids scan --- gprofiler/heartbeat.py | 103 ++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 53 deletions(-) diff --git a/gprofiler/heartbeat.py b/gprofiler/heartbeat.py index f9cd947f2..f702f63bb 100644 --- a/gprofiler/heartbeat.py +++ b/gprofiler/heartbeat.py @@ -211,23 +211,28 @@ def __init__(self, base_args: configargparse.Namespace, heartbeat_client: Heartb def scan_available_pids(self) -> Dict[str, List[int]]: """Scan the host and return top-N PIDs by CPU, grouped by language.""" available_pids: Dict[str, List[int]] = {} - + try: # Import regex patterns from granulate_utils from granulate_utils.java import DETECTED_JAVA_PROCESSES_REGEX from granulate_utils.python import DETECTED_PYTHON_PROCESSES_REGEX - + # Define profiler patterns profiler_patterns = { 'java': DETECTED_JAVA_PROCESSES_REGEX, 'python': DETECTED_PYTHON_PROCESSES_REGEX, - 'ruby': r"^.+/libruby", # Ruby pattern - 'dotnet': r"^.+/libcoreclr\.so", # .NET pattern - 'php': r"^.+/(lib)?php[^/]*$", # PHP pattern + 'ruby': r"^.+/libruby", + 'dotnet': r"^.+/libcoreclr\.so", + 'php': r"^.+/(lib)?php[^/]*$", } - - # First, discover processes per language - processes_by_language: Dict[str, List[Process]] = {} + + # Selection limit (0 means unlimited) + max_processes = getattr(self.base_args, "max_processes_per_profiler", 50) + + # Single pass: scan per language, deduplicate by PID as we go + seen_pids: set[int] = set() + pid_to_info: Dict[int, tuple[Process, str]] = {} + for language, pattern in profiler_patterns.items(): try: if language == 'python': @@ -236,61 +241,53 @@ def scan_available_pids(self) -> Dict[str, List[int]]: else: processes = pgrep_maps(pattern) - if processes: - processes_by_language[language] = processes - logger.debug(f"Found {len(processes)} {language} processes") + if not processes: + continue + + if max_processes and max_processes > 0: + # Collect unique processes for later CPU-based selection + for proc in processes: + pid = getattr(proc, "pid", None) + if pid is None or pid in seen_pids: + continue + seen_pids.add(pid) + pid_to_info[pid] = (proc, language) + else: + # Unlimited mode: build the result directly while deduplicating + for proc in processes: + pid = getattr(proc, "pid", None) + if pid is None or pid in seen_pids: + continue + seen_pids.add(pid) + available_pids.setdefault(language, []).append(pid) except Exception as e: logger.debug(f"Error scanning {language} processes: {e}") continue - # Flatten and deduplicate by PID across languages - pid_to_process: Dict[int, Process] = {} - pid_to_language: Dict[int, str] = {} - for language, processes in processes_by_language.items(): - for proc in processes: - pid = getattr(proc, "pid", None) - if pid is None: - continue - if pid in pid_to_process: - continue - pid_to_process[pid] = proc - pid_to_language[pid] = language - - # Determine selection limit (0 means unlimited) - max_processes = getattr(self.base_args, "max_processes_per_profiler", 50) - if max_processes and max_processes > 0: - # Measure CPU percent for each unique process (short interval) - pid_cpu_pairs = [] - for pid, proc in pid_to_process.items(): + # Non-blocking CPU query (fast): may return 0.0 on first call but avoids per-PID sleep + from heapq import nlargest + from operator import itemgetter + + pid_cpu_lang: List[tuple[int, float, str]] = [] + for pid, (proc, language) in pid_to_info.items(): try: - cpu_percent = proc.cpu_percent(interval=0.1) + cpu_percent = proc.cpu_percent(interval=0.0) except Exception: cpu_percent = 0.0 - pid_cpu_pairs.append((pid, cpu_percent)) + pid_cpu_lang.append((pid, cpu_percent, language)) - # Select top-N by CPU - pid_cpu_pairs.sort(key=lambda x: x[1], reverse=True) - selected = pid_cpu_pairs[:max_processes] - selected_pids = {pid for pid, _ in selected} - pid_to_cpu = {pid: cpu for pid, cpu in selected} + # Select global top-N by CPU efficiently + top_k = nlargest(max_processes, pid_cpu_lang, key=itemgetter(1)) if pid_cpu_lang else [] - # Group back by language, sorted by CPU within language - for pid in selected_pids: - language = pid_to_language.get(pid) - if language is None: - continue + # Group back by language preserving CPU-descending order + for pid, _cpu, language in top_k: available_pids.setdefault(language, []).append(pid) - - for language, pids in available_pids.items(): - pids.sort(key=lambda p: pid_to_cpu.get(p, 0.0), reverse=True) else: - # Unlimited: return all PIDs discovered per language, sorted - for language, processes in processes_by_language.items(): - pids = sorted([p.pid for p in processes]) - if pids: - available_pids[language] = pids - + # Unlimited: sort per language for determinism + for language, pids in available_pids.items(): + pids.sort() + if available_pids: total_pids = sum(len(pids) for pids in available_pids.values()) logger.debug( @@ -299,10 +296,10 @@ def scan_available_pids(self) -> Dict[str, List[int]]: ) else: logger.debug("No available PIDs found for any supported languages") - + except Exception as e: logger.debug(f"Error scanning available PIDs: {e}") - + return available_pids def start_heartbeat_loop(self):