chore: release main

.release-please-manifest.json

@@ -1,4 +1,4 @@
 {
-  ".": "0.22.2",
-  "adk": "0.22.2"
+  ".": "0.23.0",
+  "adk": "0.23.0"
 }

.stats.yml

@@ -1,4 +1,4 @@
 configured_endpoints: 75
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/sgp/agentex-sdk-644a4ec06aa1f055c614cbef3379684819a4edd84eeb20d2fb29ae01663622a3.yml
-openapi_spec_hash: a6a4dc0c09691ac9783bf38e9653a464
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/sgp/agentex-sdk-330ce4f0d8feed6caeb73d6b12277cfd89f6ad85535b8c8a6f509743b0b6f8cb.yml
+openapi_spec_hash: ed6b33682c511df6de538714c0864aa3
 config_hash: 593e89b291976a5e84e4c3c3f8324354

CHANGELOG.md

@@ -12,6 +12,16 @@
 
 * **tracing:** emit OTel metrics for async span queue depth, batch drain, and SGP export success/failure (HTTP status labels). Disable SDK-side recording with ``AGENTEX_TRACING_METRICS=0``.
 
+## 0.23.0 (2026-08-07)
+
+Full Changelog: [agentex-client-v0.22.2...agentex-client-v0.23.0](https://github.com/scaleapi/scale-agentex-python/compare/agentex-client-v0.22.2...agentex-client-v0.23.0)
+
+### Features
+
+* propagate error categories to SGP spans ([#486](https://github.com/scaleapi/scale-agentex-python/issues/486)) ([f2b1808](https://github.com/scaleapi/scale-agentex-python/commit/f2b18087c5049f7e2a159f8e1ad3bf069b01eb23))
+* **tracing:** correlate business spans with obs via dedicated wrapper span ([#484](https://github.com/scaleapi/scale-agentex-python/issues/484)) ([72732b7](https://github.com/scaleapi/scale-agentex-python/commit/72732b7c07700df840a2308424154f11a30e39f2))
+* **tracing:** propagate OTel trace context across Temporal boundaries ([#485](https://github.com/scaleapi/scale-agentex-python/issues/485)) ([da7ea15](https://github.com/scaleapi/scale-agentex-python/commit/da7ea1558683da05a3f9ecb119b91bf873437be1))
+
 ## 0.22.2 (2026-07-30)
 
 Full Changelog: [agentex-client-v0.22.1...agentex-client-v0.22.2](https://github.com/scaleapi/scale-agentex-python/compare/agentex-client-v0.22.1...agentex-client-v0.22.2)

adk/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.23.0 (2026-08-07)
+
+Full Changelog: [agentex-sdk-v0.22.2...agentex-sdk-v0.23.0](https://github.com/scaleapi/scale-agentex-python/compare/agentex-sdk-v0.22.2...agentex-sdk-v0.23.0)
+
+### Features
+
+* propagate error categories to SGP spans ([#486](https://github.com/scaleapi/scale-agentex-python/issues/486)) ([f2b1808](https://github.com/scaleapi/scale-agentex-python/commit/f2b18087c5049f7e2a159f8e1ad3bf069b01eb23))
+
 ## 0.22.2 (2026-07-30)
 
 Full Changelog: [agentex-sdk-v0.22.1...agentex-sdk-v0.22.2](https://github.com/scaleapi/scale-agentex-python/compare/agentex-sdk-v0.22.1...agentex-sdk-v0.22.2)

adk/pyproject.toml

@@ -4,7 +4,7 @@
 # (agentex/{__init__.py, _*.py, types/, resources/}) ships from the slim
 # sibling package `agentex-client` which is pinned as a runtime dep.
 name = "agentex-sdk"
-version = "0.22.2"
+version = "0.23.0"
 description = "Agent Development Kit (ADK) overlay for the Agentex API — FastACP server, Temporal workflows, LLM provider integrations, observability"
 license = "Apache-2.0"
 authors = [
@@ -53,7 +53,7 @@ dependencies = [
     "pydantic-ai-slim>=1.0,<2",
     "langgraph-checkpoint>=2.0.0",
     "scale-gp>=0.1.0a59",
-    "scale-gp-beta>=0.2.0",
+    "scale-gp-beta>=0.5.0",
     "mcp>=1.4.1",
     # Observability
     "ddtrace>=3.13.0",

pyproject.toml

@@ -3,7 +3,7 @@
 # overlay (formerly `src/agentex/lib/*`) now lives in `adk/` and ships
 # as the sibling `agentex-sdk` package — see `adk/pyproject.toml`.
 name = "agentex-client"
-version = "0.22.2"
+version = "0.23.0"
 description = "The official Python REST client for the Agentex API"
 dynamic = ["readme"]
 license = "Apache-2.0"

src/agentex/_version.py

@@ -1,4 +1,4 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 __title__ = "agentex"
-__version__ = "0.22.2"  # x-release-please-version
+__version__ = "0.23.0"  # x-release-please-version

src/agentex/lib/adk/_modules/tracing.py

@@ -19,6 +19,7 @@
     StartSpanParams,
     TracingActivityName,
 )
+from agentex.lib.core.tracing.span_error import set_span_error
 from agentex.lib.core.tracing.tracer import AsyncTracer
 from agentex.lib.core.harness.types import TurnUsage
 from agentex.types.span import Span
@@ -236,6 +237,24 @@ async def span(
         )
         try:
             yield span
+        except Exception as exc:
+            # Record the failure on the span so the obs span reflects the error
+            # instead of a false green. Agents use THIS context manager (not
+            # AsyncTrace.span, which is the only other place set_span_error is
+            # called), so without this a failed step closes green. end_span (in
+            # finally) reads it via get_span_error and propagates it to
+            # close_obs_span. Stored on span.data, so it round-trips through the
+            # END_SPAN activity on the Temporal path too.
+            #
+            # Guard set_span_error itself: it's obs work and must never replace
+            # the app's exception on the way out. We always re-raise the ORIGINAL
+            # exc regardless.
+            if span:
+                try:
+                    set_span_error(span, exc)
+                except Exception:  # pragma: no cover - obs must not break app path
+                    pass
+            raise
         finally:
             if span:
                 await self.end_span(

src/agentex/lib/core/clients/temporal/utils.py

@@ -9,6 +9,8 @@
 from temporalio.converter import PayloadCodec, DataConverter
 from temporalio.contrib.pydantic import pydantic_data_converter
 
+from agentex.lib.core.tracing.temporal import temporal_tracing_interceptors
+
 # class DateTimeJSONEncoder(AdvancedJSONEncoder):
 #     def default(self, o: Any) -> Any:
 #         if isinstance(o, datetime.datetime):
@@ -136,6 +138,9 @@ async def get_temporal_client(
     connect_kwargs: dict[str, Any] = {
         "target_host": temporal_address,
         "plugins": plugins,
+        # Propagate OTel trace context on outbound start_workflow / execute_activity
+        # (enabled by default; AGENTEX_TEMPORAL_TRACE_INTERCEPTOR_ENABLED=false to disable).
+        "interceptors": temporal_tracing_interceptors(),
     }
 
     if data_converter is not None:

src/agentex/lib/core/temporal/services/temporal_task_service.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
+import sys
 from typing import Any
 from datetime import timedelta
+from contextlib import contextmanager
+from collections.abc import Iterator
 
 from agentex.types.task import Task
 from agentex.types.agent import Agent
@@ -13,6 +16,55 @@
 from agentex.lib.core.clients.temporal.temporal_client import TemporalClient
 
 
+@contextmanager
+def _acp_dispatch_span(name: str, task_id: str | None = None) -> Iterator[None]:
+    """Wrap an ACP -> Temporal dispatch (start_workflow / signal) in an OTel span.
+
+    The Temporal OpenTelemetry interceptor propagates trace context by injecting
+    the CURRENTLY ACTIVE span into the Temporal message headers on the caller
+    side (``start_workflow`` / ``signal_workflow``); the worker then extracts it
+    and roots the workflow / activity spans under it. But the ACP server dispatches
+    from a bare async handler with no active span, so nothing is injected and the
+    workflow's activities become DETACHED trace roots -- the business work shows up
+    in Tempo as a fresh trace with no link back to the ``task/create`` /
+    ``event/send`` that triggered it.
+
+    Opening a span here gives the interceptor something to inject. It becomes a
+    child of the ingress request span when one is active (front-of-request
+    propagation), or a fresh per-turn root otherwise.
+
+    Fail-open across the WHOLE obs setup, not just the import: ``get_tracer`` and
+    entering ``start_as_current_span`` run the sampler and every
+    ``SpanProcessor.on_start`` (the SDK does not guard those), so a broken
+    provider or a custom sampler/processor that raises would otherwise fail the
+    dispatch itself. If any of it fails we run the dispatch untraced. The dispatch
+    body (the ``yield``) is OUTSIDE the guard so its exceptions still propagate.
+    """
+    span_cm = None
+    try:
+        from opentelemetry import trace as _otel_trace
+
+        tracer = _otel_trace.get_tracer("agentex.acp")
+        # task_id goes on an attribute, NOT in the span name: a per-task span name is
+        # high-cardinality and breaks span-name aggregation in Tempo.
+        attributes = {"agentex.task_id": task_id} if task_id else None
+        span_cm = tracer.start_as_current_span(name, kind=_otel_trace.SpanKind.PRODUCER, attributes=attributes)
+        span_cm.__enter__()
+    except Exception:  # pragma: no cover - obs must never break a dispatch
+        span_cm = None
+
+    try:
+        yield
+    finally:
+        if span_cm is not None:
+            # Pass exc info so the span reflects a failed dispatch; guard __exit__
+            # so closing the span can never mask the dispatch outcome.
+            try:
+                span_cm.__exit__(*sys.exc_info())
+            except Exception:  # pragma: no cover - best-effort close
+                pass
+
+
 class TemporalTaskService:
     """
     Submits Agent agent_tasks to the async runtime for execution.
@@ -26,7 +78,6 @@ def __init__(
         self._temporal_client = temporal_client
         self._env_vars = env_vars
 
-
     async def submit_task(self, agent: Agent, task: Task, params: dict[str, Any] | None) -> str:
         """
         Submit a task to the async runtime for execution.
@@ -37,22 +88,19 @@ async def submit_task(self, agent: Agent, task: Task, params: dict[str, Any] | N
         # indefinitely, which long-lived chat/session agents rely on). A positive
         # value bounds the whole continue-as-new chain's wall-clock lifetime.
         timeout_seconds = self._env_vars.WORKFLOW_EXECUTION_TIMEOUT_SECONDS
-        execution_timeout = (
-            timedelta(seconds=timeout_seconds)
-            if timeout_seconds and timeout_seconds > 0
-            else None
-        )
-        return await self._temporal_client.start_workflow(
-            workflow=self._env_vars.WORKFLOW_NAME,
-            arg=CreateTaskParams(
-                agent=agent,
-                task=task,
-                params=params,
-            ),
-            id=task.id,
-            task_queue=self._env_vars.WORKFLOW_TASK_QUEUE,
-            execution_timeout=execution_timeout,
-        )
+        execution_timeout = timedelta(seconds=timeout_seconds) if timeout_seconds and timeout_seconds > 0 else None
+        with _acp_dispatch_span("acp.task_create", task_id=task.id):
+            return await self._temporal_client.start_workflow(
+                workflow=self._env_vars.WORKFLOW_NAME,
+                arg=CreateTaskParams(
+                    agent=agent,
+                    task=task,
+                    params=params,
+                ),
+                id=task.id,
+                task_queue=self._env_vars.WORKFLOW_TASK_QUEUE,
+                execution_timeout=execution_timeout,
+            )
 
     async def get_state(self, task_id: str) -> WorkflowState:
         """
@@ -63,16 +111,17 @@ async def get_state(self, task_id: str) -> WorkflowState:
         )
 
     async def send_event(self, agent: Agent, task: Task, event: Event, request: dict | None = None) -> None:
-        return await self._temporal_client.send_signal(
-            workflow_id=task.id,
-            signal=SignalName.RECEIVE_EVENT.value,
-            payload=SendEventParams(
-                agent=agent,
-                task=task,
-                event=event,
-                request=request,
-            ).model_dump(),
-        )
+        with _acp_dispatch_span("acp.event_send", task_id=task.id):
+            return await self._temporal_client.send_signal(
+                workflow_id=task.id,
+                signal=SignalName.RECEIVE_EVENT.value,
+                payload=SendEventParams(
+                    agent=agent,
+                    task=task,
+                    event=event,
+                    request=request,
+                ).model_dump(),
+            )
 
     async def interrupt(self, agent: Agent, task: Task, request: dict | None = None) -> None:
         """Forward a task/interrupt to the running workflow as a dedicated signal.

src/agentex/lib/core/temporal/workers/worker.py

@@ -29,6 +29,7 @@
 
 from agentex.lib.utils.logging import make_logger
 from agentex.lib.utils.registration import register_agent
+from agentex.lib.core.tracing.temporal import temporal_tracing_interceptors
 from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.compat.version_guard import assert_backend_compatible
 
@@ -126,6 +127,9 @@ async def get_temporal_client(
     connect_kwargs: dict[str, Any] = {
         "target_host": temporal_address,
         "plugins": plugins,
+        # Propagate OTel trace context on outbound start_workflow / execute_activity
+        # (enabled by default; AGENTEX_TEMPORAL_TRACE_INTERCEPTOR_ENABLED=false to disable).
+        "interceptors": temporal_tracing_interceptors(),
     }
 
     if data_converter is not None:
@@ -229,7 +233,9 @@ async def run(
             max_concurrent_activities=self.max_concurrent_activities,
             build_id=str(uuid.uuid4()),
             debug_mode=debug_enabled,  # Disable deadlock detection in debug mode
-            interceptors=self.interceptors,  # Pass interceptors to Worker
+            # Tracing interceptor OUTERMOST so business interceptors (and the spans
+            # they create) nest under the propagated workflow/activity span.
+            interceptors=[*temporal_tracing_interceptors(), *self.interceptors],
         )
 
         logger.info(f"Starting workers for task queue: {self.task_queue}")

src/agentex/lib/core/tracing/__init__.py

@@ -1,6 +1,12 @@
 from agentex.types.span import Span
 from agentex.lib.core.tracing.trace import Trace, AsyncTrace
 from agentex.lib.core.tracing.tracer import Tracer, AsyncTracer
+from agentex.lib.core.tracing.span_error import (
+    ErrorCategory,
+    PlatformError,
+    ApplicationError,
+    CategorizedError,
+)
 from agentex.lib.core.tracing.span_queue import (
     AsyncSpanQueue,
     get_default_span_queue,
@@ -13,6 +19,10 @@
     "Span",
     "Tracer",
     "AsyncTracer",
+    "CategorizedError",
+    "ApplicationError",
+    "PlatformError",
+    "ErrorCategory",
     "AsyncSpanQueue",
     "get_default_span_queue",
     "shutdown_default_span_queue",

src/agentex/lib/core/tracing/obs_ids.py

@@ -11,14 +11,20 @@
 persisted business span to the Tempo/Datadog trace for the turn that produced it,
 while the business trace still groups the entire run by task id.
 
-Source selection follows SGP_OBS_MODE, matching egp-api-backend:
+Source selection follows SGP_OBS_MODE:
   - unset / "dd_only": ddtrace context (current stack)
-  - "dual":            OTel/LGTM preferred, ddtrace fallback
   - "lgtm":            OTel/LGTM only
 
+("dual" was removed: co-resident ddtrace+OTel can't be bridged in-process --
+you can't run ddtrace-run and the OTel operator's auto-instrumentation in the
+same process, and DD_TRACE_OTEL_ENABLED yields a single tracer with nothing to
+bridge. Two-backend export is a collector fan-out under "lgtm", not a mode here.
+An unrecognized SGP_OBS_MODE -- including a stale "dual" -- degrades to dd_only.)
+
 This never fabricates ids -- if no observability context is active, it returns
 an empty dict and the span is simply not tagged.
 """
+
 from __future__ import annotations
 
 import os
@@ -27,10 +33,9 @@
 __all__ = ("get_obs_mode", "obs_correlation")
 
 DD_ONLY = "dd_only"
-DUAL = "dual"
 LGTM = "lgtm"
 _DEFAULT_MODE = DD_ONLY
-_VALID_MODES = (DD_ONLY, DUAL, LGTM)
+_VALID_MODES = (DD_ONLY, LGTM)
 
 
 def get_obs_mode() -> str:
@@ -64,20 +69,31 @@ def _ddtrace_ids() -> Optional[Tuple[str, str]]:
     return None
 
 
-def obs_correlation() -> Dict[str, str]:
-    """Return ``{"obs.trace_id": ..., "obs.span_id": ...}`` for the active
+def obs_correlation(prefer_otel: bool = False) -> Dict[str, str]:
+    """Return ``{"obs_trace_id": ..., "obs_span_id": ...}`` for the active
     observability context, or ``{}`` if none is active.
 
+    These land in the business span's ``data`` -> egp ``operation_metadata``
+    (an existing JSONB column, GIN-indexed) -> ClickHouse ``metadata_raw``, so
+    the correlation edge needs no schema migration. Underscored keys (not
+    dotted) keep them addressable via Postgres JSON paths
+    (``operation_metadata->>'obs_trace_id'``).
+
+    ``prefer_otel``: on the Temporal path the active span is the temporalio OTel
+    ``TracingInterceptor`` span regardless of ``SGP_OBS_MODE``, so callers there
+    read OTel first (falling back to ddtrace) -- otherwise the default ``dd_only``
+    mode would read ids for an unrelated ddtrace trace, not the activity span.
+
     Never fabricates ids -- this is a correlation tag, not the span's id.
     """
-    mode = get_obs_mode()
-    if mode == LGTM:
-        ids = _lgtm_ids()
-    elif mode == DUAL:
-        ids = _lgtm_ids() or _ddtrace_ids()
-    else:  # dd_only
-        ids = _ddtrace_ids()
+    try:
+        if prefer_otel:
+            ids = _lgtm_ids() or _ddtrace_ids()
+        else:
+            ids = _lgtm_ids() if get_obs_mode() == LGTM else _ddtrace_ids()
+    except Exception:  # obs must never fail an app call
+        return {}
 
     if not ids:
         return {}
-    return {"obs.trace_id": ids[0], "obs.span_id": ids[1]}
+    return {"obs_trace_id": ids[0], "obs_span_id": ids[1]}