From 13a57205ec83e060e5ae9f6d45a3cf2d3068f67f Mon Sep 17 00:00:00 2001 From: minixalpha Date: Wed, 2 Sep 2026 22:46:48 +0800 Subject: [PATCH 1/9] docs: outline Harbor end-to-end acceptance --- docs/dev_notes/zh-CN/0.8.x.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 5caf3ae..175473b 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -241,3 +241,17 @@ uv run --project benchmarks/harbor \ ```bash uv run pytest ``` + +#### Harbor 接入与端到端验收 + +**要做什么：** + +1. 让仓库内的 Harbor adapter 为每次 trial 指定容器内的 trajectory 输出路径，并在 agent 结束后告诉 Harbor 该文件的路径和格式，由 Harbor 读取 agent 生成的 ATIF-v1.7 文件。 +2. 将 ATIF 中的完整 steps，以及 step metrics 和 final metrics 中的 token、cache token、cost 与完整性状态回填给 Harbor。 +3. 明确 adapter 的职责边界与失败语义：adapter 只负责路径和数据交接，不解析 Event Journal，也不维护 native trajectory 转换；trajectory 缺失、无效或部分完成时给出明确诊断，不能误报成零消耗或完整结果。 + +**如何验证：** + +1. 扩展 `benchmarks/harbor/tests` 中的 adapter 契约测试，验证 trajectory 路径能够正确传入容器，ATIF 能够被声明和读取。 +2. 用契约测试验证 steps、tokens 和 cost 的统计回填，并覆盖 trajectory 文件缺失、校验失败及指标不完整时的行为。 +3. 使用仓库固定的 Harbor 版本运行一次真实 Terminal-Bench trial，核对 agent 日志、Harbor 保存的 trajectory、steps、tokens、cost 和 reward，确认从容器安装、任务执行、trajectory 落盘、Harbor 采集到官方 verifier 评分的完整链路，并区分任务解答失败与 adapter 基础设施故障。 From 9335009100f9f37721dbe15d3dfbea55e3ac0baf Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 3 Sep 2026 22:04:53 +0800 Subject: [PATCH 2/9] feat(benchmark): collect Harbor ATIF trajectories --- .gitignore | 3 + benchmarks/harbor/README.md | 11 +- benchmarks/harbor/README.zh-CN.md | 9 +- .../harbor/src/harbor_adapter/adapter.py | 71 +++++++++ benchmarks/harbor/tests/test_adapter.py | 139 ++++++++++++++++++ docs/dev_notes/zh-CN/0.8.x.md | 20 +++ 6 files changed, 251 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 64e5f11..9ac5658 100644 --- a/.gitignore +++ b/.gitignore @@ -218,3 +218,6 @@ __marimo__/ .streamlit/secrets.toml references/ + +# Harbor benchmark results +/jobs/ diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 6dfeef3..cfd8183 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -35,13 +35,22 @@ uv run --project benchmarks/harbor harbor run \ Use `--agent-kwarg version=` instead of `git_ref` to install a published PyPI release. The two pins are mutually exclusive. If neither is provided, the adapter installs the latest published release. For reproducible -benchmark results, always provide one of them. +benchmark results, always provide one of them. Prefer a full 40-character commit +SHA for `git_ref`; Harbor may parse an unquoted abbreviated SHA such as `83e6271` +as a number. To use an abbreviated revision, preserve its string type with +`--agent-kwarg 'git_ref="83e6271"'`. The adapter sends the instruction through stdin, runs the agent in the task container's current directory, and saves combined stdout/stderr to `/logs/agent/nanopycodeagent.txt`. It uses the CLI's 50-turn default; override that with `--agent-kwarg max_turns=20`. +The adapter also asks the agent to write an ATIF-v1.7 trajectory directly to +`/logs/agent/trajectory.json`. Harbor collects that file as the trial's native +ATIF output and backfills prompt, completion, cache-token, and cost totals into +the agent result. Missing, invalid, or partial trajectories remain explicitly +diagnosed; unknown usage or cost is not reported as zero. + `ANTHROPIC_MODEL` takes precedence over Harbor's `--model`. When it is absent, the adapter strips the first provider prefix from `--model`. Harbor-native provider credentials and configured base URLs are also normalized to the diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 62ac9dd..fe298ab 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -34,12 +34,19 @@ uv run --project benchmarks/harbor harbor run \ 如需安装已经发布到 PyPI 的版本，请使用 `--agent-kwarg version=` 代替 `git_ref`。这两个版本参数互斥； 如果都未提供，adapter 会安装最新发布版本。为了让 benchmark 结果可复现，请始终 -提供其中一个参数。 +提供其中一个参数。`git_ref` 最好使用完整的 40 位 commit SHA；Harbor 可能把 +`83e6271` 这样的无引号短 SHA 解析成数字。如果需要使用短 revision，请通过 +`--agent-kwarg 'git_ref="83e6271"'` 保留其字符串类型。 adapter 通过 stdin 发送任务指令，在 task 容器的当前目录中运行 agent，并把合并后 的 stdout/stderr 保存到 `/logs/agent/nanopycodeagent.txt`。它默认沿用 CLI 的 50 轮限制；可以通过 `--agent-kwarg max_turns=20` 覆盖此设置。 +adapter 还会要求 agent 将 ATIF-v1.7 trajectory 直接写入 +`/logs/agent/trajectory.json`。Harbor 会把该文件作为 trial 的原生 ATIF 输出采集， +并将 prompt、completion、cache token 和 cost 汇总回填到 agent result。trajectory +缺失、无效或指标不完整时会保留明确诊断；未知 usage 或 cost 不会被记成零。 + `ANTHROPIC_MODEL` 的优先级高于 Harbor 的 `--model`。未设置该环境变量时，adapter 会移除 `--model` 中的第一个 provider 前缀。Harbor 原生的 provider 凭证和已经配置 的 base URL 也会转换为 nanoPyCodeAgent SDK 所需的 `ANTHROPIC_*` 变量。 diff --git a/benchmarks/harbor/src/harbor_adapter/adapter.py b/benchmarks/harbor/src/harbor_adapter/adapter.py index 0624fc8..b4c9e0e 100644 --- a/benchmarks/harbor/src/harbor_adapter/adapter.py +++ b/benchmarks/harbor/src/harbor_adapter/adapter.py @@ -13,17 +13,20 @@ from harbor.agents.model_connection import ModelConnectionSpec from harbor.environments.base import BaseEnvironment from harbor.models.agent.context import AgentContext +from harbor.models.trajectories.trajectory import Trajectory _DEFAULT_MAX_TURNS = 50 _PACKAGE_NAME = "nanoPyCodeAgent" _REPOSITORY_URL = "https://github.com/minixalpha/nanoPyCodeAgent.git" _UV_VERSION = "0.9.11" _PATH_SETUP = 'export PATH="$HOME/.local/bin:$PATH"; ' +_TRAJECTORY_PATH = "/logs/agent/trajectory.json" class NanoPyCodeAgent(BaseInstalledAgent): """Install and run nanoPyCodeAgent inside a Harbor task environment.""" + SUPPORTS_ATIF = True MODEL_CONNECTION = ModelConnectionSpec( api_key_envs=("ANTHROPIC_API_KEY",), base_url_envs=("ANTHROPIC_BASE_URL",), @@ -39,6 +42,11 @@ class NanoPyCodeAgent(BaseInstalledAgent): def __init__(self, *args, git_ref: str | None = None, **kwargs): if git_ref is not None: + if not isinstance(git_ref, str): + raise ValueError( + "git_ref must be a string; use a full commit SHA or quote " + "an abbreviated revision in --agent-kwarg" + ) git_ref = git_ref.strip() if not git_ref: raise ValueError("git_ref must not be blank") @@ -131,7 +139,70 @@ async def run( f"unset {instruction_env_var}; " f'printf "%s" "${instruction_shell_var}" | ' f"nanoPyCodeAgent {cli_flags} " + f"--trajectory {_TRAJECTORY_PATH} " "2>&1 | tee /logs/agent/nanopycodeagent.txt" ), env=env, ) + + @override + def populate_context_post_run(self, context: AgentContext) -> None: + trajectory_path = self.logs_dir / "trajectory.json" + try: + trajectory = Trajectory.model_validate_json( + trajectory_path.read_text(encoding="utf-8") + ) + if trajectory.schema_version != "ATIF-v1.7": + raise ValueError( + f"expected ATIF-v1.7, got {trajectory.schema_version}" + ) + except FileNotFoundError: + self._record_trajectory_diagnostic(context, "missing") + self.logger.warning("No ATIF trajectory found at %s", trajectory_path) + return + except (OSError, UnicodeError, ValueError) as exc: + self._record_trajectory_diagnostic( + context, + "invalid", + error=f"{type(exc).__name__}: {exc}", + ) + self.logger.warning( + "Failed to read ATIF trajectory at %s: %s", + trajectory_path, + exc, + ) + return + + metrics = trajectory.final_metrics + extra = metrics.extra if metrics and metrics.extra else {} + is_partial = extra.get("usage_complete") is False or ( + extra.get("cost_is_partial") is True + ) + self._record_trajectory_diagnostic( + context, + "partial" if is_partial else "complete", + total_steps=len(trajectory.steps), + usage_complete=extra.get("usage_complete"), + cost_is_partial=extra.get("cost_is_partial"), + known_cost_usd=extra.get("known_cost_usd"), + missing_generation_ids=extra.get("missing_generation_ids"), + ) + if metrics is None: + return + context.n_input_tokens = metrics.total_prompt_tokens + context.n_output_tokens = metrics.total_completion_tokens + context.n_cache_tokens = metrics.total_cached_tokens + context.cost_usd = metrics.total_cost_usd + + @staticmethod + def _record_trajectory_diagnostic( + context: AgentContext, + status: str, + **details: object, + ) -> None: + context.metadata = context.metadata or {} + context.metadata["trajectory"] = { + "format": "ATIF-v1.7", + "status": status, + **{key: value for key, value in details.items() if value is not None}, + } diff --git a/benchmarks/harbor/tests/test_adapter.py b/benchmarks/harbor/tests/test_adapter.py index 61b0930..fafe681 100644 --- a/benchmarks/harbor/tests/test_adapter.py +++ b/benchmarks/harbor/tests/test_adapter.py @@ -1,10 +1,12 @@ """Contract tests for the repository-local Harbor installed-agent adapter.""" import asyncio +import json from pathlib import Path from types import SimpleNamespace import pytest +from harbor.models.agent.context import AgentContext from harbor_adapter import NanoPyCodeAgent @@ -26,6 +28,32 @@ def make_adapter(tmp_path: Path, **kwargs) -> NanoPyCodeAgent: return NanoPyCodeAgent(logs_dir=tmp_path, **kwargs) +def write_trajectory(tmp_path: Path, final_metrics: dict) -> None: + (tmp_path / "trajectory.json").write_text( + json.dumps( + { + "schema_version": "ATIF-v1.7", + "session_id": "run-1", + "trajectory_id": "run-1", + "agent": { + "name": "nanoPyCodeAgent", + "version": "0.8.0", + "model_name": "test-model", + }, + "steps": [ + { + "step_id": 1, + "source": "user", + "message": "fix it", + } + ], + "final_metrics": final_metrics, + } + ), + encoding="utf-8", + ) + + def test_install_pins_the_requested_release_and_checks_its_version(tmp_path): adapter = make_adapter(tmp_path, version="0.8.0") environment = RecordingEnvironment() @@ -67,6 +95,14 @@ def test_install_source_rejects_ambiguous_or_blank_pins(tmp_path): make_adapter(tmp_path, git_ref=" ") +def test_install_source_rejects_a_revision_coerced_to_a_number(tmp_path): + with pytest.raises( + ValueError, + match="use a full commit SHA or quote an abbreviated revision", + ): + make_adapter(tmp_path, git_ref=float("inf")) + + def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path): instruction = 'fix "quoted" input; echo $TOKEN\nthen run the tests' adapter = make_adapter( @@ -88,6 +124,7 @@ def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path assert instruction not in command assert 'printf "%s" "$harbor_nanopycodeagent_instruction_' in command assert "nanoPyCodeAgent --max-turns 20" in command + assert "--trajectory /logs/agent/trajectory.json" in command assert command.endswith("2>&1 | tee /logs/agent/nanopycodeagent.txt") run_env = run_call["env"] @@ -123,3 +160,105 @@ def test_run_normalizes_harbor_provider_configuration_for_the_anthropic_sdk( assert run_env["ANTHROPIC_BASE_URL"] == "https://openrouter.example/api" assert run_env["ANTHROPIC_MODEL"] == "deepseek/deepseek-v4-flash-0731" assert "OPENROUTER_API_KEY" not in run_env + + +def test_adapter_declares_atif_support_and_populates_complete_context(tmp_path): + adapter = make_adapter(tmp_path) + write_trajectory( + tmp_path, + { + "total_prompt_tokens": 120, + "total_completion_tokens": 30, + "total_cached_tokens": 20, + "total_cost_usd": 0.0042, + "total_steps": 1, + }, + ) + context = AgentContext() + + adapter.populate_context_post_run(context) + + assert adapter.SUPPORTS_ATIF is True + assert context.n_input_tokens == 120 + assert context.n_output_tokens == 30 + assert context.n_cache_tokens == 20 + assert context.cost_usd == 0.0042 + assert context.metadata == { + "trajectory": { + "format": "ATIF-v1.7", + "status": "complete", + "total_steps": 1, + } + } + + +def test_partial_trajectory_preserves_known_values_and_completeness(tmp_path): + adapter = make_adapter(tmp_path) + write_trajectory( + tmp_path, + { + "total_steps": 1, + "extra": { + "usage_complete": False, + "known_cost_usd": 0.001, + "cost_is_partial": True, + "missing_generation_ids": ["generation-1"], + }, + }, + ) + context = AgentContext() + + adapter.populate_context_post_run(context) + + assert context.n_input_tokens is None + assert context.n_output_tokens is None + assert context.n_cache_tokens is None + assert context.cost_usd is None + assert context.metadata == { + "trajectory": { + "format": "ATIF-v1.7", + "status": "partial", + "total_steps": 1, + "usage_complete": False, + "cost_is_partial": True, + "known_cost_usd": 0.001, + "missing_generation_ids": ["generation-1"], + } + } + + +@pytest.mark.parametrize( + ("contents", "expected_status"), + [ + (None, "missing"), + ("not JSON", "invalid"), + ( + json.dumps( + { + "schema_version": "ATIF-v1.6", + "agent": {"name": "agent", "version": "1"}, + "steps": [ + {"step_id": 1, "source": "user", "message": "task"} + ], + } + ), + "invalid", + ), + ], +) +def test_missing_or_invalid_trajectory_records_a_diagnostic( + tmp_path, + contents, + expected_status, +): + adapter = make_adapter(tmp_path) + if contents is not None: + (tmp_path / "trajectory.json").write_text(contents, encoding="utf-8") + context = AgentContext() + + adapter.populate_context_post_run(context) + + diagnostic = context.metadata["trajectory"] + assert diagnostic["format"] == "ATIF-v1.7" + assert diagnostic["status"] == expected_status + assert ("error" in diagnostic) is (expected_status == "invalid") diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 175473b..530200b 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -255,3 +255,23 @@ uv run pytest 1. 扩展 `benchmarks/harbor/tests` 中的 adapter 契约测试，验证 trajectory 路径能够正确传入容器，ATIF 能够被声明和读取。 2. 用契约测试验证 steps、tokens 和 cost 的统计回填，并覆盖 trajectory 文件缺失、校验失败及指标不完整时的行为。 3. 使用仓库固定的 Harbor 版本运行一次真实 Terminal-Bench trial，核对 agent 日志、Harbor 保存的 trajectory、steps、tokens、cost 和 reward，确认从容器安装、任务执行、trajectory 落盘、Harbor 采集到官方 verifier 评分的完整链路，并区分任务解答失败与 adapter 基础设施故障。 + +**端到端测试步骤：** + +1. 确认 `docker info` 可用，设置 `ANTHROPIC_API_KEY`、`ANTHROPIC_BASE_URL` 和 `ANTHROPIC_MODEL`，并准备一个远端可安装的完整 40 位 commit SHA；短 SHA 必须写成 `--agent-kwarg 'git_ref="83e6271"'`，否则可能被 Harbor 解析成数字。 +2. 从仓库根目录运行一次固定任务： + + ```bash + uv run --project benchmarks/harbor harbor run \ + --task terminal-bench/openssl-selfsigned-cert \ + --agent harbor_adapter:NanoPyCodeAgent \ + --agent-kwarg git_ref= \ + --model openrouter/deepseek/deepseek-v4-flash-0731 \ + --env docker \ + --n-concurrent 1 \ + --n-attempts 1 + ``` + +3. 在 Harbor 输出的 `jobs///` 中检查 `agent/nanopycodeagent.txt`、`agent/trajectory.json`、`verifier/test-stdout.txt` 和 `result.json`；运行 `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator ` 校验 ATIF，并确认 schema 为 ATIF-v1.7、steps 完整、`final_metrics` 与 `agent_result` 的 tokens/cache tokens/cost 一致、`exception_info` 为空且 verifier 产生 reward。 + +本次实跑的官方 verifier 为 6/6、reward 为 1.0、Harbor 异常为 0；trajectory 共 11 个 steps，记录 49,878 input tokens、42,240 cache tokens、5,845 output tokens 和 0.00222441 USD，所有 10 次成本对账均首次成功。 From 3bd6597517a4405fc11056b5f3baa73832222984 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 3 Sep 2026 22:09:06 +0800 Subject: [PATCH 3/9] docs: clarify Harbor end-to-end steps --- docs/dev_notes/zh-CN/0.8.x.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 530200b..ebdb3e1 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -265,13 +265,15 @@ uv run pytest uv run --project benchmarks/harbor harbor run \ --task terminal-bench/openssl-selfsigned-cert \ --agent harbor_adapter:NanoPyCodeAgent \ - --agent-kwarg git_ref= \ + --agent-kwarg git_ref=83e6271e9e9d0a22fb7c2510a8df76acff320cd3 \ --model openrouter/deepseek/deepseek-v4-flash-0731 \ --env docker \ --n-concurrent 1 \ --n-attempts 1 ``` -3. 在 Harbor 输出的 `jobs///` 中检查 `agent/nanopycodeagent.txt`、`agent/trajectory.json`、`verifier/test-stdout.txt` 和 `result.json`；运行 `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator ` 校验 ATIF，并确认 schema 为 ATIF-v1.7、steps 完整、`final_metrics` 与 `agent_result` 的 tokens/cache tokens/cost 一致、`exception_info` 为空且 verifier 产生 reward。 +3. 从 Harbor 输出中取得本次结果目录 `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/`，确认其中存在 `agent/nanopycodeagent.txt`、`agent/trajectory.json`、`verifier/test-stdout.txt` 和 `result.json`。 +4. 运行 `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json`，确认 Harbor 0.21.0 validator 返回 0。 +5. 对照 `agent/trajectory.json` 与 `result.json`，确认 schema 为 ATIF-v1.7、steps 完整、`final_metrics` 与 `agent_result` 的 tokens/cache tokens/cost 一致、`exception_info` 为空且 verifier 产生 reward。 本次实跑的官方 verifier 为 6/6、reward 为 1.0、Harbor 异常为 0；trajectory 共 11 个 steps，记录 49,878 input tokens、42,240 cache tokens、5,845 output tokens 和 0.00222441 USD，所有 10 次成本对账均首次成功。 From ff0e88247c11a1fdc68294ac2d3f5d9bdccfd0aa Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 3 Sep 2026 22:12:49 +0800 Subject: [PATCH 4/9] docs: document Harbor environment startup --- docs/dev_notes/zh-CN/0.8.x.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index ebdb3e1..722241c 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -258,8 +258,11 @@ uv run pytest **端到端测试步骤：** -1. 确认 `docker info` 可用，设置 `ANTHROPIC_API_KEY`、`ANTHROPIC_BASE_URL` 和 `ANTHROPIC_MODEL`，并准备一个远端可安装的完整 40 位 commit SHA；短 SHA 必须写成 `--agent-kwarg 'git_ref="83e6271"'`，否则可能被 Harbor 解析成数字。 -2. 从仓库根目录运行一次固定任务： +1. 在已安装 Docker 的宿主机运行 `sudo systemctl start docker.service`，启动 Docker daemon。 +2. 如果当前用户尚不能访问 Docker socket，运行 `sudo usermod -aG docker "$USER"` 将其加入 `docker` 组；这项系统配置只需执行一次。 +3. 重新登录，或在当前终端运行 `newgrp docker` 激活组权限；再运行 `docker info`，确认能够连接 daemon。 +4. 设置 `ANTHROPIC_API_KEY`、`ANTHROPIC_BASE_URL` 和 `ANTHROPIC_MODEL`，并准备一个远端可安装的完整 40 位 commit SHA。 +5. 从仓库根目录运行固定任务；`--env docker` 会让 Harbor 自动拉取或构建镜像、启动任务容器并在结束后清理环境，不需要手动运行 `docker compose`： ```bash uv run --project benchmarks/harbor harbor run \ @@ -272,8 +275,8 @@ uv run pytest --n-attempts 1 ``` -3. 从 Harbor 输出中取得本次结果目录 `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/`，确认其中存在 `agent/nanopycodeagent.txt`、`agent/trajectory.json`、`verifier/test-stdout.txt` 和 `result.json`。 -4. 运行 `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json`，确认 Harbor 0.21.0 validator 返回 0。 -5. 对照 `agent/trajectory.json` 与 `result.json`，确认 schema 为 ATIF-v1.7、steps 完整、`final_metrics` 与 `agent_result` 的 tokens/cache tokens/cost 一致、`exception_info` 为空且 verifier 产生 reward。 +6. 从 Harbor 输出中取得本次结果目录 `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/`，确认其中存在 `agent/nanopycodeagent.txt`、`agent/trajectory.json`、`verifier/test-stdout.txt` 和 `result.json`。 +7. 运行 `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json`，确认 Harbor 0.21.0 validator 返回 0。 +8. 对照 `agent/trajectory.json` 与 `result.json`，确认 schema 为 ATIF-v1.7、steps 完整、`final_metrics` 与 `agent_result` 的 tokens/cache tokens/cost 一致、`exception_info` 为空且 verifier 产生 reward。 本次实跑的官方 verifier 为 6/6、reward 为 1.0、Harbor 异常为 0；trajectory 共 11 个 steps，记录 49,878 input tokens、42,240 cache tokens、5,845 output tokens 和 0.00222441 USD，所有 10 次成本对账均首次成功。 From 5fc9d7eb877d031edd2503f622748a6628deaf7f Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 3 Sep 2026 22:26:19 +0800 Subject: [PATCH 5/9] docs: make Harbor model selection unambiguous --- benchmarks/harbor/README.md | 17 ++++++++++------- benchmarks/harbor/README.zh-CN.md | 13 +++++++------ docs/dev_notes/zh-CN/0.8.x.md | 2 +- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index cfd8183..f1151c4 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -9,13 +9,13 @@ workspace's lockfile. ## Run a benchmark -Set the same `ANTHROPIC_*` environment variables used by the CLI. For a -third-party or proxy endpoint, set all three explicitly: +Set the connection credentials used by the CLI. For a third-party or proxy +endpoint, configure the API key and base URL; select the model with Harbor's +`--model` option shown below: ```bash export ANTHROPIC_API_KEY="..." export ANTHROPIC_BASE_URL="https://gateway.example" -export ANTHROPIC_MODEL="provider-model-name" ``` From the repository root, run Harbor through this workspace and pin the agent @@ -51,10 +51,13 @@ ATIF output and backfills prompt, completion, cache-token, and cost totals into the agent result. Missing, invalid, or partial trajectories remain explicitly diagnosed; unknown usage or cost is not reported as zero. -`ANTHROPIC_MODEL` takes precedence over Harbor's `--model`. When it is absent, -the adapter strips the first provider prefix from `--model`. Harbor-native -provider credentials and configured base URLs are also normalized to the -`ANTHROPIC_*` variables expected by nanoPyCodeAgent's SDK. +By default, the adapter strips the first provider prefix from `--model` and +passes the result to nanoPyCodeAgent as `ANTHROPIC_MODEL`. Set +`ANTHROPIC_MODEL` only when a custom endpoint requires an actual model name that +differs from Harbor's `provider/model` identity; this explicit override takes +precedence. Harbor-native provider credentials and configured base URLs are +also normalized to the `ANTHROPIC_*` variables expected by nanoPyCodeAgent's +SDK. ## Test the adapter diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index fe298ab..1af3f2f 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -9,13 +9,12 @@ ## 运行 benchmark -设置与 CLI 相同的 `ANTHROPIC_*` 环境变量。使用第三方或代理 endpoint 时，显式 -设置以下三个变量： +设置 CLI 使用的连接凭据。使用第三方或代理 endpoint 时，配置 API key 和 base +URL；模型由下方 Harbor 命令中的 `--model` 选择： ```bash export ANTHROPIC_API_KEY="..." export ANTHROPIC_BASE_URL="https://gateway.example" -export ANTHROPIC_MODEL="provider-model-name" ``` 从仓库根目录通过这个 workspace 运行 Harbor，并固定安装到 task 容器中的 agent： @@ -47,9 +46,11 @@ adapter 还会要求 agent 将 ATIF-v1.7 trajectory 直接写入 并将 prompt、completion、cache token 和 cost 汇总回填到 agent result。trajectory 缺失、无效或指标不完整时会保留明确诊断；未知 usage 或 cost 不会被记成零。 -`ANTHROPIC_MODEL` 的优先级高于 Harbor 的 `--model`。未设置该环境变量时，adapter -会移除 `--model` 中的第一个 provider 前缀。Harbor 原生的 provider 凭证和已经配置 -的 base URL 也会转换为 nanoPyCodeAgent SDK 所需的 `ANTHROPIC_*` 变量。 +默认情况下，adapter 会移除 `--model` 中的第一个 provider 前缀，并将结果作为 +`ANTHROPIC_MODEL` 传给 nanoPyCodeAgent。只有自定义 endpoint 要求的实际模型名与 +Harbor 的 `provider/model` 身份不同时，才显式设置 `ANTHROPIC_MODEL`；该覆盖值优先。 +Harbor 原生的 provider 凭证和已经配置的 base URL 也会转换为 nanoPyCodeAgent SDK +所需的 `ANTHROPIC_*` 变量。 ## 测试 adapter diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 722241c..7006bcd 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -261,7 +261,7 @@ uv run pytest 1. 在已安装 Docker 的宿主机运行 `sudo systemctl start docker.service`，启动 Docker daemon。 2. 如果当前用户尚不能访问 Docker socket，运行 `sudo usermod -aG docker "$USER"` 将其加入 `docker` 组；这项系统配置只需执行一次。 3. 重新登录，或在当前终端运行 `newgrp docker` 激活组权限；再运行 `docker info`，确认能够连接 daemon。 -4. 设置 `ANTHROPIC_API_KEY`、`ANTHROPIC_BASE_URL` 和 `ANTHROPIC_MODEL`，并准备一个远端可安装的完整 40 位 commit SHA。 +4. 设置 `ANTHROPIC_API_KEY` 和 `ANTHROPIC_BASE_URL`，并准备一个远端可安装的完整 40 位 commit SHA；模型只通过下一步的 `--model` 指定，由 adapter 转换成 nanoPyCodeAgent 使用的 `ANTHROPIC_MODEL`。 5. 从仓库根目录运行固定任务；`--env docker` 会让 Harbor 自动拉取或构建镜像、启动任务容器并在结束后清理环境，不需要手动运行 `docker compose`： ```bash From fe8faaaebdd00b25758294565b5d2d49ad2b4219 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 3 Sep 2026 22:27:12 +0800 Subject: [PATCH 6/9] docs: explain Harbor validation artifacts --- docs/dev_notes/zh-CN/0.8.x.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 7006bcd..d31d448 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -275,8 +275,8 @@ uv run pytest --n-attempts 1 ``` -6. 从 Harbor 输出中取得本次结果目录 `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/`，确认其中存在 `agent/nanopycodeagent.txt`、`agent/trajectory.json`、`verifier/test-stdout.txt` 和 `result.json`。 -7. 运行 `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json`，确认 Harbor 0.21.0 validator 返回 0。 +6. 从 Harbor 输出中取得本次结果目录 `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/`，检查四份产物：`agent/nanopycodeagent.txt` 是 agent 的 stdout/stderr 文本日志；`agent/trajectory.json` 是 agent 生成的 ATIF steps、metrics 与终态；`verifier/test-stdout.txt` 是官方测试的执行明细；`result.json` 是 Harbor 汇总的 agent 指标、reward 与异常信息。 +7. 运行 `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json`，用仓库固定的 Harbor 0.21.0 检查 trajectory 的 ATIF schema 与跨字段约束；命令返回 0 表示格式和语义校验通过。 8. 对照 `agent/trajectory.json` 与 `result.json`，确认 schema 为 ATIF-v1.7、steps 完整、`final_metrics` 与 `agent_result` 的 tokens/cache tokens/cost 一致、`exception_info` 为空且 verifier 产生 reward。 本次实跑的官方 verifier 为 6/6、reward 为 1.0、Harbor 异常为 0；trajectory 共 11 个 steps，记录 49,878 input tokens、42,240 cache tokens、5,845 output tokens 和 0.00222441 USD，所有 10 次成本对账均首次成功。 From 302ada03a930b30ca3a98a09092bffca1f91747e Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 3 Sep 2026 22:28:49 +0800 Subject: [PATCH 7/9] docs: format Harbor artifact descriptions --- docs/dev_notes/zh-CN/0.8.x.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index d31d448..f02a99d 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -275,7 +275,13 @@ uv run pytest --n-attempts 1 ``` -6. 从 Harbor 输出中取得本次结果目录 `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/`，检查四份产物：`agent/nanopycodeagent.txt` 是 agent 的 stdout/stderr 文本日志；`agent/trajectory.json` 是 agent 生成的 ATIF steps、metrics 与终态；`verifier/test-stdout.txt` 是官方测试的执行明细；`result.json` 是 Harbor 汇总的 agent 指标、reward 与异常信息。 +6. 从 Harbor 输出中取得本次结果目录 `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/`，检查四份产物： + + - `agent/nanopycodeagent.txt`：agent 的 stdout/stderr 文本日志。 + - `agent/trajectory.json`：agent 生成的 ATIF steps、metrics 与终态。 + - `verifier/test-stdout.txt`：官方测试的执行明细。 + - `result.json`：Harbor 汇总的 agent 指标、reward 与异常信息。 + 7. 运行 `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json`，用仓库固定的 Harbor 0.21.0 检查 trajectory 的 ATIF schema 与跨字段约束；命令返回 0 表示格式和语义校验通过。 8. 对照 `agent/trajectory.json` 与 `result.json`，确认 schema 为 ATIF-v1.7、steps 完整、`final_metrics` 与 `agent_result` 的 tokens/cache tokens/cost 一致、`exception_info` 为空且 verifier 产生 reward。 From 24e7281a2889f01d9b60a425d06e5f4cb9f58fc9 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 3 Sep 2026 22:30:41 +0800 Subject: [PATCH 8/9] docs: format Harbor end-to-end results --- docs/dev_notes/zh-CN/0.8.x.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index f02a99d..2dbbff5 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -285,4 +285,11 @@ uv run pytest 7. 运行 `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json`，用仓库固定的 Harbor 0.21.0 检查 trajectory 的 ATIF schema 与跨字段约束；命令返回 0 表示格式和语义校验通过。 8. 对照 `agent/trajectory.json` 与 `result.json`，确认 schema 为 ATIF-v1.7、steps 完整、`final_metrics` 与 `agent_result` 的 tokens/cache tokens/cost 一致、`exception_info` 为空且 verifier 产生 reward。 -本次实跑的官方 verifier 为 6/6、reward 为 1.0、Harbor 异常为 0；trajectory 共 11 个 steps，记录 49,878 input tokens、42,240 cache tokens、5,845 output tokens 和 0.00222441 USD，所有 10 次成本对账均首次成功。 +**本次实跑结果：** + +- 官方 verifier：6/6。 +- Reward：1.0。 +- Harbor 异常：0。 +- Trajectory：11 个 steps，其中 10 个模型 steps。 +- Tokens：49,878 input、42,240 cache、5,845 output。 +- Cost：0.00222441 USD，10 次成本对账均首次成功。 From e2a42cf745541a26d581725e07fffc90d1ca3f5b Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 3 Sep 2026 22:39:27 +0800 Subject: [PATCH 9/9] docs: sync harbor integration notes --- docs/changelogs/0.8.x.md | 5 +++ docs/dev_notes/en/0.8.x.md | 86 ++++++++++++++++++++++++++++++++++---- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/docs/changelogs/0.8.x.md b/docs/changelogs/0.8.x.md index b6b223a..4470ab1 100644 --- a/docs/changelogs/0.8.x.md +++ b/docs/changelogs/0.8.x.md @@ -42,6 +42,11 @@ All notable changes in the **0.8.x** release series are documented here. generation ID with bounded retries, support both API-key and bearer-token authentication, and retain per-attempt diagnostics when reconciliation remains incomplete. +- Harbor ATIF-v1.7 collection and metric reporting. The adapter requests the + agent trajectory at Harbor's standard log path, validates and exposes it to + Harbor, populates step, token, cache-token, and cost totals without inventing + zeroes for unknown values, and records explicit diagnostics for partial, + missing, or invalid trajectories. ### Fixed - Declare `httpx` as a direct runtime dependency so clean and containerized diff --git a/docs/dev_notes/en/0.8.x.md b/docs/dev_notes/en/0.8.x.md index 2f951a7..1112de1 100644 --- a/docs/dev_notes/en/0.8.x.md +++ b/docs/dev_notes/en/0.8.x.md @@ -166,8 +166,8 @@ These describe the same occurrence rather than two events. A `Native Event` is t Rather than splitting the implementation into seven technically layered and disconnected steps, the work is organized as independently useful and independently testable capabilities. Each capability defines its implementation and acceptance criteria together instead of postponing all validation until the end: 1. **Runtime facts and the Event Journal.** Establish versioned contracts for `Native Event` and `Journal Entry`. The agent loop emits facts only at user, model, tool, and run boundaries, and appends them to an internal Event Journal. Text output is projected from those same facts so existing user-visible behavior remains unchanged. -2. **Public ATIF trajectory.** Implement a one-way Event Journal to ATIF-v1.7 projector and expose it for headless runs through an independent `--trajectory PATH`. `PATH` identifies one complete ATIF JSON snapshot rather than the internal Journal and does not change stdout. The projector maps messages, tools, timestamps, durations, token and cache usage, and terminal state already defined by the Journal. Cost mapping depends on a separate actual-cost contract and backfill mechanism. -3. **Actual-cost backfill.** Record `usage.cost` directly when available. For the current OpenRouter Messages path, use the stored `X-Generation-Id` and bounded retries at run finalization to query the generation's `total_cost`, then append `model.cost_resolved`. Query failures do not change task results; the projector fills resolved costs into the corresponding step and `final_metrics`, and never writes an unknown cost as `0`. +2. **Public ATIF trajectory.** Implement a one-way Event Journal to ATIF-v1.7 projector and expose it for headless runs through an independent `--trajectory PATH`. `PATH` identifies one complete ATIF JSON snapshot rather than the internal Journal and does not change stdout. The projector maps messages, tools, timestamps, durations, token and cache usage, and terminal state already defined by the Journal. Cost mapping depends on a separate actual-cost contract and reconciliation mechanism. +3. **Provider-reported cost collection and reconciliation.** Prefer the provider's actual charge from the model response's `usage.cost`. When a response has no charge but does include a generation ID, mark the cost as pending and query the provider's Generation API for `total_cost` while the run is finalizing. Save query results as appended events without modifying existing Journal Entries, and do not change the task result if a query fails. The projector maps known charges to their steps and the run total, explicitly describes whether cost is complete, and never records an unknown charge as zero. 4. **Harbor integration and end-to-end acceptance.** The Harbor adapter only passes a trajectory path, declares and reads the agent-produced ATIF, then populates steps, tokens, and cost. It no longer converts a native trajectory. Harbor contract tests verify collection and statistics first, followed by one real trial of the complete path. `--output-format` and `stream-json` are separate run-output capabilities and are outside the trajectory work above. @@ -191,7 +191,7 @@ Development acceptance requires these tests to return zero and every mandatory p #### Public ATIF trajectory -**Goal:** implement a one-way Event Journal to ATIF-v1.7 projector that exports one headless Agent Run as a complete ATIF JSON document. The CLI gains an independent `--trajectory PATH`; `PATH` selects the output file while stdout keeps its existing text behavior. The internal Event Journal remains private. Actual-cost backfill, Harbor adapter collection, `stream-json`, and multi-run trajectories for interactive sessions are outside this capability. +**Goal:** implement a one-way Event Journal to ATIF-v1.7 projector that exports one headless Agent Run as a complete ATIF JSON document. The CLI gains an independent `--trajectory PATH`; `PATH` selects the output file while stdout keeps its existing text behavior. The internal Event Journal remains private. Actual-cost collection and reconciliation, Harbor adapter collection, `stream-json`, and multi-run trajectories for interactive sessions are outside this capability. **Protocol:** the target format is ATIF-v1.7 as implemented by Harbor 0.21.0: @@ -213,15 +213,83 @@ Development acceptance requires the validator to return zero, stdout to retain i #### Provider-reported cost collection and reconciliation -**Goal:** trajectories record the actual model-call charges reported by the provider. The provider response is the source of truth; the Event Journal preserves the direct or deferred resolution state; and ATIF exposes per-step `metrics.cost_usd` plus a complete run total only when every billable call has been resolved. +**Goal:** trajectories record the actual model-call charges reported by the provider. Three similarly named layers serve different purposes: + +- **Provider responses** are the source of truth for cost. `usage.cost` is the `cost` field in the response's `usage` object; `X-Generation-Id` is the generation ID in an HTTP response header; and `data.total_cost` is the total-charge field inside the Generation API response's `data` object. +- **The Event Journal** preserves runtime facts collected by the agent. `model.completed` is the event emitted when a model call finishes, and its `payload.cost` stores the call's cost state. `model.cost_resolved` is a separate event appended after a deferred lookup succeeds; its `payload` stores the generation ID, amount, currency, and source. +- **The ATIF trajectory** is the public result projected from the Event Journal. A model step's `metrics.cost_usd` represents the cost of one call, while the trajectory-level `final_metrics` stores the run-wide cost summary and completeness state. There are two collection paths: -- **Synchronous response accounting.** OpenRouter's [Usage Accounting](https://openrouter.ai/docs/cookbook/administration/usage-accounting) defines a complete `usage` object for OpenAI-compatible Chat Completions and streaming responses. Its `usage.cost` is the total amount charged for the request, delivered in the complete non-streaming response or the final SSE message. When present, nano records it immediately as a resolved provider-reported USD cost. Token usage does not imply that cost is present: OpenRouter's [Anthropic Messages API](https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-messages) preserves an Anthropic-compatible usage schema containing token, cache, service-tier, and speed fields but does not define `cost`. nano currently calls this endpoint through the Anthropic SDK, so a response may contain token counts and an extension such as `speed: "standard"` without a cost. -- **Asynchronous generation reconciliation.** When `usage.cost` is absent but `X-Generation-Id` is available, nano records the generation ID and marks the cost pending. This covers the current OpenRouter Anthropic Messages path and compatible APIs whose usage record becomes visible after the model response. OpenRouter's documented alternative is to retain the generation ID, then call [`GET /api/v1/generation?id=...`](https://openrouter.ai/docs/api/api-reference/generations/get-generation) and read `data.total_cost`. The endpoint documents 404, 429, and 5xx responses, so possession of an ID does not guarantee that its billing record is immediately queryable. +- **Synchronous response accounting.** OpenRouter's [Usage Accounting](https://openrouter.ai/docs/cookbook/administration/usage-accounting) defines a complete `usage` object for OpenAI-compatible Chat Completions and streaming responses. Its `usage.cost` is the total amount charged for the request, delivered in the complete non-streaming response or the final SSE message. When present, nanoPyCodeAgent immediately records it in `model.completed.payload.cost` as a resolved, provider-reported USD amount. Token usage does not imply that cost is present: OpenRouter's [Anthropic Messages API](https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-messages) preserves an Anthropic-compatible usage schema containing token, cache, service-tier, and speed fields but does not define `cost`. nanoPyCodeAgent currently calls this endpoint through the Anthropic SDK, so a response may contain token counts and an extension such as `speed: "standard"` without a cost. +- **Asynchronous generation reconciliation.** When `usage.cost` is absent but the HTTP headers include `X-Generation-Id`, nanoPyCodeAgent records the generation ID in `model.completed.payload.generation_id` and marks `payload.cost.status` as `pending`. This covers the current OpenRouter Anthropic Messages path and compatible APIs whose usage record becomes visible after the model response. OpenRouter's documented alternative is to retain the generation ID, then call [`GET /api/v1/generation?id=...`](https://openrouter.ai/docs/api/api-reference/generations/get-generation) and read `data.total_cost`. The endpoint documents 404, 429, and 5xx responses, so possession of an ID does not guarantee that its billing record is immediately queryable. + +The reconciliation endpoint is derived from the current Anthropic SDK base URL on the same origin as `v1/generation`. It is not tied to a provider hostname and never sends API credentials to a hard-coded destination; OpenRouter is currently used to validate this extension protocol. Reconciliation makes six bounded attempts with exponential delays of 1, 2, 4, 8, and 15 seconds, for about 30 seconds of total waiting. A 404, 408, 409, 429, common 5xx response, network error, or HTTP 200 response without `total_cost` is retryable; permanent failures such as 400, 401, 402, and 403 stop immediately. A successful lookup appends `model.cost_resolved`, storing the amount in `payload.amount`, the association key in `payload.generation_id`, and the currency and source in `payload.currency` and `payload.source`. Existing `model.completed` events are never rewritten, so the Event Journal remains append-only. + +Reconciliation must be observable instead of collapsing every failure into `None`. The terminal event for each run carries an optional `cost_reconciliation` list. For every generation it records a final `resolved` or `unresolved` state and classifies every attempt as `resolved`, HTTP 200 with `cost_unavailable`, `http_error` with a status code, `request_error` with an exception type, or `unsupported_endpoint` when the base URL cannot produce a lookup endpoint. Credentials and provider response bodies are never persisted. The ATIF projector copies this list to `extra.terminal.cost_reconciliation`, allowing incomplete costs to be diagnosed from either the internal Journal or the public trajectory. + +Cost collection is best-effort enrichment: HTTP errors, billing records that are not yet available, and invalid responses never change whether the agent's original task succeeded or failed. The ATIF projector associates deferred results with model steps through the matching `payload.generation_id` and emits: + +- The cost of each model call in that ATIF step's `metrics.cost_usd`, with the cost source and generation ID in `metrics.extra.cost_source` and `metrics.extra.generation_id`. +- The run total in the trajectory-level `final_metrics.total_cost_usd` when every billable call has a complete cost. +- When cost is incomplete, the known subtotal and incompleteness marker in `final_metrics.extra.known_cost_usd` and `final_metrics.extra.cost_is_partial`; if any recognizable calls are still pending reconciliation, their generation IDs appear in `final_metrics.extra.missing_generation_ids`. + +Unknown costs always remain unknown and are never written as zero. + +**Automated acceptance:** cost unit tests cover synchronous response accounting, pending and unknown states, successful Generation API retries, permanent HTTP-error short circuiting, bounded failure, and per-attempt diagnostics. Agent event tests verify that reconciliation occurs before the run's terminal event and that failure diagnostics are persisted. ATIF tests verify step costs, complete totals, partial totals, and terminal diagnostics. The complete test suite is: + +```bash +uv run pytest +``` + +#### Harbor integration and end-to-end acceptance + +**Goal:** + +1. Have the repository's Harbor adapter specify a trajectory output path inside the container for every trial. After the agent exits, tell Harbor the file's path and format so Harbor can read the ATIF-v1.7 document generated by the agent. +2. Populate Harbor with the complete ATIF steps and with the token, cache-token, cost, and completeness state from step metrics and final metrics. +3. Define the adapter's responsibility and failure semantics clearly: it only coordinates the path and data handoff; it does not parse the Event Journal or maintain a native-trajectory conversion. A missing, invalid, or partial trajectory must produce an explicit diagnostic instead of being misreported as zero usage or a complete result. + +**Validation:** + +1. Extend the adapter contract tests under `benchmarks/harbor/tests` to verify that the trajectory path is passed into the container correctly and that the ATIF can be declared and read. +2. Use contract tests to verify population of steps, tokens, and cost, including behavior for a missing trajectory file, validation failure, and incomplete metrics. +3. Run one real Terminal-Bench trial with the Harbor version pinned by the repository. Compare the agent log, Harbor's saved trajectory, steps, tokens, cost, and reward to confirm the complete path from container installation and task execution through trajectory persistence, Harbor collection, and official verifier scoring, while distinguishing a failed task solution from an adapter infrastructure failure. + +**End-to-end test procedure:** + +1. On a host with Docker installed, run `sudo systemctl start docker.service` to start the Docker daemon. +2. If the current user cannot yet access the Docker socket, run `sudo usermod -aG docker "$USER"` to add that user to the `docker` group. This system configuration is required only once. +3. Log in again, or run `newgrp docker` in the current terminal to activate the group membership. Then run `docker info` and confirm that it can connect to the daemon. +4. Set `ANTHROPIC_API_KEY` and `ANTHROPIC_BASE_URL`, and prepare a complete 40-character commit SHA that can be installed from the remote repository. Select the model only through `--model` in the next step; the adapter converts it into the `ANTHROPIC_MODEL` used by nanoPyCodeAgent. +5. From the repository root, run the pinned task. With `--env docker`, Harbor pulls or builds the image, starts the task container, and cleans up the environment when it finishes; there is no need to run `docker compose` manually: + + ```bash + uv run --project benchmarks/harbor harbor run \ + --task terminal-bench/openssl-selfsigned-cert \ + --agent harbor_adapter:NanoPyCodeAgent \ + --agent-kwarg git_ref=83e6271e9e9d0a22fb7c2510a8df76acff320cd3 \ + --model openrouter/deepseek/deepseek-v4-flash-0731 \ + --env docker \ + --n-concurrent 1 \ + --n-attempts 1 + ``` + +6. Read the result directory `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/` from Harbor's output and inspect these four artifacts: + + - `agent/nanopycodeagent.txt`: the agent's stdout/stderr text log. + - `agent/trajectory.json`: the agent-generated ATIF steps, metrics, and terminal state. + - `verifier/test-stdout.txt`: the official tests' execution details. + - `result.json`: Harbor's aggregate agent metrics, reward, and exception information. -The lookup endpoint is derived from the configured Anthropic SDK base URL and stays on the same origin. Reconciliation uses six bounded attempts with exponential delays of 1, 2, 4, 8, and 15 seconds, for about 30 seconds of total waiting. A missing record, transient HTTP status, network error, or HTTP 200 response without `total_cost` remains retryable; permanent client and authentication failures stop immediately. A successful lookup appends `model.cost_resolved` without rewriting the earlier `model.completed` entry. +7. Run `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json`. This uses the repository-pinned Harbor 0.21.0 to check the trajectory's ATIF schema and cross-field constraints; an exit code of 0 means the format and semantics are valid. +8. Compare `agent/trajectory.json` with `result.json`. Confirm that the schema is ATIF-v1.7, steps are complete, `final_metrics` agrees with `agent_result` for tokens, cache tokens, and cost, `exception_info` is empty, and the verifier produced a reward. -Reconciliation is observable rather than collapsing every failure into `None`. The terminal Journal event carries an optional `cost_reconciliation` list with the final status for each generation and each attempt classified as resolved, cost unavailable, HTTP error with its status code, request error with its exception type, or unsupported endpoint. Credentials and response bodies are never persisted. The ATIF projector copies the diagnostics into `extra.terminal.cost_reconciliation`. Cost enrichment remains best effort and never changes the task outcome; an unknown amount is never reported as zero. +**Result of this run:** -**Validation:** unit tests cover direct response extraction, pending and unknown states, successful deferred retry, permanent-error short circuiting, bounded failure, and attempt diagnostics. Agent event tests verify successful reconciliation before the terminal event and persistence of unresolved diagnostics. ATIF tests cover per-step costs, complete and partial totals, and terminal diagnostics. The complete acceptance command is `uv run pytest`. +- Official verifier: 6/6. +- Reward: 1.0. +- Harbor exceptions: 0. +- Trajectory: 11 steps, including 10 model steps. +- Tokens: 49,878 input, 42,240 cache, and 5,845 output. +- Cost: USD 0.00222441; all 10 reconciliation lookups succeeded on their first attempt.