diff --git a/evals/harbor/README.md b/evals/harbor/README.md new file mode 100644 index 00000000..bb6633a0 --- /dev/null +++ b/evals/harbor/README.md @@ -0,0 +1,129 @@ +# Harbor adapter for Corbits Code + +Thin Harbor `BaseInstalledAgent` that runs headless product Corbits: + +```text +corbits exec --cwd … --config … --provider … --model … + --dangerously-skip-permissions +``` + +No second agent loop. Credentials are translated from Harbor kwargs/env into a +temporary `settings.json` passed with `--config` — product Corbits still does +not read API keys from the environment. + +## Layout + +| Path | Role | +| -------------------- | ------------------------------------------------------ | +| `argv.py` | Pure settings + argv builders (unit-tested, no Harbor) | +| `agent.py` | `Corbits` installed agent (requires Harbor at import) | +| `tasks/trivial/` | Minimal smoke task (`hello.txt`) | +| `tests/test_argv.py` | Argv/settings unit tests | + +## Prerequisites + +1. **Harbor CLI** installed in the host Python env (`pip install harbor` / uv). +2. **Linux ELF `corbits` binary** for the task container (Darwin host builds + will not run inside Linux Docker). Acquire one of: + - Build on Linux: `bun run build:bin` → `dist/corbits` + - Release / CI tarball that unpacks a `corbits` binary +3. **git** inside the task image (adapter also installs it via Harbor system + packages). Corbits storage requires git — there is no git-less fallback. +4. **Provider API key** for the model under test. +5. **Provider `base_url`** (required — see below). The adapter fail-closes if + none is resolved; it does not invent a default. + +## Secrets / credentials + +Pass a key through Harbor agent kwargs or env. The adapter writes it into the +temp settings file only: + +| Source | Notes | +| ----------------------- | ---------------------------------------------- | +| `api_key=` agent kwarg | Preferred for one-off runs | +| `CORBITS_API_KEY` | Generic adapter env | +| `{PROVIDER}_API_KEY` | Only the selected provider, e.g. `XAI_API_KEY` | +| Harbor model connection | Falls back to `model_connection.api_key` | + +### Required base URL + +`providers..baseURL` is always written. Resolve it via one of: + +| Source | Notes | +| ----------------------- | ----------------------------------------------- | +| `base_url=` agent kwarg | Preferred for one-off runs | +| `CORBITS_BASE_URL` | Adapter env | +| Harbor model connection | `model_connection.configured_base_url` when set | + +If none are set, the adapter raises before writing settings. + +Example values: + +| Cell | Example `base_url` | +| ----------------------------- | ------------------------------------------------------------------------------------ | +| xAI API key | `https://api.x.ai/v1` | +| Product OAuth / grok-cli path | `https://cli-chat-proxy.grok.com/v1` (`XAI_BASE_URL` in `src/auth/xai/constants.ts`) | +| OpenAI-compatible | e.g. `https://api.openai.com/v1` or your cell's gateway | + +Optional: `shell_timeout_ms=` → `shell.timeoutMs` in settings. + +Default provider/model when Harbor does not pass `provider/model`: **xai** / +**grok-4.5**. Codex cells typically use `--model openai/` (or pass +`provider=` / `model=` kwargs). + +## Linux binary acquisition + +The adapter installs the binary onto PATH from **one** of: + +| Kwarg | Behavior | +| --------------------- | ------------------------------------------------- | +| `corbits_binary_path` | Upload a host file into `/usr/local/bin/corbits` | +| `corbits_binary_url` | `curl` a raw binary URL | +| `corbits_tarball_url` | `curl` + extract; expects a `corbits` file inside | + +Archive vs raw binary is detected from the URL suffix (`.tar.gz`, `.tgz`, +`.tar`) — the install script does not call `file(1)`. + +If none are set and `corbits` is not already on PATH in the environment, +`install()` raises with this README pointer. + +## Invoke + +From the repo root (so `evals.harbor.agent` is importable): + +```bash +# Unit tests (no Harbor package required; pytest may be absent) +PYTHONPATH=. python3 -m unittest evals.harbor.tests.test_argv -v + +# Dry-run trivial task (needs Harbor CLI + Linux binary + API key + base URL) +export CORBITS_API_KEY=… # or XAI_API_KEY=… +export CORBITS_BASE_URL=https://api.x.ai/v1 +harbor run \ + -p evals/harbor/tasks/trivial \ + -a evals.harbor.agent:Corbits \ + -m xai/grok-4.5 \ + --ae corbits_binary_path=/absolute/path/to/linux/corbits +``` + +Equivalent kwargs via Harbor job config: + +```yaml +agents: + - name: evals.harbor.agent:Corbits + kwargs: + provider: xai + model: grok-4.5 + api_key: ${CORBITS_API_KEY} + base_url: https://api.x.ai/v1 + corbits_binary_path: /absolute/path/to/linux/corbits + # or: corbits_tarball_url: https://…/corbits-linux.tar.gz +``` + +## Known gaps (CL-6924) + +- Full Harbor dry-run + Terminal-Bench smoke are **not** claimed by this change. + They need Harbor CLI, a Linux ELF binary, Docker, and provider credentials on + a machine that can run the harness end-to-end — tracked as **CL-6924**. +- This adapter does not parse Corbits trajectories into Harbor ATIF; exit + metadata is limited to `context.metadata["exit_code"]` plus tee'd stdout in + `/logs/agent/corbits.txt`. diff --git a/evals/harbor/__init__.py b/evals/harbor/__init__.py new file mode 100644 index 00000000..94e375ec --- /dev/null +++ b/evals/harbor/__init__.py @@ -0,0 +1,7 @@ +"""Harbor installed-agent adapter for Corbits Code (CL-6923). + +Import the agent class as ``evals.harbor.agent:Corbits`` when running Harbor. +Pure helpers live in ``argv`` and are unit-tested without the Harbor package. +""" + +__all__ = ["argv"] diff --git a/evals/harbor/agent.py b/evals/harbor/agent.py new file mode 100644 index 00000000..7d209be9 --- /dev/null +++ b/evals/harbor/agent.py @@ -0,0 +1,271 @@ +"""Harbor ``BaseInstalledAgent`` bridge that runs headless ``corbits exec``. + +Thin adapter: install a Linux Corbits binary + git, write a temporary +settings.json from Harbor kwargs/env, then exec with +``--dangerously-skip-permissions``. No second agent loop. + +Requires the Harbor package at import time (normal for Harbor plugins). +Unit tests import ``evals.harbor.argv`` only. +""" + +from __future__ import annotations + +import json +import shlex +from pathlib import Path, PurePosixPath +from typing import Any, override + +from harbor.agents.installed.base import ( + BaseInstalledAgent, + with_prompt_template, +) +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from harbor.models.trial.paths import EnvironmentPaths + +from evals.harbor.argv import api_key_env_names, build_exec_argv, build_settings + +_REMOTE_BIN_DIR = PurePosixPath("/usr/local/bin") +_REMOTE_CORBITS = _REMOTE_BIN_DIR / "corbits" +_REMOTE_SETTINGS = PurePosixPath("/tmp/corbits-settings.json") +_OUTPUT_FILENAME = "corbits.txt" +_DEFAULT_PROVIDER = "xai" +_DEFAULT_MODEL = "grok-4.5" +_DEFAULT_TASK_CWD = "/app" + + +class Corbits(BaseInstalledAgent): + """Installed-agent adapter: Harbor → ``corbits exec`` (yolo for that process).""" + + def __init__( + self, + *args: Any, + provider: str | None = None, + model: str | None = None, + api_key: str | None = None, + base_url: str | None = None, + shell_timeout_ms: int | None = None, + corbits_binary_url: str | None = None, + corbits_tarball_url: str | None = None, + corbits_binary_path: str | None = None, + task_cwd: str = _DEFAULT_TASK_CWD, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self._provider_override = provider + self._model_override = model + self._api_key_override = api_key + self._base_url = base_url + self._shell_timeout_ms = shell_timeout_ms + self._corbits_binary_url = corbits_binary_url + self._corbits_tarball_url = corbits_tarball_url + self._corbits_binary_path = corbits_binary_path + self._task_cwd = task_cwd + self._last_exit_code: int | None = None + + @staticmethod + @override + def name() -> str: + # Custom import path agent — do not require AgentName enum membership. + return "corbits" + + @override + def get_version_command(self) -> str | None: + # The CLI has no --version flag; record the help banner as a best-effort + # identity string. Harbor ignores failures here. + return "corbits --help | head -n 1" + + def _resolve_provider_model(self) -> tuple[str, str]: + if self._provider_override and self._model_override: + return self._provider_override, self._model_override + + if self.model_name and "/" in self.model_name: + provider, model = self.model_name.split("/", 1) + return ( + self._provider_override or provider, + self._model_override or model, + ) + + if self.model_name: + return ( + self._provider_override or _DEFAULT_PROVIDER, + self._model_override or self.model_name, + ) + + return ( + self._provider_override or _DEFAULT_PROVIDER, + self._model_override or _DEFAULT_MODEL, + ) + + def _resolve_api_key(self, provider: str) -> str: + if self._api_key_override: + return self._api_key_override + + # Adapter-only translation: Harbor env/kwargs → settings.json. + # Product Corbits still sees credentials only via --config. + for name in api_key_env_names(provider): + value = self._get_env(name) + if value: + return value + + access = self.model_connection + if getattr(access, "api_key", None): + return str(access.api_key) + + raise ValueError( + "No API key for Corbits Harbor adapter. Pass api_key=… in agent " + "kwargs, set CORBITS_API_KEY or {PROVIDER}_API_KEY, " + "or configure Harbor model credentials. Keys are written into a " + "temporary settings.json for --config only." + ) + + def _resolve_base_url(self) -> str: + if self._base_url: + return self._base_url + access = self.model_connection + configured = getattr(access, "configured_base_url", None) + if configured: + return str(configured) + env_url = self._get_env("CORBITS_BASE_URL") + if env_url: + return env_url + raise ValueError( + "No base URL for Corbits Harbor adapter. Pass base_url=… in agent " + "kwargs, set CORBITS_BASE_URL, or configure Harbor model connection " + "base URL. Corbits settings require providers..baseURL." + ) + + @override + async def install(self, environment: BaseEnvironment) -> None: + # Corbits storage requires git in the environment (no git-less fallback). + await self.ensure_system_dependencies( + environment, ("git", "curl", "ca_certificates", "tar") + ) + + check = await environment.exec(command="command -v corbits >/dev/null 2>&1") + if check.return_code == 0: + self.logger.debug("corbits already on PATH") + return + + if self._corbits_binary_path: + await self._install_from_host_path(environment, self._corbits_binary_path) + return + + url = self._corbits_binary_url or self._corbits_tarball_url + if url: + await self._install_from_url(environment, url) + return + + raise RuntimeError( + "Corbits Harbor adapter needs a Linux ELF binary. Pass one of " + "corbits_binary_path (host file uploaded into the env), " + "corbits_binary_url, or corbits_tarball_url in agent kwargs. " + "Build with `bun run build:bin` on Linux, or use a release tarball. " + "See evals/harbor/README.md." + ) + + async def _install_from_host_path( + self, environment: BaseEnvironment, host_path: str + ) -> None: + source = Path(host_path).expanduser() + if not source.is_file(): + raise FileNotFoundError(f"corbits_binary_path not found: {source}") + remote = _REMOTE_CORBITS.as_posix() + await environment.upload_file(source, remote) + await self.exec_as_root( + environment, + command=f"chmod 755 {shlex.quote(remote)} && corbits --help >/dev/null", + ) + + async def _install_from_url(self, environment: BaseEnvironment, url: str) -> None: + quoted_url = shlex.quote(url) + remote = _REMOTE_CORBITS.as_posix() + # Detect archive by URL suffix — do not use file(1) (absent in trivial images). + path_part = url.lower().split("?", 1)[0] + is_archive = path_part.endswith((".tar.gz", ".tgz", ".tar")) + if is_archive: + command = ( + "set -euo pipefail; " + f"tmp=$(mktemp -d); " + f"curl -fsSL {quoted_url} -o \"$tmp/artifact\"; " + "tar -xaf \"$tmp/artifact\" -C \"$tmp\"; " + "bin=$(find \"$tmp\" -type f -name corbits | head -n 1); " + "if [ -z \"$bin\" ]; then " + " echo 'tarball did not contain a corbits binary' >&2; exit 1; " + "fi; " + f"install -m 755 \"$bin\" {shlex.quote(remote)}; " + "corbits --help >/dev/null" + ) + else: + command = ( + "set -euo pipefail; " + f"curl -fsSL {quoted_url} -o {shlex.quote(remote)}; " + f"chmod 755 {shlex.quote(remote)}; " + "corbits --help >/dev/null" + ) + await self.exec_as_root(environment, command=command) + + @override + def populate_context_post_run(self, context: AgentContext) -> None: + meta = dict(context.metadata or {}) + if self._last_exit_code is not None: + meta["exit_code"] = self._last_exit_code + meta["agent"] = self.name() + context.metadata = meta + + @override + @with_prompt_template + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + provider, model = self._resolve_provider_model() + api_key = self._resolve_api_key(provider) + settings = build_settings( + provider, + model, + api_key, + base_url=self._resolve_base_url(), + shell_timeout_ms=self._shell_timeout_ms, + ) + + remote_config = _REMOTE_SETTINGS.as_posix() + await self._upload_config_text( + environment, + content=json.dumps(settings, indent=2) + "\n", + remote_path=remote_config, + filename="settings.json", + ) + + # Persist a redacted copy under agent logs for debugging. + redacted = json.loads(json.dumps(settings)) + redacted["providers"][provider]["apiKey"] = "***" + (self.logs_dir / "settings.redacted.json").write_text( + json.dumps(redacted, indent=2) + "\n" + ) + + argv = build_exec_argv( + cwd=self._task_cwd, + config_path=remote_config, + provider=provider, + model=model, + prompt=instruction, + binary="corbits", + ) + command = ( + "set -euo pipefail; " + + " ".join(shlex.quote(part) for part in argv) + + f" 2>&1 | tee {shlex.quote((EnvironmentPaths.agent_dir / _OUTPUT_FILENAME).as_posix())}; " + "exit ${PIPESTATUS[0]}" + ) + + (self.logs_dir / "command.txt").write_text(command + "\n") + + # Bypass Harbor's _exec so the real exit code survives a failure; it + # raises NonZeroAgentExitCodeError without exposing the code. + result = await environment.exec(command=command) + self._last_exit_code = result.return_code + if result.return_code != 0: + raise self._classify_exec_error(command, result) diff --git a/evals/harbor/argv.py b/evals/harbor/argv.py new file mode 100644 index 00000000..80cd55a0 --- /dev/null +++ b/evals/harbor/argv.py @@ -0,0 +1,81 @@ +"""Pure Corbits settings + exec argv builders for the Harbor adapter. + +No Harbor imports — unit-testable in isolation. +""" + +from __future__ import annotations + +from typing import Any + + +def build_settings( + provider: str, + model: str, + api_key: str, + *, + base_url: str, + shell_timeout_ms: int | None = None, +) -> dict[str, Any]: + """Build a Corbits ``settings.json`` dict for ``--config``. + + Shape matches product settings: ``providers..{apiKey,models,baseURL}`` + plus optional ``shell.timeoutMs``. Credentials go into the file only — Corbits + does not read API keys from the environment. + + ``base_url`` is required and always written as ``baseURL``. Callers must + resolve it (agent kwargs / ``CORBITS_BASE_URL`` / Harbor model connection); + this helper does not invent a default URL. + """ + provider_entry: dict[str, Any] = { + "apiKey": api_key, + "models": [model], + "baseURL": base_url, + } + + settings: dict[str, Any] = { + "providers": { + provider: provider_entry, + }, + } + if shell_timeout_ms is not None: + settings["shell"] = {"timeoutMs": shell_timeout_ms} + return settings + + +def api_key_env_names(provider: str) -> tuple[str, str]: + """Env vars consulted for the API key, in priority order. + + Deliberately no cross-provider fallback: a key for another provider is + never silently written into ``providers..apiKey``. + """ + return ("CORBITS_API_KEY", f"{provider.upper()}_API_KEY") + + +def build_exec_argv( + *, + cwd: str, + config_path: str, + provider: str, + model: str, + prompt: str, + binary: str = "corbits", +) -> list[str]: + """Build the exact ``corbits exec`` argv for a Harbor trial. + + Always includes ``--dangerously-skip-permissions`` so the headless process + cannot block on operator approval. + """ + return [ + binary, + "exec", + "--cwd", + cwd, + "--config", + config_path, + "--provider", + provider, + "--model", + model, + "--dangerously-skip-permissions", + prompt, + ] diff --git a/evals/harbor/tasks/trivial/environment/Dockerfile b/evals/harbor/tasks/trivial/environment/Dockerfile new file mode 100644 index 00000000..69c2a992 --- /dev/null +++ b/evals/harbor/tasks/trivial/environment/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:24.04 + +WORKDIR /app + +# git is required by Corbits storage; curl helps the adapter fetch a binary. +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* diff --git a/evals/harbor/tasks/trivial/instruction.md b/evals/harbor/tasks/trivial/instruction.md new file mode 100644 index 00000000..9e2447de --- /dev/null +++ b/evals/harbor/tasks/trivial/instruction.md @@ -0,0 +1,7 @@ +Create a file named `hello.txt` in the working directory (`/app`) whose entire contents are exactly: + +```text +hello +``` + +Do not add a trailing blank line beyond a single trailing newline. No other files are required. diff --git a/evals/harbor/tasks/trivial/solution/solve.sh b/evals/harbor/tasks/trivial/solution/solve.sh new file mode 100755 index 00000000..5f5ff359 --- /dev/null +++ b/evals/harbor/tasks/trivial/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +printf 'hello\n' > /app/hello.txt diff --git a/evals/harbor/tasks/trivial/task.toml b/evals/harbor/tasks/trivial/task.toml new file mode 100644 index 00000000..2f5f59c6 --- /dev/null +++ b/evals/harbor/tasks/trivial/task.toml @@ -0,0 +1,19 @@ +version = "1.0" + +[metadata] +author_name = "Corbits" +author_email = "dev@corbits.dev" +difficulty_explanation = "Write a single file with a fixed string" +category = "smoke" +tags = ["trivial", "corbits", "harbor"] + +[verifier] +timeout_sec = 60.0 + +[agent] +timeout_sec = 300.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/evals/harbor/tasks/trivial/tests/test.sh b/evals/harbor/tasks/trivial/tests/test.sh new file mode 100755 index 00000000..4e870fed --- /dev/null +++ b/evals/harbor/tasks/trivial/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /logs/verifier + +if [ -f /app/hello.txt ] && [ "$(cat /app/hello.txt)" = "hello" ]; then + echo 1 > /logs/verifier/reward.txt + exit 0 +fi + +echo 0 > /logs/verifier/reward.txt +exit 1 diff --git a/evals/harbor/tests/__init__.py b/evals/harbor/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/harbor/tests/test_argv.py b/evals/harbor/tests/test_argv.py new file mode 100644 index 00000000..a28ff6da --- /dev/null +++ b/evals/harbor/tests/test_argv.py @@ -0,0 +1,117 @@ +"""Unit tests for Harbor adapter argv/settings helpers (no Harbor package).""" + +from __future__ import annotations + +import unittest + +from evals.harbor.argv import api_key_env_names, build_exec_argv, build_settings + + +class BuildSettingsTests(unittest.TestCase): + def test_requires_base_url_and_always_emits_base_url(self) -> None: + settings = build_settings( + "xai", + "grok-4.5", + "sk-test", + base_url="https://api.x.ai/v1", + ) + self.assertEqual( + settings, + { + "providers": { + "xai": { + "apiKey": "sk-test", + "models": ["grok-4.5"], + "baseURL": "https://api.x.ai/v1", + } + } + }, + ) + + def test_base_url_and_shell_timeout(self) -> None: + settings = build_settings( + "openai", + "gpt-5", + "sk-openai", + base_url="https://example.com/v1", + shell_timeout_ms=120_000, + ) + self.assertEqual( + settings["providers"]["openai"], + { + "apiKey": "sk-openai", + "models": ["gpt-5"], + "baseURL": "https://example.com/v1", + }, + ) + self.assertEqual(settings["shell"], {"timeoutMs": 120_000}) + + def test_omits_shell_when_unset(self) -> None: + settings = build_settings( + "codex", + "gpt-5.3-codex", + "sk", + base_url="https://api.openai.com/v1", + ) + self.assertEqual( + settings["providers"]["codex"]["baseURL"], + "https://api.openai.com/v1", + ) + self.assertNotIn("shell", settings) + + +class BuildExecArgvTests(unittest.TestCase): + def test_exact_order_includes_skip_permissions(self) -> None: + argv = build_exec_argv( + cwd="/app", + config_path="/tmp/corbits-settings.json", + provider="xai", + model="grok-4.5", + prompt="Write hello.txt", + ) + self.assertEqual( + argv, + [ + "corbits", + "exec", + "--cwd", + "/app", + "--config", + "/tmp/corbits-settings.json", + "--provider", + "xai", + "--model", + "grok-4.5", + "--dangerously-skip-permissions", + "Write hello.txt", + ], + ) + + def test_custom_binary(self) -> None: + argv = build_exec_argv( + cwd="/work", + config_path="/cfg.json", + provider="xai", + model="grok-4.5", + prompt="hi", + binary="/usr/local/bin/corbits", + ) + self.assertEqual(argv[0], "/usr/local/bin/corbits") + self.assertEqual(argv[-2:], ["--dangerously-skip-permissions", "hi"]) + self.assertNotIn("--force", argv) + + +class ApiKeyEnvNamesTests(unittest.TestCase): + def test_only_generic_and_selected_provider(self) -> None: + self.assertEqual( + api_key_env_names("openai"), ("CORBITS_API_KEY", "OPENAI_API_KEY") + ) + + def test_no_cross_provider_fallback(self) -> None: + names = api_key_env_names("xai") + self.assertNotIn("OPENAI_API_KEY", names) + self.assertNotIn("ANTHROPIC_API_KEY", names) + + +if __name__ == "__main__": + unittest.main() diff --git a/evals/public/README.md b/evals/public/README.md index 028615c3..87b37075 100644 --- a/evals/public/README.md +++ b/evals/public/README.md @@ -56,8 +56,9 @@ real public issue?** | -------------------------------- | --------------------------------------------------------- | | Corbits@Grok patch on instance X | Yes (this smoke) | | % resolved on SWE-bench Lite | Only after official Docker eval on a frozen instance list | -| vs Claude Code on TB2 | Harbor adapter (not this script) | +| vs Claude Code on TB2 | Harbor adapter: `evals/harbor/` | ## Related - Product gates: `evals/capability/` +- Harbor adapter: `evals/harbor/`