diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 660c63b32c..f93eecce10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,10 @@ jobs: # on a transient GitHub API failure (it sets no outputs when it fails). kilocode_backend: ${{ steps.filter.outputs.kilocode_backend || steps.filter_retry.outputs.kilocode_backend }} cloud_agent_next: ${{ steps.filter.outputs.cloud_agent_next || steps.filter_retry.outputs.cloud_agent_next }} + # harness-sdk is detected by its own step pair below: the shared pair's + # filter set is pinned byte-for-byte by scripts/changes-filter-retry.test.mjs + # and must stay exactly as origin/main wrote it. + harness_sdk: ${{ steps.filter_harness.outputs.harness_sdk || steps.filter_harness_retry.outputs.harness_sdk }} workspace_matrix: ${{ steps.workspaces.outputs.matrix }} steps: - uses: useblacksmith/checkout@41cdeedae8edb2e684ba22896a5fd2a3cb85db6b # v1 @@ -115,6 +119,25 @@ jobs: - 'pnpm-lock.yaml' cloud_agent_next: - 'services/cloud-agent-next/**' + # The harness-sdk filter runs as its own attempt/retry pair: it cannot + # drift from the shared pair above, whose filter set is pinned by + # scripts/changes-filter-retry.test.mjs. + - name: Detect harness-sdk changes + id: filter_harness + continue-on-error: true + uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + with: + filters: | + harness_sdk: + - 'packages/harness-sdk/**' + - name: Detect harness-sdk changes (retry) + id: filter_harness_retry + if: steps.filter_harness.outcome == 'failure' + uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + with: + filters: | + harness_sdk: + - 'packages/harness-sdk/**' - name: Detect changed workspaces with tests id: workspaces run: | @@ -394,6 +417,35 @@ jobs: - name: Run cloud-agent-next tests run: pnpm --filter cloud-agent-next test:all + harness-sdk: + needs: [changes, typecheck, lint, format-check, drizzle-check] + if: needs.changes.outputs.harness_sdk == 'true' + runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} + timeout-minutes: 15 + steps: + - uses: useblacksmith/checkout@41cdeedae8edb2e684ba22896a5fd2a3cb85db6b # v1 + with: + lfs: true + + - name: Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + + - name: Setup Node + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Everything but the timing gate, which needs a machine nobody else is + # using. The migration check is why this job exists: the store's SQL is + # inlined into the bundle, and nothing at run time notices when the + # inlined copy stops matching the schema. + - name: Check harness-sdk + run: pnpm --filter @kilocode/harness-sdk check:ci + workspace-tests: needs: [changes, typecheck, lint, format-check, drizzle-check] if: ${{ needs.changes.outputs.workspace_matrix != '[]' }} @@ -431,7 +483,17 @@ jobs: notify-main-failure: if: ${{ always() && github.ref == 'refs/heads/main' && contains(join(needs.*.result, ','), 'failure') }} needs: - [typecheck, lint, format-check, drizzle-check, test, build, cloud-agent-next, workspace-tests] + [ + typecheck, + lint, + format-check, + drizzle-check, + test, + build, + cloud-agent-next, + harness-sdk, + workspace-tests, + ] runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} timeout-minutes: 5 steps: diff --git a/.gitignore b/.gitignore index cd5d47235a..5292be9c51 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ node_modules/ /out/ /build packages/trpc/dist/ +packages/harness-sdk/dist/ **/wrapper/dist/ *.tsbuildinfo next-env.d.ts diff --git a/apps/web/AGENTS.md b/apps/web/AGENTS.md index 41ff7f6695..57f6af3303 100644 --- a/apps/web/AGENTS.md +++ b/apps/web/AGENTS.md @@ -21,3 +21,13 @@ When `subscriptionSchedules.create()` uses `from_subscription`, do not set `meta ## Database-Backed APIs For database-backed API work, consult `packages/db/AGENTS.md` for shared PostgreSQL data-contract requirements. + + + +# This is NOT the Next.js you know + +This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. + +This block is written and re-added by `next dev` — verify at `node_modules/next/dist/server/lib/generate-agent-files.js`. Removing it from a diff only re-creates the uncommitted change; committing it with your work keeps the tree clean. + + diff --git a/packages/harness-sdk/.gitignore b/packages/harness-sdk/.gitignore new file mode 100644 index 0000000000..3ff939b5ed --- /dev/null +++ b/packages/harness-sdk/.gitignore @@ -0,0 +1,3 @@ +# `e2e/readme-check.ts` is typechecked, never run — but it names a database at +# module scope, so running it by hand leaves this behind. +sessions.db diff --git a/packages/harness-sdk/.oxlintrc.json b/packages/harness-sdk/.oxlintrc.json new file mode 100644 index 0000000000..6d9c25f9f9 --- /dev/null +++ b/packages/harness-sdk/.oxlintrc.json @@ -0,0 +1,93 @@ +{ + "$schema": "../../node_modules/oxlint/configuration_schema.json", + "plugins": ["oxc", "typescript", "unicorn", "import", "promise", "node", "vitest"], + "options": { + "typeAware": true + }, + "categories": { + "correctness": "error", + "suspicious": "error", + "pedantic": "error", + "perf": "error", + "style": "error" + }, + "env": { + "builtin": true, + "node": true, + "es2024": true + }, + "rules": { + "no-console": "error", + "typescript/no-explicit-any": "error", + "typescript/consistent-type-imports": "error", + "typescript/explicit-module-boundary-types": "error", + "typescript/no-non-null-assertion": "error", + "typescript/no-unnecessary-type-assertion": "error", + "typescript/no-floating-promises": "error", + "typescript/no-misused-promises": "error", + "unicorn/filename-case": [ + "error", + { + "case": "kebabCase" + } + ], + "import/no-named-export": "off", + "import/no-default-export": "error", + "no-ternary": "off", + "new-cap": [ + "error", + { + "capIsNewExceptions": ["Tag", "GenericTag", "TaggedError", "TaggedClass"] + } + ], + "sort-imports": "off", + "vitest/no-importing-vitest-globals": "off", + "vitest/prefer-to-be-truthy": "off", + "id-length": "off", + "import/consistent-type-specifier-style": "off", + "max-classes-per-file": "off", + "sort-keys": "off", + "no-magic-numbers": "off", + "promise/prefer-await-to-then": "off", + "require-await": "off", + "typescript/require-await": "off", + "unicorn/no-array-callback-reference": "off", + "import/prefer-default-export": "off", + "func-names": "off", + "unicorn/no-array-method-this-argument": "off" + }, + "overrides": [ + { + "files": ["**/*.test.ts", "**/*-fixture.ts"], + "rules": { + "typescript/explicit-module-boundary-types": "off", + "import/max-dependencies": "off" + } + }, + { + "files": ["src/index.test.ts"], + "rules": { + "import/no-namespace": "off" + } + }, + { + "files": ["src/index.ts"], + "rules": { + "unicorn/require-module-specifiers": "off" + } + }, + { + "files": [ + "src/plugins/store/node.ts", + "src/plugins/store/*.test.ts", + "src/plugins/conformance.test.ts", + "src/core/resume-fixture.ts", + "src/core/image.test.ts" + ], + "rules": { + "import/no-nodejs-modules": "off", + "unicorn/no-null": "off" + } + } + ] +} diff --git a/packages/harness-sdk/AGENTS.md b/packages/harness-sdk/AGENTS.md new file mode 100644 index 0000000000..7a9903f352 --- /dev/null +++ b/packages/harness-sdk/AGENTS.md @@ -0,0 +1,2075 @@ +# Harness SDK + +`@kilocode/harness-sdk` is the SDK that runs a coding agent harness. Read this +file before you change any file in this package. + +## Principles + +1. Use the strictest TypeScript settings. Do not relax a compiler flag. +2. Use the strictest oxlint and oxfmt settings. Do not add an inline disable. +3. Write a unit test only if it proves behavior. Do not write a test that + proves the absence of behavior. +4. Performance is a hard requirement. See "Performance" below. +5. Use Effect (`effect`) for the control flow, the errors, and the resources. +6. Make every part pluggable, where a second implementation is real. A seam + earns its place by the plugin somebody will actually write. Do not add one + whose only other implementation would be wrong: see "What is not + pluggable" below. +7. Own the core plugins. The package ships its own default for each plugin + point. +8. Use a library when a library does the work. Do not write what a dependency + already gives you. If you write more than about ten lines of a solved + problem, say which library you rejected and why. + + | Job | Library | + |---|---| + | Server-sent events | `eventsource-parser` | + | Schemas and validation | `typia` (compile time, via `ttsc`) | + | Effects, streams, retry, layers | `effect` | + | Request body shapes | `@anthropic-ai/sdk` and `openai`, types only | + + The two model SDKs are imported with `import type` and never called, so they + add no runtime code. They make the compiler reject a wrong field name in a + request body. + + **Rejected: `ulid`.** Its `node` export condition resolves to a build whose + first line is `import crypto from 'node:crypto'`, and its other build calls + `detectPRNG()` at module scope, which throws on a runtime with no global + `crypto` — so importing it either drags a runtime into the core or fails at + import on a mobile build. `core/id.ts` encodes the ULID itself in about + forty lines of arithmetic and takes its randomness from a plugin. This is + the one place the package writes what a library already does, and principle + 12 is why. +9. Prove behavior with a local end-to-end run. +10. Validate every incoming value at the edge with typia. An edge is any point + where a value enters from outside the package: a store, a model reply, a + tool result, a caller's input. Do not validate a value the package already + made; that costs CPU and proves nothing. + + A caller's input is the one edge with no runtime check today, and that is + a decision rather than an omission. The package ships types, a caller who + holds them cannot build a bad part without a cast, and a check here would + need a public error tag of its own for a case nobody has hit. The one + caller value that is checked is an image's media type, in + `wire/messages.ts`, because there the shape genuinely cannot carry what a + string allows. Add the rest when a caller arrives who needs it. + + The TypeScript type is the schema. `createIs()` returns a boolean and + `createAssert()` throws; both are rewritten into inlined checks when + `ttsc` compiles the file, so no schema object exists at run time. + + Prefer a boolean check on a hot path. Building an error is far more + expensive than answering no: an is-check that misses costs 0.004 us, and + an assert that throws costs 2.8 us, which is 700 times as much. Measured + 2026-09-03, Node v24.14.1, macOS arm64, median of 7 runs. +11. Keep the package maintainable. Give a file one job, and keep it small + enough to hold in your head. Past about 100 lines, ask whether it has + picked up a second job; oxlint refuses one past 300. + + Thirteen files are over 100 today and none of them is wrong. A gateway + shape is one job whether it takes 190 lines or 250, and splitting + `responses.ts` into a renderer and a reader would make a reader who asks + "how does this shape work" open two files instead of one. Length is the + prompt to ask the question, never the answer. Split when the second job + has a name — that is how `store/sqlite.ts` became four files and how + `ask.ts` gave up `exchange.ts`. +12. Be agnostic about the platform. Anything a runtime does differently is a + plugin point, not a branch and not an import. Do not reference `node:`, + `Buffer`, `process`, `globalThis`, or a DOM type anywhere in `core/`. + + There are two such points today. `FetchLike` is how a request leaves, and + `EntropySource` is where random bytes come from. Both are the same shape of + problem: every runtime has one, no two agree on where it lives, and a + package that picks for the caller stops running somewhere. + + `tsconfig.json` sets `"types": []`, which makes a first-party `process` or + `Buffer` a compile error. That is all it does — it cannot see a + dependency's own imports, and `skipLibCheck: true` removes the rest of the + leverage. So the rule is checked by reading the build, not by trusting the + compiler, and `pnpm check:platform` is what reads it. It runs after + `pnpm build` as the last step of `pnpm check`, and it fails on an import of + a Node builtin anywhere, or on `globalThis`, `process` or `Buffer` under + `core/`. It matches code, not prose: `core/id.ts` names `node:crypto` in a + comment explaining why it does not import it. +13. Measure, do not guess. A decision about performance is made from data. + Write the benchmark, run it, and put the number in the commit message or + in this file. "This looks slow" is not a finding, and neither is "this + should be faster". + + This cuts both ways. Before you optimise, measure that the cost is real + and that it is worth the change: two of the three debts this package + recorded in its first pass turned out not to exist, and the obvious fix + for one of them was twice as slow as the code it replaced. + + Measure the environment too, not just the machine in front of you. A + validator that is fast on Node is slower where `new Function()` is + forbidden, so a benchmark run only on Node can ship the wrong library. + + Measure the whole path before you optimise a part of it. This package + spent its first pass making validation 43 times faster, which was 0.005 us + of a 7 us path, and did not measure the other 99.9 percent until an + adversarial review did. Being right about the small number is not the same + as being right about where the cost is. + +## Performance + +These are requirements: + +- Use the least CPU. +- Use the least RAM. + +A change that adds an allocation on a hot path needs a measurement. A change +that reorders or rewrites the prompt prefix breaks the cache; treat it as a +regression until a measurement says otherwise. + +The cache hit ratio is a requirement on this package's own work: keep the +prefix byte-identical as the session grows. That, and not the breakpoint this +package marks, is what holds it — measured on 2026-09-04, this gateway places +its own breakpoints and caches the same whether the marker is sent or not. It +is not a requirement on the number a given provider returns, which the package +does not control. See the model run below, where served models range from 0.28 to +0.9997 on identical breakpoints, and where the same model moved from 0.80 to +0.9987 between two runs of the same prompts. + +### The validator + +Measured on the SSE hot path, per streamed token, 200k events, Node v24.14.1, +macOS arm64: + +| | codegen allowed | codegen blocked | +|---|---|---| +| zod 4, three schemas per event | 11.3 us | not measured | +| typia, boolean checks | 0.25 us | 0.25 us | + +Read the typia row carefully. `JSON.parse` is 0.250 us of it and the checks +are 0.005 us: the row is very nearly the cost of parsing, not of validating. + +The second column is what a Cloudflare Worker, an MV3 extension and a React +Native release build see, because all three reject `new Function()`. typia +costs the same in both because its checks are generated at compile time. zod +is slower there, and by how much was never measured — do not quote a figure +for that cell until someone runs it. Do not reintroduce a validator that +builds its checks at run time. + +### What a streamed token actually costs + +Marginal cost through `openSession`, `ask`, the gateway and a fake transport, +200 turns of history: + +| | us / token | +|---|---:| +| the whole path | 7.1 | +| the gateway alone | 7.6 | +| the gateway's own work: read a frame, parse it, ask the wire | 0.32 | +| SSE parse and wire read (the validator table above) | 0.25 | +| typia validation alone | 0.005 | + +Read the third row against the second. The work this package does per event is +four percent of what the gateway takes; the rest is Effect's stream runtime, +which pulls one element at a time through three stages. That gap is not +optimisable from here, and a wall-clock ceiling on it would only ever fail +because a dependency changed. + +A row that used to sit here said the Effect operator chain in +`gateway/index.ts` was 4.7 us of the 7.1 and a plain loop was 0.46, which read +as a ten-fold win waiting to be taken. It was taken on 2026-09-04 — the per +event path is now a plain function over a mutable tally, and the whole gateway +went from 8.60 to 7.61 us per event. Eleven percent, not ten-fold. The rewrite +was kept because the code is simpler, not because the number moved. + +So the guard in `gateway.perf.test.ts` is CPU busy time, not wall clock: 14.4 +us per event over 2000, 5.0 over 5000, measured with `process.cpuUsage`. It +catches a change in the shape of the work, which is all a ceiling can do. + +**Memory was measured and the measurement was thrown away.** Heap held per turn +of a 4000 turn session read 2.05, 3.72 and 0.09 kB on three runs of the same +code, through `process.memoryUsage().heapUsed` with `--expose-gc` and three +collections between readings. Per streamed answer it read -8 MB on one run and ++29 MB on the next. A number that moves like that is not a guard, and shipping +it as one would be worse than having none. If somebody needs this, it wants a +heap snapshot and not a delta — do not re-add the delta version. + +What one exchange holds while the reply streams is one record behind one ref, +not a ref per field. Copying the other field on every token costs 0.054 us +against 0.402 for the update itself, measured 2026-09-04 over 200000 rounds — +a third of a percent of what a token costs through the whole session, for one +concept instead of four and a single read at the end. The record is two fields +now: the answer's text, and the thinking as one ordered list. See "Reasoning +goes back exactly as it came" for why the list is not two fields. + +Identifiers, measured the same day: + +| | us | +|---|---:| +| one identifier, through `Effect.runSync` | 0.86 | +| two identifiers, which is one question | 1.69 | +| one `entropy.bytes(16)` draw | 0.59 | + +The draw looks expensive next to the rest, and it is — but the monotonic +counter only draws when the millisecond changes. Measured over 200000 +identifiers spanning 179 ms: 180 draws, one per millisecond, 1111 identifiers +each. The cost of randomness is bounded by the clock, not by how many +identifiers are asked for. + +That cost does not grow with the answer. Measured 2026-09-04 through the whole +session, one pass each: 19.6 us per token over 200 deltas, 9.3 over 1000, 7.0 +over 5000, 7.1 over 20000. The median of five rounds is steadier and lower at +the short end: 13.4 over 200 and 7.1 over 5000. Either way the figure falls and +then flattens, because the fixed cost of opening a session is spread over more +tokens and the append itself is flat. A copy of the answer per delta would be +quadratic, so `pnpm test:perf` compares the short answer against the long one +rather than pinning either number. + +Widening the failure reader on 2026-09-04, which added a typia union check to +every frame, cost nothing measurable: 7.11 us per token against 7.04 before it. + +Nothing here is being changed: 7 us per token is 7 ms on a thousand token +answer, against seconds of model latency. The table exists so the next change +to this path argues from data, in either direction. + +### What a whole request costs before the socket + +Everything one question costs on this side, for a 200 turn session and a 200 +rule system prompt, measured 2026-09-04 on the same machine: + +| | us | +|---|---:| +| `assemble` | 16.8 | +| `messagesWire.toBody` | 5.8 | +| `responsesWire.toBody` | 11.0 | +| `completionsWire.toBody` | 7.5 | +| `JSON.stringify` of the body, 27 kB | 32.0 | +| all three together | 48.1 | + +Against that, the ten model matrix reported a median time to the first piece +of the answer between 849 ms and 4064 ms. The whole client path is a +ten-thousandth of the wait, and `JSON.stringify` is two thirds of it — so +there is nothing here worth optimising, and a change that claims to speed up +a request has to say what it is actually speeding up. The perf suite gates it +at 250 us to catch a rewrite that makes it matter. + +### What CI runs + +`pnpm check` is the local gate. `pnpm check:ci` is the same thing without the +timing gate, which needs a machine nobody else is using, and it is what the +`harness-sdk` job in `.github/workflows/ci.yml` runs whenever a file under +`packages/harness-sdk/` changes. + +The job exists for `check:migrations`. The repository's own workspace test job +runs `pnpm test` and nothing else, so without it the inlined SQL could drift +from the schema and every check in the repository would still be green. + +The live runs are not in CI and must not be: they cost money and they need a +kilo token. + +## The toolchain + +The compiler is `ttsc`, not `tsc` or `tsgo`. It is the TypeScript-Go compiler +with a plugin host, and typia's transform runs inside it. Stock `tsc`, `tsgo` +and `tsx` all emit code where every `createIs` and `createAssert` call throws +`no transform has been configured` when it runs. + +| Job | Command | +|---|---| +| Typecheck | `pnpm typecheck` (`ttsc --noEmit`) | +| Build | `pnpm build` (`ttsc -p tsconfig.build.json`) | +| Tests | `pnpm test` (vitest, transformed by `@ttsc/unplugin`) | +| Timing | `pnpm test:perf` (`vitest.perf.config.ts`, one file at a time) | +| End-to-end | `pnpm test:e2e` (`ttsx`, not `tsx`) | +| One live run | `pnpm test:e2e:` + `image`, `cancel`, `reasoning`, `stop`, `compact`, `shapes`, `session`, `resume`, `clone`, `replay`, `models`, `queue`, `together`, `subagent`, `tool-matrix`, `conversation`, `time`, `todo` | +| Every live run | `pnpm test:e2e:all` (add names to pick a few, `full` for all eleven models) | +| Raw frames | `pnpm test:e2e:probe ` (asserts nothing) | + +`pnpm test:perf` is a separate config because its files must not run beside the +unit tests: parallel workers compete for the CPU being measured. Its ceilings +are about five times the recorded numbers, so it catches a regression in order +of magnitude and not a busy laptop. A timing test that fails on a loaded +machine would be turned off within a week, which is worth less than no test. + +Because the transform rewrites source, the package ships `dist/` and not +`src/`. A consumer importing the TypeScript directly would get the throwing +version. Run `pnpm build` after changing a validated shape, or a dependent +package reads a stale check. + +Every test here runs against `src/`; a consumer runs against `dist/`, through +the `exports` map. `pnpm check:package` is the only thing that reads the build +the way a caller does: it imports every subpath, asks each for a name it +promises, and then asks one session a question through the built gateway +against a `fetch` that answers from memory. That last part runs a compiled +validator over a stream event, which nothing else does. Both halves were shown +to fail on purpose on 2026-09-04 — a subpath pointed at a file that is not +there, and a built `toDelta` that returns nothing. + +The first `ttsc` run on a machine compiles typia's plugin from Go and takes +minutes. Later runs read a cache and take about a second. + +**A type error anywhere in `src/` disables the transform everywhere.** typia +bails when the program does not compile, and every `createIs` and +`createAssert` then throws `no transform has been configured` — including in +files that have nothing to do with the error. One unused import in a test file +made `pnpm test:e2e` fail inside `wire/completions.ts`. If a `ttsx` run throws +that message, run `pnpm typecheck` first and read the error it reports, not the +stack it printed. + +**A filtered install breaks the same thing, with `pnpm typecheck` still green.** +`pnpm install --filter @kilocode/harness-sdk` left `ttsx` throwing +`no transform has been configured` on 2026-09-04 while `ttsc` and vitest kept +transforming, so only the live runs failed. A full `pnpm install` from the +repository root fixed it. Install for the whole workspace, not for this +package. + +## Rules + +- Do not add an abstraction with one implementation, unless it is a declared + plugin point. +- Do not add a dependency for work that a few lines do. +- Keep the file count low. Put one plugin point in one file. +- Name a file in kebab case. Export a type with `export type`. +- Format first, then check. `pnpm -w run format:changed` can reflow a file past + the 300-line cap, and a check run before it says nothing about what it wrote. +- Run `pnpm check` in this directory before you commit. It runs the compiler + over `src/` and over `e2e/`, the linter, the boundary check, the migration + check, the tests, the build, the platform check, the package check, and the + timing gate. +- If you change a code block in the README, change `e2e/readme-check.ts` with + it. It is every README snippet against this source tree, and it is + typechecked and never run: a snippet that does not compile is worse than no + snippet. `pnpm typecheck:e2e` is what catches it. `e2e/plugins-check.ts` does + the same for `PLUGINS.md`. + +## The kilo gateway + +`POST {baseUrl}/api/gateway/v1/messages` takes the Anthropic Messages body, so +`cache_control` reaches the model. + +| Header | Value | +|---|---| +| `authorization` | `Bearer {user token}` | +| `x-kilocode-organizationid` | The organization id. Leave it out for a personal account. | + +The route serves three shapes. A model does not always speak all three, and the +gateway resolves that from the serving provider without publishing it, so the +caller gives the plugin an `apiKinds` function. + +| Shape | Path | What it sends about the cache | +|---|---|---| +| `messages` | `/api/gateway/v1/messages` | An explicit `cache_control` breakpoint | +| `responses` | `/api/gateway/v1/responses` | A `prompt_cache_key` the caller names | +| `chat_completions` | `/api/gateway/v1/chat/completions` | Nothing | + +**The hit ratio is not comparable between shapes.** `pnpm test:e2e:shapes` asks +the same two questions of the same model through each one: + +| Shape | Cache read | Input | Ratio | +|---|---:|---:|---:| +| `messages` | 11224 | 6 | 0.9995 | +| `responses` | 11224 | 11247 | 0.4995 | +| `chat_completions` | 11224 | 11246 | 0.4995 | + +All three read the same 11224 tokens, so all three cached equally well. What +differs is the billing of the cold call: `messages` reports the first prefix as +`cache_creation`, and the other two report it as plain input, which the ratio +then divides by. So the ratio measures caching *and* how a provider books a +cache write, and only the `messages` column can be read as a cache figure. Hold +a shape to `cacheReadTokens > 0`, not to a ratio, unless it is `messages`. + +The plugin picks `messages` first, then `responses`, then `chat_completions`. +That order is what each shape lets a caller control, best first. + +**On this gateway, none of it changes the cache.** Measured on 2026-09-04 with +a prefix nobody had sent before — a nonce in the header and in all 200 system +rules, so the first call of every run is cold and the second measures a cache +that run wrote: + +| Shape | Model | Sent | Cache read on the second call | +|---|---|---|---:| +| `messages` | `anthropic/claude-haiku-4.5` | breakpoint | 14032 | +| `messages` | `anthropic/claude-haiku-4.5` | nothing | 14032 | +| `messages` | `openai/gpt-5.6-luna` | breakpoint | 12229 | +| `messages` | `openai/gpt-5.6-luna` | nothing | 12229 | +| `responses` | `openai/gpt-5.6-luna` | `prompt_cache_key` | 12229 | +| `responses` | `openai/gpt-5.6-luna` | nothing | 12229 | +| `chat_completions` | `anthropic/claude-haiku-4.5` | breakpoint | 13630 | +| `chat_completions` | `anthropic/claude-haiku-4.5` | nothing | 13630 | +| `chat_completions` | `openai/gpt-5.6-luna` | breakpoint | 12229 | +| `chat_completions` | `openai/gpt-5.6-luna` | nothing | 12229 | + +Every pair is identical to the token, on a native Anthropic model as much as on +a relayed one. The gateway places its own breakpoints, so what this package +sends is redundant there today. + +What that does **not** change: the prefix discipline is still what makes the +cache work. Append-only turns, a frozen system prompt, model and effort, and +images kept as the base64 the wire wants — the gateway's own breakpoints need a +stable prefix as much as ours would. Only the marker is redundant, not the rule +that nothing in front of it may move. + +`chat_completions` stopped sending its breakpoint on 2026-09-04, because +`cache_control` is not part of that API at all — it was a non-standard field +the gateway would have had to translate, and it bought nothing. `messages` +keeps its breakpoint: it is the documented mechanism for the body being sent, +it costs nothing, and its absence would show up only as a bill if the gateway +ever stopped inserting its own. Re-run the measurement before changing that; +`wire/image.test.ts` holds the method. + +A call is tried again on a transport failure and on 408, 409, 425, 429, 500, +502, 503, and 504. The retry stops as soon as the status is good, before the +body is read, so a second try never repeats text the caller has already seen. +The older `/api/openrouter` prefix also works; the package does not use it. + +### The gateway wants the session id in a header + +Every call carries `x-kilo-session`, which is the session's own id. The gateway +hashes it into the upstream provider's cache key — a `prompt_cache_key` on the +OpenAI shapes, a `session_id` on OpenRouter's — and seeds the routing that keeps +one conversation on one provider with it. A call that sends none gets neither: +`applyTrackingIds` in `apps/web/src/lib/ai-gateway/providerHash.ts` skips both +fields when the hash is empty. `x-kilocode-taskid` is the same field under the +editor's name for it; the gateway documents `x-kilo-session` for everybody else. + +The package sent nothing until 2026-09-05, so every call in a session looked to +the gateway like a call from a stranger. + +**It did not change what a provider cached, measured.** Two calls over an 11.2k +prefix on `z-ai/glm-5.3-flash`, three times — no header, `x-kilo-session`, +`x-kilocode-taskid` — read 6912 tokens from the cache and were billed 4322 as +fresh input every time, on the same provider. The header is sent because the +gateway asks for it and it is what carries the cache key upstream on the OpenAI +shapes, not because it fixed a number here. + +### Why a hit ratio differs so much between providers + +`z-ai/glm-5.3-flash` reads 0.61 of its prompt from cache where Haiku reads 0.98 +of the same conversation. It is the upstream provider, not the package: friendli +caches 6912 tokens of an 11.2k prefix — a whole number of 256-token blocks — and +bills the remaining 4322 every call. Anthropic caches to the breakpoint. + +So a floor on the hit ratio is a floor on somebody else's implementation. +`e2e/live.ts` asserts 0.5, which every provider in the list clears, and the +number itself goes in the table for a reader to compare. + +## The local end-to-end run + +`pnpm test:e2e:all` runs every live check in one sweep, cheapest first, and +reports one line each. One failure does not stop the rest: the point of a sweep +is to learn everything that broke. Name one or more to run a subset, as in +`pnpm test:e2e:all stop reasoning`. These runs cost real money and real time, so +they are not part of `pnpm check` and never will be. + +### One model by default, eleven when you say so + +Every run takes a **list** of models and works through it. The list holds one +model, `z-ai/glm-5.3-flash`, and the word `full` on the command line asks for +all eleven — the ten most used models on OpenRouter, from six vendors, and +Haiku for the lab that list leaves out. `pnpm test:e2e:all full`, or +`pnpm test:e2e:conversation full` for one run. `KILO_MODELS` names any list and +wins over `full`. + +One by default is about money: a sweep of eleven is eleven times the bill, and +most changes touch one code path. The list lives in `e2e/setup.ts` and nowhere +else, because two copies drift — the tool matrix ran Haiku for a week that the +model run did not. + +Run the affected runs after a change, on the one model. Run `full` when the +change is to the wire, or before saying a thing works everywhere: a run tuned to +one model's habits is a run that passes for the wrong reason. + +### What a live run may assert, and what it may only count + +Eleven models is eleven vendors' habits, and a run that asserts a habit goes red +on a different model every sweep for something no change here can fix. So a live +run draws one line: + +- **What this package does is asserted.** The shape carried the tool. The word + the tool wrote came back. The line kept its order. The store held the rounds. + The signal ended aborted. These are absolute, on every model, always. +- **What the model chooses is counted.** Whether it calls the tool, sends two + calls together, sets `wait`, leaves a question outstanding, or answers before + the caller can send a call away. The run prints how many models chose it and + asserts the **floor**: that some model did, because none of them doing it is + the package and not eleven vendors agreeing. + +Two more rules hold the line honest: + +- **A round that failed is tried once more before it counts.** A relay having a + bad minute is not a finding; twice is. +- **A model that cannot do the thing is not asked to.** `e2e/image.ts` skips a + model on the gateway's own "does not accept image input", read from the + refusal rather than a list of names here that would rot without saying so. + +Measured on 2026-09-06, this is what the line separates in practice: +`nvidia/nemotron-3.5-lightning` sends the two calls of one turn one after the +other and takes no pictures at all, `openai/gpt-5.6-luna` answers two questions +out of its own head rather than asking, `google/gemini-3.7-flash` answers before +a subagent can be sent away, and `minimax/minimax-m3` spends over thirty seconds +thinking before its first word. Not one of those is a defect in this package, +and every one of them failed a run before the line was drawn. + +`e2e/report.ts` is how a run says both: `wrongIf` for what is asserted, a +printed count for what is chosen. + +The whole sweep, one model, 2026-09-05, 8 minutes 30 seconds: + +``` +PASS live 6s every model read the prefix back from the cache +PASS shapes 11s every shape carried the conversation, and both cached +PASS stop 8s a finished answer, told from one the ceiling cut off +PASS tools 19s every shape ran a tool, and a late answer drove a round +PASS image 8s every shape carried the picture and replayed it +PASS cancel 12s the call stopped when the caller did +FLAKE queue 6s one empty answer; passed on the next run, unchanged +PASS together 276s a late answer and a typed message shared one line +PASS subagent 9s a task went down, one answer came up, a call was sent away +PASS session 19s the prefix held across 10 calls, a busy session refused +PASS resume 8s the stored count is the provider's own +PASS clone 9s every clone read its prefix and wrote next to nothing +PASS reasoning 24s every shape took its own thinking back +PASS replay 25s every shape took back thinking that had been stored +PASS compact 10s the session compacted itself and kept what it was told +PASS models 10s 1 of 1 models answered every turn +PASS tool-matrix 18s every model called every tool with a payload it could read +PASS conversation 40s one whole conversation, tools to summary +``` + +`together` is the long one: it waits for three particular rounds, and a model +that takes its time over one of them holds the whole run there. + +`models` fails about one run in five on a single third-party model that answers +one turn with nothing. It is the model, not the package: the same model passes +on the next run, and the failure is an empty answer rather than a refused call. + +`shapes` reads a ratio of zero about as often, and for the provider's own +reason: the entry is not readable yet. `google/gemini-3.7-flash` on `messages` +read nothing on 2026-09-06 and read 8010 tokens back on the next run, unchanged. +The one pairing that reads nothing back *every* time is an Anthropic model on +the responses shape, which this gateway does not cache; `shapes.ts` names that +pair rather than explaining it in a failure, so every other pairing still fails. + +The sweep was run whole on a second model, `openai/gpt-5.6-luna`, on 2026-09-04, +back when each run named its own. Thirteen of the fifteen passed unchanged. Both +failures were the run's fault and neither was the package's: + +- `image` opened its session with a ceiling of 16 tokens, which a model that + thinks spends before it says anything, so two shapes answered with nothing. + The ceiling is what `stop.ts` tests; this run tests the picture, and now asks + for 256. +- `replay` took its model from the environment, and what it proves is that a + seal survives SQLite. A model that seals nothing reported the run's own + subject missing and called it a defect. It names its model now, as + `reasoning.ts` already did. + +Moving the default to `z-ai/glm-5.3-flash` on 2026-09-05 found two more of the +same kind, both fixed: `live` opened with a ceiling of 32 tokens and `cancel` +with 16, and a model that thinks spends that before it says a word. The first +sweep across all eleven, on 2026-09-06, found two more. `shapes` opened with 64 +and reported twelve empty answers as twelve broken shapes; at 256 every model +answered on every shape. `stop` gave 64 tokens to a question whose answer is one +word, and seven model-shape pairs reported the wall they were meant to clear — +its *other* half, the ceiling it means to hit, stays at 24. A run that has only +ever seen one model has not been held to this rule yet, and a run whose subject +is the ceiling has to be read twice. **A live +run that gives a model under 256 tokens is testing the ceiling, whether it means +to or not.** The same move loosened `live`'s cache floor from 0.95 to 0.5: Haiku +reads back over 0.99 of a prefix and glm reads 0.61 of the same conversation, +because a provider caches at a granularity of its own. What the run defends is +that the prefix was read rather than built again. + +### Nothing goes around the reading of `session.continued` + +`Effect.timeout(watch(...), '180 seconds')` reads like a deadline on collecting +the rounds a session runs on its own. It is not one. Measured live on +2026-09-05, three times each: with `Effect.timeout` around it the reading +subscribes and then receives nothing at all, while the rounds run and the store +fills; with `Stream.interruptAfter` inside it the stream ends at once and +reports no rounds. Both put a race around the subscription to the session's +feed, and the subscription's scope goes with the race. + +`e2e/rounds.ts` forks the reading bare and puts the deadline on the *waiting* +for it, where a race costs nothing, and it reports what was collected before the +deadline rather than throwing it away with the failure. Two rounds of three is a +far better failure to read than none. A caller doing this in their own harness +needs the same shape. + + +`pnpm test:e2e` asks two questions in one session against the real gateway and +checks that the second call reads the cache the first one wrote. It uses the +kilo CLI token from `~/.local/share/kilo/auth.json`, never prints it, and +spends a small amount of real credit. + +The system prompt is long on purpose: the cached prefix must clear the model's +minimum, 4096 tokens on Haiku 4.5, or nothing caches and the check fails for +the wrong reason. + +About one run in five fails with a ratio of 0, because the provider has not +made the entry readable yet. Re-run before looking for a bug — a real prefix +regression fails every time, not one time in five. + +Measured on 2026-09-03, `anthropic/claude-haiku-4.5`, the Kilo organization: + +| Call | input | cache read | cache write | +|---|---:|---:|---:| +| First | 3 | 0 | 11822 | +| Second | 3 | 11822 | 13 | + +The second call read the whole prefix and wrote only what the first exchange +added. The hit ratio was 0.9995. That is the healthy shape; a growing `input` +or a repeated large `cache write` means the prefix moved. + +`webFetch` is the whole adapter for a runtime that has a WHATWG `fetch`, and +`src/plugins/fetch/web.ts` is about twenty lines of which fifteen are the +ambient declarations. That is the measure of what `FetchLike` asks of a +caller. + +### The shapes, and a session that grows + +`pnpm test:e2e:shapes` forces each of the three shapes by telling the catalog a +model speaks only that one. It exists because every model in the model run +picks `messages`, so the other two shapes had only ever run against a fake +`fetch`. + +`pnpm test:e2e:session` asks ten questions of one session and reads the counts +per call. It is the live form of the append-only invariant: the unit test +proves `assemble` does not rewrite an earlier message, and this proves the +provider agrees. Measured on 2026-09-03 with `anthropic/claude-haiku-4.5`: + +| Call | input | cache read | cache write | +|---|---:|---:|---:| +| 1 | 3 | 11822 | 0 | +| 2 | 3 | 11835 | 0 | +| 3 | 3 | 11835 | 13 | +| … | 3 | +13 each | 13 | +| 10 | 3 | 11926 | 13 | + +Cache read grows by exactly one exchange per call and cache write stays at +exactly that exchange. A prefix that moved would show as a large write on a +late call, which is what the run asserts. It also asks a second question while +the first is still streaming, and requires `SessionBusyError`. + +### Tools, on every shape + +`pnpm test:e2e:tools` is the only proof that the provider reads what this +package writes for a tool call. The three shapes disagree about how one is +written — `messages` writes blocks, `responses` writes items beside the message, +`chat_completions` writes a field on the assistant message and a role of its own +for the result — and a shape that refuses a round refuses the whole session, +because a call whose result it will not read can never be answered. + +The run holds three claims. Each shape carries a round: the model calls the +tool, reads what it said, and answers with a word it could not have invented. +The calls of one turn overlap, measured on the clock rather than assumed. And a +question answered slower than the model waits drives a round of its own, which +is the whole of backgrounding end to end, against a real model deciding for +itself whether to carry on. + +Measured 2026-09-04, `anthropic/claude-haiku-4.5`: + +``` +shape calls answered +messages 1 "The weather in Oslo is kestrel." +responses 1 "The weather in Oslo is kestrel." +chat_completions 1 "The weather in Oslo is kestrel." + +two calls in one turn overlapped by 600ms +asked, not waited: "I'm waiting for your answer about your favourite colour." +told later: "You said your favourite colour is ultramarine." +``` + +### The line, against a model that remembers + +`pnpm test:e2e:queue` proves the line is a conversation. A fake proves the line +forms; only the provider proves a queued message is answered from the same +transcript, in the order it joined. + +One session and three questions. Two messages are handed over from inside the +first answer's own stream, which is the one moment the session is certainly +held, and neither refuses. A third is queued between them and cancelled while +it waits: it is never said to the model, and cancelling it twice answers false +the second time. The first queued message asks the model for the word it last +answered, so an answer carrying that word is the round running in this session +and not beside it. + +Measured 2026-09-04, `anthropic/claude-haiku-4.5`: + +``` +asked while free: "ferret" +round 1 answered: "ferret" +round 2 answered: "badger" + +waiting while busy: ["Answer with the word you last answered.","Answer with the word 'pangolin'.","Answer with the word 'badger'."] +took one back: true, and again: false +left in the line: 0 +what the session was asked: ["Answer with the word 'ferret'.","Answer with the word you last answered.","Answer with the word 'badger'."] +``` + +The run has its own system prompt rather than the shared one. The shared prompt +forbids everything but a single word, and the model read "answer with the word +you last answered" as a rule it had to refuse — which failed the run for the +prompt rather than for the line. + +Breaking `cancelQueued` so it removes nothing fails five of the run's claims, +including `the cancelled message was said to the model anyway`. + +### A subagent, and a call sent away + +`pnpm test:e2e:subagent` proves the two things a fake cannot. That a real model +reads the subagent tool as something to hand a whole task to, and that what +comes back is usable as an answer. + +The subagent is told a codename the parent never sees, so an answer carrying it +came up through the tool. Its identifier is not the parent's and its counts are +its own. Then the same tool is run again with a five-minute deadline and sent +away by the caller the moment `session.running` shows the model waiting on it: +nothing here is the clock. The model answers without it and is told the result +in a round of its own. + +Measured 2026-09-04, `anthropic/claude-haiku-4.5`: + +``` +the parent answered: "This quarter's codename is \"nightjar.\"" +the subagent said: "nightjar" +parent spent 1621 tokens, subagent 64 + +sent away: "subagent" +answered without it: "I've asked the subagent to find this quarter's codename and am waiting for its response." +told later: "This quarter's codename is Nightjar." +``` + +### One line, contended + +`pnpm test:e2e:together` is the run that makes a late tool result and a queued +message wait for the same session at the same moment. They are the same thing to +the session — a message it owes when it is free — and they are held in one line +so the order between them is defined rather than a race. + +The model is asked to find out two things, and takes both in one call to the +question tool, with choices: the tool's richer shape, exercised by a model +rather than by a test. The asker sleeps far longer than the model waits, so the +model is told the question is out and answers without it. A message is queued +that holds the session open on a tool that sleeps, and the run then waits until +it can see the answer waiting in the line before queueing a second message +behind it. All three are certainly in the line together. + +The claim is that the rounds run in the order the entries joined. It is checked +against the identifiers rather than against a clock: an identifier is made when +its entry joins the line and sorts by when it was made, so sorting the three is +the order the session owes them. An earlier draft asserted a fixed sequence and +failed twice on timing the model controls, not on anything the package did. + +Measured 2026-09-04, `anthropic/claude-haiku-4.5`: + +``` +asked in one call: 2 questions + [colour] Which colour do you want for your bicycle? {ultramarine, vermilion} + [animal] Which animal should be on the bell? {marmoset, kestrel} + +the line, once the answer had joined it: ["toolResult","message"] +and once a message was typed after it: ["toolResult","message","message"] + +round 1 (the late answer): "You picked ultramarine for your bicycle colour and a marmoset for the animal on the bell." +round 2 (the slow message): "Narwhal." +round 3 (the message typed after): "Pelican." +``` + +Breaking `takeRun` so it prefers a message to a tool result fails exactly one +claim, and names the order it ran instead. + +This run found a gap in what the package said. `done` ends one call to the +model, not the round: a round that calls a tool makes several. A caller watching +`continued` therefore cannot count rounds by `done`, and knows a queued message +is answered in full only on the first `done` that stops on anything but +`tools`. That was true and undocumented; `SessionHandle.continued` and the +README say it now. + +### A picture, and a caller who walks away + +`pnpm test:e2e:image` sends a coloured circle through each of the three shapes +and asks what colour it is. Each shape gets a **different** colour, so a model +that never saw the picture would have to guess three specific words to pass. + +The second question is the one that matters. It asks whether the background is +white or black, which the first answer never said, so it can only be answered +from the picture. An assembler that dropped the image from the second request +would leave the model with its own earlier word and nothing else. Checked by +returning `[]` for an image part in `assemble`: five of the six assertions fire +and the model answers `i don't see any image attached to your message`. + +`pnpm test:e2e:cancel` asks for a long answer twice, reads one to the end, and +walks away from the other after ten pieces. It waits for pieces and not for a +clock, because the time to the first piece is longer than a short wait: a fixed +700 ms only ever cancelled a request that had not started answering. + +It asserts four things a fake `fetch` cannot show: the run stops mid-stream +(11 pieces of 76, 1.9 s of 3.3 s), every signal ends aborted, the abort leaves +no unhandled rejection in undici, and the session can still be asked a question +afterwards. Checked by replacing the release with `Effect.void`: the timing does +**not** change, because interrupting the fiber stops the reader either way and +only the socket leaks. The signal assertion is what catches it. + +**What neither run proves:** that the provider stops generating and stops +charging. Nothing this package can read reports that. + +### The thinking, handed back to the provider + +`pnpm test:e2e:reasoning` asks a reasoning model two questions in one session, +so the second request carries the first answer's thinking block. The provider is +the only judge of whether the block is right, and it answers with a status, not +with a different answer. + +Measured on 2026-09-04 with `anthropic/claude-sonnet-4.5` at `medium` effort: +363 characters of thinking, one reasoning part stored, a 792 character +signature, and a second answer that built on the first. + +Checked on the messages shape by reversing the signature before sending it: + + 400 messages.1.content.0: Invalid `signature` in `thinking` block + +So that shape does validate it, and a pass means the block went back intact. + +**The responses shape carries no such proof.** Reversing its +`encrypted_content` changes nothing: the call succeeds and the answer is right. +So the item is built to the published shape and is sent, and nothing observable +says the provider reads it. Treat that replay as unverified. + +Pick a model that thinks. One that does not would pass this run vacuously, +which is why the run fails when a sealing shape reports no thinking. + +`pnpm test:e2e:probe ` prints the raw frames of one call. It +asserts nothing; it exists so a question about the wire is answered by reading +the wire. It is how the `response.reasoning.delta` name was found. + +### A session that outgrows its window + +`pnpm test:e2e:compact` plants a fact, fills a deliberately tiny window with +unrelated talk, and asks for the fact back after the session has compacted +itself. The window is the caller's, not the model's: filling 200k tokens to +test this would cost real money and take an hour, and what is live here is the +summariser, the prompt it builds, and whether the fact survives. + +Measured on 2026-09-04 with `anthropic/claude-haiku-4.5` and an 80 token window: +one compaction, and a summary that opens `- Vault code: 4417`. The run reads the +summary itself, not just the answer, because the model could reach the answer +another way. + +It also adds up the `done` events. Those are what the caller's own questions +cost, so `session.usage` has to come out above them — the difference is the +summary call, which the caller never asked for and is billed for anyway. The +same run: 70 output tokens against 37 the questions account for. + +### Many models, a longer conversation + +`pnpm test:e2e:models` holds a five turn conversation with each of the ten most +used models on OpenRouter. The last question can only be answered from the +history, so the run proves the prompt actually carries the conversation. + +Measured on 2026-09-04, five questions each, the Kilo organization. `first` is +the median wait for the first piece of an answer, `whole` for all of it: + +| Model | Recalled | First | Whole | Cache read | Input | Ratio | +|---|---|---:|---:|---:|---:|---:| +| `openai/gpt-5.6-luna` | yes | 1124 ms | 1309 ms | 56324 | 15 | 0.9997 | +| `qwen/qwen3.8-flash` | yes | 2480 ms | 2481 ms | 59014 | 30 | 0.9995 | +| `minimax/minimax-m3` | yes | 1109 ms | 1122 ms | 57009 | 75 | 0.9987 | +| `xiaomi/mimo-v2.5` | yes | 2023 ms | 2348 ms | 57600 | 194 | 0.9966 | +| `tencent/hy3` | yes | 3143 ms | 3223 ms | 55808 | 491 | 0.9913 | +| `deepseek/deepseek-v4-flash` | yes | 3671 ms | 3671 ms | 54528 | 2116 | 0.9626 | +| `deepseek/deepseek-v4-flash-0731` | yes | 3891 ms | 3948 ms | 53760 | 2884 | 0.9491 | +| `z-ai/glm-5.3-flash` | yes | 1373 ms | 1471 ms | 34560 | 21754 | 0.6137 | +| `google/gemini-3.7-flash` | yes | 2266 ms | 2370 ms | 24432 | 34207 | 0.4167 | +| `nvidia/nemotron-3-ultra-550b-a55b` | yes | 869 ms | 869 ms | 16384 | 42475 | 0.2784 | + +Ten of ten answered every turn from the history. Every one used the `messages` +shape. + +`tencent/hy4-preview` was in this list and is not served to this team — a 404 +reading `model_not_allowed` — so `qwen/qwen3.8-flash` took its place. + +The waits are the provider's, not the package's: building a whole request +costs 48 us on this side, against a first piece between 869 ms and 20 s. The +same model varies by a factor of ten between runs — `xiaomi/mimo-v2.5` took +1419 ms on one run, 20 s on the next, and 2023 ms on the one above. + +The ratios move too, and by more than the request does. `tencent/hy3` read +0.79 of its input from the cache on one run and 0.99 on the next; `minimax/ +minimax-m3` went 0.80 to 0.9987, and `google/gemini-3.7-flash` 0.14 to 0.42, on +the same prompts through the same code. What the ratio measures is whether the +upstream provider still held the prefix, which is its decision and not this +package's. Read one run as weather. The one number that is climate is the top +of the table: a model whose provider caches at all lands above 0.99, and that +has held on every run. + +The prompts are identical between runs, so a ratio that fell is the provider +having dropped a prefix it once held, not a change here. Do not chase one. + +These ratios are lower than the ones recorded before the usage merge was +fixed, and the lower ones are the honest ones. The earlier table had +`deepseek/deepseek-v4-flash-0731` at 0.9943 on a cumulative input of 324 +tokens, which cannot happen: no cache exists on the first call, so the cold +call alone spends the whole prefix as uncached input. Reading the raw frames +for that model shows why the counts move: + +``` +first call, cold message_start {"input_tokens":0,"output_tokens":0} + message_delta {"input_tokens":6899,"output_tokens":44} +second call, warm message_start {"input_tokens":0,"output_tokens":0} + message_delta {"input_tokens":125,"output_tokens":39, + "cache_read_input_tokens":6784} +``` + +This relay puts zeros in `message_start` and the counts in `message_delta`, +which is the inverse of Anthropic direct. Raising rather than overwriting is +right for both: `max(0, 6899)` is 6899 either way round. When you doubt a +count, dump the frames before you reason about the aggregate. + +The cache also expires. Anthropic's entries live five minutes, measured from +the start of the request that wrote or read them, and a read refreshes them at +no cost. A `ttl: '1h'` on the `cache_control` block buys an hour at twice the +base input price. This package does not send it: the trade depends on how long +a caller's sessions idle and what its tokens cost, which is the caller's number +and not one this package can guess. Read on 2026-09-04. Anything measured after +an idle gap is measuring the expiry, not the breakpoints. + +Two lessons hold beyond any one run: + +- **The ratio is partly the provider's.** The package places the same + breakpoints for every model, the spread above runs from 0.28 to 0.9997 on + identical breakpoints, and one model moved 0.80 to 0.9987 between two runs + of the same prompts. Read a low number as a question, not a bug — but rule + out the package first, and check the arithmetic closes before trusting a + high one. +- **A small token budget reads as a broken transport.** At 64 tokens a + reasoning model spends the budget before it writes a word. The run uses 1024. +- **An Anthropic model on the `responses` shape caches nothing here.** Measured + 2026-09-04 with `anthropic/claude-sonnet-4.5` and the same two questions: + `messages` read 23635 tokens of cache, `chat_completions` read 23644, and + `responses` read 0 against 23663 of input. A raw probe outside this package + sent the same body twice, with `prompt_cache_key` and without, and got + `cached_tokens: 0` on all four calls, while `openai/gpt-5.6-luna` on the same + shape cached either way. So it is the gateway's translation, not the body + this package renders, and no field it could send would change it. + + Nothing acts on that. `pickKind` ranks `messages` first, so a catalog that + lists all three never reaches the trap, and vendor-sniffing in the core to + reorder the rest would be a guess about a relay that can change next week. + What it does mean: if `shapes.ts` is ever run against an Anthropic model, the + `responses` row reads 0 and that is the relay, not a regression. + +### Effort is not the token ceiling + +`maxTokens` is a wall the server enforces and the model cannot see. `effort` is +a dial the model follows. A reasoning model pays for its thinking out of +`maxTokens`, so the two meet, but one does not replace the other. + +Measured at 64 tokens: low effort raised answers on two of four models and +rescued none of the models that answered nothing. Raise `maxTokens` first; +reach for `effort` to cut cost once answers arrive. + +The five levels are exactly Anthropic's `output_config.effort` set, checked +against the SDK on 2026-09-04, and a subset of OpenAI's `ReasoningEffort`, +which also has `none` and `minimal`. So every value this package accepts +reaches every shape it can send. Do not add one that only one of them takes. + +## Decisions + +### The session bridges; the plugin decides + +`openSession` resolves every plugin once, at open, so the handle it returns +carries no requirement. It then tells each plugin what happened and lets the +plugin decide what to do about it. The session never decides when a store +writes, how a transport retries, or how an identifier is made. + +Three values are frozen for the life of a session: the system prompt, the model, +and the effort. The system prompt is the front of the cached prefix, a cache +belongs to one model, and a change of effort invalidates the messages cache. +Changing any of them mid-session throws the cache away, so the type does not +allow it. + +`maxTokens` is not frozen. It never reaches the rendered prefix, so it costs no +cache. Three places may set it, and the nearest one wins: + +1. The question: `session.ask(text, { maxTokens })`. +2. The session: `openSession({ maxTokens })`. +3. The `ModelCatalog` plugin's `maxOutputTokens`, when neither names one. +4. `4096`, when the catalog names none either. + +One session does one thing at a time, because two answers built on one prefix +means the second one misses the cache. A second question asked while the first +still streams fails with `SessionBusyError`. + +`session.compact` takes the same lock. Compaction rewrites the whole +conversation, and a question in flight holds the session as it stood before it +was asked, to put back if no answer comes; both at once put the pre-summary +session back while the store keeps the summary, and the two then disagree +forever. `whileFree` in `ask.ts` is the lock, and `compactIfFull` does not take +it — it runs inside a question that already holds it. + +`ask` is refused rather than made to wait. A waiting `ask` cannot work: under +`Stream.merge` the merged stream holds every child resource until all children +finish, so the first question cannot release what the second waits on, and the +acquire is uninterruptible, so `Effect.timeout` cannot break the deadlock +either. Four acquire shapes were tried and every one that waits deadlocks. + +A caller who wants to send anyway calls `queue`, which is a different thing and +not a waiting `ask`: it hands the message over and returns, and the answer comes +back somewhere else. See below. + +An answer turn is added only when the stream reaches `done`. A half written turn +would sit in the prefix of every later request. + +### A queued message is handed over, not waited on + +`ask` streams the answer where the caller stands, so it cannot wait: the stream +it would have to return is the same stream holding the lock it is waiting for. +`queue` sidesteps that by not returning a stream at all. It puts the message in +a line, returns the identifier that cancels it, and the answer comes back on +`continued`. + +One line holds two kinds of thing, because both are the same shape — words to +put in front of the model, in a turn of their own. A message a caller queued, +and the result of a tool the model stopped waiting for. Keeping them in one line +is what makes the order between them defined: a tool result that arrived before +a caller's message reaches the model first, which is the order they happened in. + +A message is a round of its own — a caller who wrote two of them meant two turns +— and tool results waiting at the front run together, because the model asked +for those calls in one turn and is waiting on all of them. Answering them one at +a time would cost a request each and tell the model less every time. + +**The driver holds the session before it takes anything out of the line.** That +order is the whole of why `cancel` can be honest. Taking first and then finding +the session busy would leave a message neither waiting nor asked: `queued` would +not show it, and `cancel` would say it was too late while nothing had been sent. +So `background.ts` runs the take and the round inside one `whileFree`, and +`ask.ts` exposes `askHeld` for a caller that already holds the lock. + +`continued` carries `{ answering, event }` rather than a bare `ModelEvent`. +Without the identifiers a caller with two messages in the line cannot tell which +answer is which, and `ModelEvent` is the transport's union — a marker event of +the session's own would have to be handled by every `switch` over it. + +The stream replays its recent events to a new reader. Otherwise the order of two +lines of a caller's own code — queue, then subscribe — would decide whether they +saw anything at all, because the round can start before the subscription does. +It is still a display buffer and not a log: it slides at 256 events, and the +transcript is the record. + +### A refused round is a value, not a failed stream + +`continued` never fails. A round the model or the store refused arrives on it as +`{ answering, failed }`, marked with the same identifiers as the events would +have been, and the driver goes straight back to the line. + +It was the other way first, as a `Take.fail` on the PubSub, which reads as the +idiomatic Effect shape and is wrong here for two reasons. A failure ends the +subscription for every watcher, so one refused message took the whole feed down +with it, and the caller lost every later round for a session that was still +running them. And the PubSub replays, so subscribing again replayed the failure +and died again: after one refused round, `continued` was unreachable for the +life of the session. A test asking only "is a later round still seen after an +earlier one failed" fails against that design, which is how it was found. + +The type is a union rather than an optional field, so the compiler makes every +caller decide what to do about a refused round. `'failed' in one` narrows it. + +### A session names its tools; the registry defines them + +The `ToolRegistry` plugin holds every tool the harness has. A session names the +ones it may use, as strings, and `openSession` resolves them at open. A name +nothing holds fails with `ToolMissingError` rather than opening a session that +would send the model a tool it cannot run — and the model would call it. + +The names are frozen for the life of the session, like the system prompt and the +model, and for the same reason: the definitions are rendered into the prefix. +Adding a tool mid-session throws the cache away. + +One question is one loop. The model answers by asking for tools, the tools +answer, the model is asked again, and only when it stops asking does any of it +reach the store. That last part is the rule: every shape refuses a call whose +result is missing, so a store holding half a round holds a session nobody can +continue. `exchange.ts` collects the turns as they are made; `commit` writes the +question and all of them in one append. + +The loop ends three ways — the model stops asking, the round ceiling is reached +(`maxRounds`, 24 by default), or the last request filled enough of the window +that the next would be refused. The last two end with one more request offering +no tools at all, so the model has to answer in words. An exchange that stopped +on a tool result would leave the transcript ending on something the model never +replied to, and no shape takes that back. + +Nothing a tool does fails the question. A tool that throws, a name the session +does not offer, arguments that are not JSON: each is a failed result handed +back, because the model is the only party that can decide what to do about it. +The words are the failure's own cause and never `Cause.pretty` — a stack trace +in a tool result is paid for on every request of the session from then on. + +The calls of one turn run at once, because the model asks for several when they +are independent. The session serialises nothing and offers no way to ask it to: +**a tool that must not be re-entered holds its own permit**, beside the thing it +is protecting. `questionTool` and `todoTool` both do, in four lines each. + +### A session is independent of every other + +Nothing in the core is shared between two sessions. Not the transcript, the +counts, the busy flag, the running calls, the queue, or the scope — `wiringFor` +builds every one of them per session. A subagent is a session, so this is the +line that says a subagent cannot reach into its parent. + +It took two tries to get here, and both wrong turns are worth knowing. + +There was a `concurrent: false` flag on `Tool`, and the runner took a permit +before calling `run`. The permit was made per session, by `locksFor` inside +`wiringFor`. That locked nothing: a permit is only a lock against whoever holds +the same permit, and two sessions held two. A harness that built +`questionTool(ask)` once and gave one registry to a parent and its subagents got +two dialogs on one person at the same moment — the one thing the flag existed to +stop. + +The first fix kept the flag and moved the permit to the tool object, in a +`WeakMap` in the core. It worked, and it was still wrong: the core was inventing +an identity for a thing it does not own, in order to protect a thing it cannot +see. It also bought a deadlock that had not existed, because a call could now +wait on a permit held in another session. + +What actually needs protecting is a terminal, a file, a person. All of them are +the caller's, and all of them arrive with the tool the caller wrote to touch +them. So the caller holds the permit, in the tool, next to the thing. The flag +is gone, `locksFor` and `lockFor` are gone, and the core has nothing left to say +about two calls overlapping. + +The rule this leaves is worth more than the flag was: **if two sessions can +observe each other, something is in the wrong place.** + +### Every call can outlive the request + +Every call is forked and run under a deadline — `inlineFor` on the tool, else on +the session, else 30 seconds. When the deadline passes, the model is told the +call is still running and carries on; the work keeps going in the session's own +scope; and what it eventually says goes on a queue that `background.ts` drains. + +That is why any tool at all can be backgrounded: the harness decides when to +stop waiting, not the tool. A tool that always outlives a request says +`inlineFor: 0`, which is read rather than timed — a zero-length deadline raced +against the work is a race the work usually wins. + +When the answer lands it joins the same line a queued message joins, and the +session asks the model about it without anybody having asked a question. A +caller watches through `session.continued`. It is not "wait for the next +question": a build that finishes, or a person who answers ten minutes later, is +work to do at that moment and not at whatever moment somebody next types. The +rounds happen whether or not anybody reads the stream. + +The answer goes back as a turn the conversation says, never as a second tool +result: the call it belongs to was already answered, and every shape refuses a +second result for one call. + +A result that lands while a question is still streaming waits for it, because +one session still does one thing at a time. The driver retries while the session +is busy, for up to five minutes, and a session busy longer than that surfaces on +`session.continued` rather than spinning forever. + +### Four parties decide how long the model waits + +`waiting.ts` holds all of it, and the most specific answer wins. The tool says +whether the model waits at all with `Tool.wait`, and how long the waiting lasts +with `inlineFor`. The session names a fallback deadline. The model answers `wait` +on the call. And whoever is watching a running call can end the waiting now. + +`Tool.wait` and `inlineFor` are two questions, not one. The first is whether +there is any waiting; the second is how long it lasts once there is. A tool that +says neither has its default read from the deadline, because a tool nobody waits +any time for is a tool nobody waits for — that is `waitsFor` in `tool.ts`, and +it is what reaches the model as the schema's `default`. + +The two shipped tools answer opposite ways on purpose. `question` says true: a +model asks in order to find something out, and the answer is what it is waiting +for. `subagent` says false: handing a task over is how a model carries on, and a +parent that sat on it would have paid for a subagent and got a subroutine. Both +are `options.wait ?? …`, so a harness that knows its own people or its own +subagents can say otherwise. + +The model's answer is honoured in both directions, which was a decision. The +argument against honouring `wait: true` over a tool's `inlineFor: 0` is that it +holds something open — and it does not. Tools run in `afterRound`, between +requests, so nothing is open at the provider while a tool runs. What waiting +spends is the caller's `ask` stream and the session lock, and the caller already +has `session.background` to take those back. A tool cannot know which call the +model is stuck on; the model can. + +The field is added to every offered tool by `asOffered` in `tool.ts`, and taken +back off by `wanted` in `waiting.ts` before the tool sees the call. Tool authors +never write it and never receive it — a tool that validates its arguments +strictly would refuse a key its author never wrote. Arguments that are not a +JSON object pass through untouched, because a malformed call is the tool's to +complain about and a rewritten one changes the words of the complaint. + +### The deadline can be brought forward + +`Tool.inlineFor` and the session's own limit are guesses made before a call +starts. `session.background(callId)` is the same decision made by somebody who +can see how long it has taken, and `session.running` is what they read to +decide. Nothing is cancelled: the call keeps running in the session's scope and +answers in a round of its own, down the path the deadline already took. + +One call serves a person and an agent. A key press and a policy in the harness's +own code say the same thing to the session, and there is no second surface for +the second one: which of them decided is the caller's business. + +It is one `Deferred` per waiting call, raced against the deadline. The call is +in `wiring.running` from the moment it starts until the model stops waiting for +it — answered, timed out, or sent away — so `background` on anything else +answers false rather than failing. Pressing twice answers false the second time, +which is a person racing their own hand and not an error. + +### A subagent is a session, not a new mechanism + +`subagentTool` calls `openSession` from inside a tool. That is the whole +implementation, and it is the answer to what the architecture makes cheap: a +subagent needs no new seam, no nesting in the session, and no change to the +loop. + +It takes the layers it runs under, because `Tool.run` is handed no context. What +crosses between parent and subagent was the only real decision: + +- **One string goes up**, and not what the subagent said on the way to its own + tools. A model narrates before it calls something, and handing that up would + put back the noise a subagent exists to absorb. The answer is what it said + after its last call. +- **The counts do not go up.** They belong to the session that spent them. + `onFinished` hands them to a caller that is adding up a conversation, which is + the one thing that cannot be recovered afterwards. +- **The store does cross**, and on purpose. A session reads `SessionStore` from + the context it runs in, through `Effect.serviceOption`, which puts no + requirement in the type. A tool runs inside the parent's context, so the + subagent writes to the same database under a session of its own: one database, + two transcripts. Pass layers with a store of their own to separate even that. +- **Depth is the harness's decision.** A subagent offered the tool that started + it can start one of its own. Nothing here stops that, because nothing here + knows what the harness is for. + +### The tool no harness can write for itself + +`questionTool` is in `plugins/`, not `core/`. +Everything about the question is the model's — how many, what each says, what +may be picked, one answer or several, whether it may be skipped. Everything +about the asking is the caller's, in one function. The package holds the middle: +the shape of a question, the shape of an answer, and the words the model reads. + +It holds one permit, so two rounds of questions queue rather than arriving on +one person at once. A caller writing an asker that owns the terminal, or one +dialog, therefore needs no lock of its own — and gets that guarantee across +sessions, because the permit sits with the asker rather than with whoever is +calling it. + +It renders the answers by walking the questions, not the answers: a caller who +answers two of three is reported as answering two of three, and an answer to a +question nobody asked is dropped rather than shown as one the model wrote. + +### One tool version, measured across every lab + +A tool ships one description for everybody. There is no per-model branch and +there must not be one: eleven descriptions cannot be kept honest, and a model +this package has never seen has to work anyway. So a description is tuned by +measuring it across labs and changing the one text. + +`pnpm test:e2e:tool-matrix` is that measurement. It offers each model one +shipped tool at a time and scores what the model chose, never what it said: +whether it called the tool, whether the payload matched the schema, whether it +put several questions in one call, and whether it waited. Waiting is read out +of the event stream rather than out of the arguments — a call the model did not +wait for gets the still-running note — so the score is what the harness actually +did. + +The models are the sweep's own list — see "One model by default, eleven when you +say so" — so `pnpm test:e2e:tool-matrix full` is the measurement below. + +Measured on 2026-09-04, eleven models, every shipped tool: + +| | question | subagent | time | todo | +|---|---:|---:|---:|---:| +| Called the tool | 11 of 11 | 11 of 11 | 11 of 11 | 11 of 11 | +| Sent a payload the schema accepted | 11 of 11 | 11 of 11 | — | 11 of 11 | +| Put every part in one call | 11 of 11 | — | — | 11 of 11 | +| Waited | 11 of 11 | 8 of 11 | — | — | + +`time` takes no arguments and reads nothing off the call, so there is no payload +to be wrong and nothing to wait on. What its column asks is whether a model +notices its own answer would be stale: asked how many days are left in the +month, without being told to check, all eleven called it. + +### A model that says it has no tools + +One model does not always see the tools it was sent. Measured on 2026-09-05, +`minimax/minimax-m3` answers "I don't have access to a time tool" about one +round in eight and then gives a date from around its training cutoff, with the +tool plainly in the request. What was tried, on `chat_completions`: + +| | Called | +|---|---:| +| A one-line description: "Answers with the current date and time." | 2 of 6 | +| The shipped description | 7 of 8 | +| The shipped description, plus "never say you are unable to check the time" | 7 of 8 | +| The shipped description, with `tool_choice: "required"` | 3 of 6 | + +The first row is why the description is worth its length. The last is the +finding: the relay does not honour `tool_choice` for this model, so there is no +switch to throw. Hand-written requests carrying no part of this package miss at +the same rate, so it is not an encoding to fix here. + +`pnpm test:e2e:time` therefore counts a miss and asserts the floor underneath +it: every shape carries a tool that takes nothing, and a model that called the +tool answers with what the tool wrote. A model that calls on none of the three +shapes still fails the run — that is the description failing rather than one bad +round. Asserting on every call would put the suite red on a different model +every sweep for something nothing here can change. + +The same sweep caught the other way to measure the wrong thing: `xiaomi/mimo-v2.5` +refused two shapes outright on one run and carried all three on the next, which +is a relay having a bad minute. A failed round is now tried once more before it +counts. + +There is no right answer to the waiting column, which is why `Tool.wait` is a +default and not a rule. Both scenarios block — a deployment nobody has answered +about, a codename the model cannot know — so waiting is correct in both, and the +two tools ship opposite defaults. What the column measures is whether the model +reads the field: `question` defaults to waiting and every model kept it, +`subagent` defaults to not and seven of eleven overrode it. A model that never +overrode would be a model ignoring the field. + +The question, time and todo tools were clean on their first run and were not +touched. `todo` was asked to start a job of three named parts, without the words +"list" or "steps": every model wrote down three or more. + +One row of that run scored a provider hiccup rather than a model. `glm-5.3-flash` +answered nothing at all — no tool call and no text — which is not a judgement and +not something a description can fix, and it answered normally on every run +since. `tried` now retries an empty answer on the other shape and keeps a second +one, because twice is a finding. That is the second time this run measured the +wrong thing, and both are recorded here so the third is caught in review. The subagent +description was not, and what it cost is the point of keeping this run: + +| Description | Delegated | +|---|---:| +| Two uses named: several steps, or more reading than you need | 9 of 11 | +| Uses opened up, led by "anything you cannot answer yourself" | 10 of 11 | +| Plus what a subagent is: "it starts from instructions of its own" | 11 of 11 | + +Both failures said the same thing in their own words — `tencent/hy3` answered +that "none of my available tools can look that up", and `minimax/minimax-m3` +asked the person for a source. Neither was wrong given the first version: a +description that names two uses reads as a description that allows two, and a +one-shot lookup is neither of them. + +One earlier run measured the prompt instead of the tool, and is worth +remembering before writing another scenario. Asked for "this quarter's +codename", four models answered by asking which project was meant. That is the +right move on a question that names none — a model that will not guess is doing +its job — and it scored as a tool failure. Naming the release left one reason +not to delegate, which is the description, which is the thing under test. + +### The two tools that only a live run can render + +`pnpm test:e2e:time` and `pnpm test:e2e:todo` are the dedicated runs for the two +tools the package writes itself. Each puts its tool on all three shapes, because +what a unit test cannot settle is what a provider does with the schema: + +- **time takes nothing.** Its parameters are an object with no properties, which + is the schema a provider is likeliest to refuse. A shape that rejected it + would refuse the whole round, not one call. The run also proves the model asks + rather than guessing: every model was trained before today, and the answer is + checked against this machine's clock at the moment of the check, so the run + cannot rot. +- **todo carries the richest schema here** — an array of objects with an + enumerated field — and the tool replaces the list rather than patching it. The + run dictates three steps, marks one done a turn later, and checks that all + three came back both times: a model that sent only the step it changed would + leave a list of one, which is the failure the run is for. + +Both print what the model chose and assert only what the package promises. +Whether a model moves a step to `doing` or straight to `done` is its own +discipline, and `pnpm test:e2e:tool-matrix` is the run that scores that. + +### One conversation, with everything in it at once + +`pnpm test:e2e:conversation` is the run that puts the package together. Every +other live run proves one thing on its own; a harness does all of it in one +session, and the defects that only show up there have no other test. One person +and one agent work through a release: the time tool, a plan written down with +the todo tool, two questions asked of the person in one call and answered slower +than the model waits, a message typed while the session is busy, a subagent sent +to look something up, then the session reopened from SQLite, cloned, and +compacted — and asked at the end for a fact planted in its first turn. + +It asserts correctness first, then performance: the median time to the first +word, the median whole answer, the share of the prompt read from the cache, and +the wall clock. The ceilings are generous on purpose. They catch a change that +makes this package slow, not a provider having a bad minute. + +Two things it taught, both about writing the scenario rather than about the +package: + +- **An open-ended task measures the model's imagination.** "Migrate my project + from npm to pnpm" had models asking for filesystem access, spawning a second + subagent to go looking, and burying the planted fact under a page of + scaffolding the summariser then dropped. Naming what to do leaves the tools as + the only thing under test. +- **A session busy with a round of its own refuses `ask`,** which is the package + doing exactly what it says. The run waits and asks again, because that is what + the harness has to do with the refusal. A run that treated it as a failure + would be testing its own impatience. + +Two more, about what a live run may assert: + +- **Wait for the session to go quiet, never for a number of rounds.** How many + rounds a conversation runs is the model's to decide — one that waits for its + subagent answers inline and runs none — and waiting for three held every other + model at a 180-second deadline for nothing. The sweep went from 210 seconds a + model to 40. +- **Read the subagent's answer out of the parent's transcript, not out of its + words.** Carrying the answer back into the conversation is what the package + promises. Repeating it to the person is the model's own manner, and three of + the eleven did not. + +Eleven models, `pnpm test:e2e:conversation full`, 2026-09-05, 96 seconds in all: + +``` +model turns todo asked rounds sub kept first whole ratio total +anthropic/claude-haiku-4.5 24 3 2 2 1 yes 3316ms 3594ms 0.981 38s +openai/gpt-5.6-luna 22 3 2 1 1 yes 3283ms 3629ms 1.000 42s +z-ai/glm-5.3-flash 26 3 2 3 1 yes 1324ms 2653ms 0.692 30s +deepseek/deepseek-v4-flash-0731 26 3 2 3 1 yes 2856ms 2965ms 0.991 36s +qwen/qwen3.8-flash 24 3 2 2 1 yes 5080ms 5285ms 0.988 69s +xiaomi/mimo-v2.5 26 3 2 2 1 yes 2192ms 3457ms 0.975 59s +tencent/hy3 24 3 2 2 1 yes 4533ms 4732ms 0.988 62s +deepseek/deepseek-v4-flash 24 3 2 2 1 yes 2814ms 2902ms 0.992 42s +minimax/minimax-m3 24 3 2 2 1 yes 2661ms 2697ms 0.996 33s +nvidia/nemotron-3.5-lightning 26 3 2 2 1 yes 1864ms 1925ms 0.843 27s +google/gemini-3.7-flash 26 3 2 3 1 yes 4782ms 4968ms 0.736 52s +``` + +Every model wrote the plan down as three steps, asked the person both questions +in one call, delegated the lookup, and answered from the conversation after it +had been through SQLite and a summary. The first word takes 1.3 to 5.1 seconds +and the whole answer 1.9 to 5.3, on a conversation whose prompt is read from +cache 0.69 to 1.00 of the time. Nothing here is the package's own cost: the +per-token work is measured in `pnpm test:perf`, and it is microseconds. + +### The session does not write to the store + +`appendTurn` is a pure function. It does not touch the `SessionStore` plugin. +Do not make it write through. + +- A pure append has no error channel, so no caller inherits a store failure. +- One step is one transaction, not one transaction per turn. +- A session runs with no store at all. + +The cost is that a crash drops whatever the store still holds. That is the +plugin's call: it hears every turn and the close, and decides whether to write +at once, to batch, or to buffer. + +### The store is one shared implementation and a one-function driver + +`src/plugins/store/sqlite.ts` holds every query. The seam is `driver.ts`: one +function — run this SQL with these parameters, give back the rows by position — +so `node.ts` and `expo.ts` are about twenty lines each and share all of it. +`rows.ts` says what a row means and asserts it, and `migrate.ts` applies the +migrations the bundle carries. + +The schema is written in `schema.ts` and the SQL is generated from it, so a +migration and a query can never disagree about a column. Run `pnpm migrations` +after editing the schema: it calls drizzle-kit and then inlines the SQL into +`src/plugins/store/migrations.ts`. + +`pnpm check:migrations` regenerates them and fails when anything moves, so the +schema, the SQL and the inlined copy can never drift apart unnoticed. It is part +of `pnpm check`. That check is the price of inlining, and it is why inlining is +allowed. `pnpm migrations` formats what it writes, so generating twice produces +the same bytes. + +**Migrations are inlined, never read from disk.** React Native has no filesystem +to read them from, and Drizzle's answer there is a Babel plugin, which a package +must not force on the people who install it. The applied version lives in +SQLite's own `user_version`, so the store needs no table to know where it +stands, and the whole set is applied in one transaction. + +**A Drizzle type states what the schema declares, not what the file on disk +holds.** Every row is asserted with typia on the way out. A database written by +an older build, or by another program, still arrives as `unknown`. + +### Three tables, and a read with no join + +| Table | Holds | +|---|---| +| `sessions` | What `SessionOptions` freezes: system, model, effort, maxTokens, and `prompted`, the count of the last request | +| `turns` | The identifier, the session, and the role. No content of its own | +| `parts` | One row per piece of a turn: text, reasoning, or an image | + +`parts.session_id` repeats what `turn_id` could reach. It is there for the +reader: loading a session is two indexed scans over two tables and no join at +all, and the parts are matched onto their turns in one pass in memory. Both +indexes cover `(session_id, id)`, so each read is a range over one index with no +sort, because a ULID already carries the order. + +A turn and its parts are written in one transaction. A turn whose parts went +missing would read back as an empty message and quietly shorten the prompt. + +**Every write of one connection stands in one line.** A session and its subagent +share a connection by design, and every adapter is async, so each `await` inside +a transaction lets the other one's statement in. SQLite cannot start a +transaction inside a transaction: the second `BEGIN` throws, its `ROLLBACK` +takes the first writer's rows with it, and the first then commits nothing. So +`driver.ts` pairs a driver with the line its writes queue in, and `transact` +holds that line for the whole unit. Reads stay out of it — a session holds +itself while it writes, so no read asks for the rows being written. + +**An image is stored as base64, not as a blob.** Base64 is what every gateway +shape wants on the wire, so storing it that way costs a third more space and +saves encoding the image again on every single request. The read path is the +prompt builder, and it is the path that matters. + +`sessions` records the options because a continued session must be reopened with +them. Resuming under a system prompt that differs by one byte drops the whole +cached prefix, and the only symptom is the bill. + +### Reasoning goes back exactly as it came + +A reasoning model's thinking arrives as its own stream event and is stored as +its own part, ahead of what the model then said. It is then **sent back +unchanged** with every later request. + +Do not strip it. The API drops what the target model cannot read and does not +bill for it, so there are no input tokens to save, and removing a block by hand +can fail the request on ordering or on its signature. Hand back what the +provider gave and let the provider decide. + +**The signature is the part that matters.** A provider signs the thinking and +refuses a block whose signature it cannot read, so `TurnPart` carries an opaque +`signature` and the `parts` table has a column for it. The signature streams on +its own event, after the thinking and with no text on it. A reasoning part with +no signature is left out of the prompt: it would only be refused. + +**A `redacted_thinking` block is thinking the provider encrypted rather than +showed.** It arrives whole at the start of the block, carries no signature and +no words, and has a part kind of its own so that nothing renders its bytes as +text. It goes back byte for byte. Only the Anthropic shape produces one, and +only for flagged content, so no live run has ever seen one. Anthropic documents +no string that triggers one either — checked on 2026-09-04 against the thinking +and extended thinking pages — so do not spend time looking for a live case. + +**The thinking blocks go back in the order the model made them.** Anthropic's +thinking page: within the latest assistant message the sequence of consecutive +thinking blocks must match what the model generated, and it may not be +rearranged, edited, or partly dropped. An encrypted block counts as one of +them, so a turn whose reasoning was redacted part way through is thinking, then +the encrypted block, then more thinking. `Spoken` therefore holds one ordered +list and not a string of words beside a list of encrypted blocks — the two +fields could not say which came first, and this package emitted every encrypted +block ahead of every word until 2026-09-04. + +**An empty thinking block is still a block.** A provider returns thinking as a +summary and defaults to no summary at all, so `thinking` is `''` while the model +thought and was billed. The part is kept whenever there is a signature, never on +whether there are words. Dropping empty ones would drop every block on the +default setting. + +**Each shape seals the thinking its own way, and the seal is opaque.** +`signature` holds whatever the shape needs to hand the thinking back, and only +that shape knows what is in it: + +| Shape | What it seals with | Replays | +|---|---|---| +| `messages` | the signature Anthropic issues with the block | yes | +| `responses` | `{id, encrypted_content}` of the reasoning item, as JSON | yes | +| `chat_completions` | nothing it will take back | no | + +A session's model is frozen and its shape follows from the model, so a seal +made by one shape is never read by another. + +The responses shape does not carry thinking inside a message. It is an item +beside the message, holding the provider's own encrypted copy, and the request +has to ask for it with `include: ['reasoning.encrypted_content']`. The summary +is replayed empty, which is how the item arrives: a summary this package wrote +instead of the provider would be a change to what the provider sealed. + +**The gateway sends the reasoning under different names again.** Measured +2026-09-04: the responses shape relays Anthropic's thinking as +`response.reasoning.delta`, not the documented `response.reasoning_summary_text +.delta`, so both are read. Two providers relayed through the chat shape name it +`reasoning` and `reasoning_content`, so both are read there. + +A turn holds as many reasoning parts as the model produced, and an encrypted +block closes the one open at the time. What still merges is two signed blocks +with nothing between them: a signature does not close a block here, so both +land in one part with the second signature. A model produces those between tool +calls, which this package does not have. Split on the signature when it does. + +Measured with `pnpm test:e2e:probe responses anthropic/claude-sonnet-4.5` on +2026-09-04: one reasoning item and one message in a turn, 43 +`response.reasoning.delta` frames and a single `response.output_item.done` +carrying the reasoning. So the merge loses nothing today, and the probe is how +to check that again. + +A thinking block is closed by its seal, not by the next event. A model produces +two signed blocks in a row between tool calls, and merging them would hand the +provider one block under the other's seal. + +The gateway then ends a block with a reasoning event carrying no words and no +signature, after the signature has already arrived. That event opens no block: +one opened on it would sit unsigned behind the signed one, the wire drops what +it cannot sign, and the thinking would go back with a hole in it. Only the live +run says so, which is why `pnpm test:e2e:reasoning` asserts that every stored +block carries a seal rather than that there is exactly one. + +### A provider may fail after the answer has started + +All three shapes may report a failure in the middle of a stream that they would +have reported as a status had the call not been streamed. Anthropic's streaming +reference names the case: an `overloaded_error` frame, which is a 529 on a call +that is not streamed. Two of the three mark it with an `error` object on the +frame. The responses shape marks it one level down, as `response.error` on a +`response.failed` frame, which OpenAI's reference gives a `code` and a +`message`. `isFailure` in `wire/wire.ts` reads both places, once, rather than +three times. + +The stream fails with `ModelError` and `reason: 'stream'`. It is a reason of +its own because the caller already holds part of an answer and has to throw it +away; no other reason leaves anything behind. The exchange is never written, so +the fragment reaches the caller and nothing else. Before 2026-09-04 the frame +matched no reader, was dropped as an unknown event, and the stream ended on +`done` with a fragment stored as a whole answer. + +Reading a field this way risks calling a good frame a failure, so it was +checked against real traffic: the whole live sweep, the ten model matrix, and +`pnpm test:e2e:probe` on each of the three shapes, all on 2026-09-04. Not one +frame of a call that succeeded carried an `error` object in either place. The +responses shape does send the key on every reply that worked, as +`response.error: null`, and null is not an object, so it does not match. The +probe prints any frame that carries one, so that check takes one command. + +### Why the model stopped is part of the answer + +`done` carries a `StopReason` beside the counts: `end`, `maxTokens`, `refusal`, +`tools`, or `unknown`. Without it a caller cannot tell a finished answer from one the +ceiling cut off mid-sentence, and would store half a thought and build every +later request on it. + +The truncated turn is still kept. It is what was paid for, and dropping it would +shorten the prompt that follows. + +`unknown` is the honest answer for a name this package has not seen, and it is +never a guess. `tool_use` and `tool_calls` map to `tools`, which is what tells +`loop.ts` to run the calls and ask again. + +One shape names no reason for a call at all: `chat_completions` can end a +streamed answer with `finish_reason: "stop"` while the assistant message +carries `tool_calls`. So the gateway plugin keeps a flag for whether the model +asked for anything, and upgrades `end` to `tools` when it did — only `end`, so a +ceiling or a refusal still reports itself. + +It is also what a caller gets when the stream ended and no frame said why. All +three shapes served here always send one, on every live run recorded in this +file, so treat that case the way you treat `maxTokens`: the answer in hand may +not be the whole answer. Whether the package should fail instead of reporting +`unknown` there is open, and needs a live case before anyone decides. + +The three shapes report it in three places — `message_delta.delta.stop_reason`, +`response.incomplete.response.incomplete_details.reason`, and +`choices[].finish_reason` — so `pnpm test:e2e:stop` asks each shape twice, once +with room to finish and once with a ceiling of 24 tokens. + +`maxTokens` covers two walls on the Anthropic shape. `max_tokens` is the +caller's ceiling and `model_context_window_exceeded` is the model's window, and +the provider's own guidance for the second is to treat it as truncated. Both +leave half a sentence, which is the whole of what the reason has to tell a +caller. The full list, read on 2026-09-04: `end_turn`, `max_tokens`, +`stop_sequence`, `tool_use`, `pause_turn`, `refusal`, and +`model_context_window_exceeded`. `pause_turn` maps to `unknown`: it waits on a +server-side tool this package does not run, and no live run has produced one. + +### A dropped stream stops the call + +Every call carries an abort signal, and the handle is scoped to the stream +rather than to the request. A streamed call resolves as soon as the headers +arrive and keeps producing for a long time after, so a handle released when the +request resolved would cancel nothing and the provider would keep charging. + +`AbortController` is read off the global, the way the entropy plugin reads +`crypto`, because it is a global in every runtime that has `fetch` and this +package already asks the caller for a `fetch`. A runtime without one still +works and simply cannot stop a call early. + +The package declares only the part of a signal it hands on, so an adapter names +the type its own runtime has. `src/plugins/fetch/web.ts` shows how the package +itself avoids the cast: it declares the members it uses and passes the signal +through as `unknown`. + +### An exchange is written whole, or not at all + +The question is added to the session in memory when it is asked, because the +prompt needs it. The **store** hears about the question and everything it +produced together, in one call, when the loop ends — the answer, and every tool +call and result of every round on the way to it. A call stored without its +result is a session nobody can continue. + +If no answer arrives — the caller walked away, the transport failed, the store +refused the write — the question is taken back out again. A transcript that ends +on an unanswered question sends it again with every later request: the caller +pays for it each time, and the model may answer it late, on top of whatever was +asked next. Seen live before the fix, after a cancelled question: + + asked again "ok\n\n1\n2\n3\n4\n5\n6\n7" + +Rolling back is a `Ref.set` to the session as it stood, which is safe because +one session answers one question at a time and nothing else can have touched it. +`StoredExchange.turns` is a list for the same reason: two appends would leave a +question committed without its answer if the second failed. + +### A session that fills the window summarises itself + +A session grows until the model refuses the request. Compaction is the answer, +and the shape of it is not a preference: + +**Summarise everything into one message and replay nothing before it.** Keeping +the recent turns verbatim and summarising only the old ones looks better and is +refused: a thinking block is signed against the whole history that stood when it +was produced, so a retained turn replayed after a summary fails on its +signature. Nothing carried over here is tied to the old transcript. + +The summary is a turn with a `summary` part. `sinceSummary` is the only rule: +a prompt starts at the last turn that holds one. The earlier turns stay in +memory and stay in the store — they are the record of what happened, and only +the prompt starts after them, so a continued session lands in the same place. + +**The trigger is the provider's own count.** After each call the session records +what that request put in front of the model, cached tokens included, and +compacts before the next question when it passes `compactAt` (0.8) of the +catalog's `contextWindow`. The count is stored with the session, so a resumed +one is measured too. No tokeniser, and no estimate that can drift. A +catalog that names no window never compacts: a guessed window would either cut a +conversation that fit, or fail to save one that did not. + +Compaction throws the model cache away, because every byte of the prefix +changes. That is the price of the session continuing at all. `session.compact` +forces one, for a caller that knows sooner than the number does. + +The summariser is told what to keep. Left to its own judgement it writes a +readable paragraph and drops the identifiers, and the summary is all the model +will have of that work. + +The summary is streamed and folded, like every other call. `ModelClient` had a +second method that fetched a whole reply, and compaction was its only caller: +one parser per shape that nothing else exercised, free to disagree with the one +that mattered. It did, and this is the bug below. It was deleted on 2026-09-04 +along with the three `toReply` readers and `ModelRequest.stream`, which the +gateway overwrote on every call anyway. + +**The summary call is counted.** It is a call to the model, it is billed, and +until 2026-09-04 its tokens went nowhere: `session.usage` under-reported every +session that had ever compacted, by the whole cost of every summary. It sets +`prompted` to zero rather than adding to it, which is deliberate — the next +request starts from the summary, so what the last one cost says nothing about +what the next one will. + +**The summary call carries the session key and the session effort**, the way +every other call of the session does. The gateway reads `cacheKey` as the +session, so a summary sent without it routes on its own and pays full price for +a prefix the session already has cached — and this is the one call that resends +everything before it. `effort` is part of that key, so leaving it off would miss +the entry even with the key. + +### The two model SDKs are types, not code + +`openai` and `@anthropic-ai/sdk` are the contract the three wires are written +against, and every import of either is a type. Nothing of either survives the +build, so both are dev dependencies and a consumer installs neither: 18.8 MB +unpacked between them, against 4 runtime dependencies that are all used. + +That holds only while no published declaration names one. Exporting a type +built out of one — `ContentBlock`, say, which two wires exported and neither +read — puts the import back into a `.d.ts`, and the consumer's own typecheck +then fails on a package nobody told them to add. The compiler here cannot see +it, because here they are installed, so `pnpm check:platform` reads the built +declarations and fails on either name. + +The store plugins go two ways. `plugins/store/expo` names no package at all: it +asks for the two methods it calls, so an Expo database satisfies it +structurally, the caller's own compiler checks the real type at the call, and +the package needs no dependency on `expo-sqlite` — not even a peer one. +`plugins/store/node` names `node:sqlite`, which needs Node 22.13 or newer to +import without a flag. + +### The count is stored beside the session + +`prompted` is the provider's own count of the last request, and nothing here +estimates one. So it is written down: a `prompted` column on `sessions`, filled +by `append`, and read back by `continueSession`. A session reopened onto a +conversation that already fills the window compacts before it asks anything. + +`pnpm test:e2e:resume` proves it live: the count the provider reported came back +out of SQLite unchanged, and the same session reopened under a window that count +fills compacted before its first question, while under a window ten times larger +it did not. + +**A clone is measured, not asserted.** `pnpm test:e2e:clone` copies a session +whose prefix is 11.8k tokens and asks the copy one question: 11848 read from the +cache and 0 written, on 2026-09-04. A clone that copied an identifier into the +prompt, or reordered one part, would pass every unit test here and double the +bill, because the only symptom is the cache write. + +Until 2026-09-04 it was not stored. A reopened session started at zero, `isFull` +was false, and its first question went out with the whole stored conversation in +front of it. That closed itself after one answer — unless that first question was +the one that would not fit. Then it was refused, `finish` never ran, the count +stayed zero, and every retry was identical: the session could not compact itself +out of it. + +`append` therefore takes a `StoredExchange`, not a list of turns: the session, +its new turns, and the count that goes with them. They are one write because +they describe one request. A store that kept the turns and lost the count would +hand back a session that does not know how full it is; one that kept the count +and lost the turns would hand back a session that thinks it is fuller than it +is. The SQLite plugin does both inside the same transaction. + +A compaction records zero, in the store as well as in memory. The next request +starts from the summary, so what the last one cost says nothing about what the +next one will. `resume.test.ts` pins both directions: a session whose stored +count fills the window compacts before its first question, and a session +reopened after a compaction does not compact again. + +An older database has no column and reads back as absent, which is treated as +zero — the behaviour it had before, for the one session that was mid-flight +when the migration ran. + +### A reloaded turn must equal the turn that was written + +The prompt prefix is rebuilt from the store. If `load` returns a turn that +differs from the written turn in one byte or in the order, the prefix changes +and the model cache misses. The SQLite plugin must prove the round trip with a +local end-to-end run. + +### A plugin point that is easy to break ships its check + +`SessionStore` and `PromptAssembler` hold invariants no type states. A store +that reorders turns, drops a reasoning signature, or writes the turns without +the count typechecks and answers every call; an assembler that rewrites an +earlier message typechecks too. Neither shows up as an error. Both show up as a +bill, because the prompt prefix moves and every question after it is written to +the cache again. + +So `core/conformance.ts` ships `checkStore` and `checkAssembler`. Each answers +`readonly string[]`, and each finding names what is wrong and what it costs. +**Neither fails.** A store that refuses a write is a finding: a caller running +these in their own test runner wants one list, not an exception to catch. +`checkStore` writes under identifiers of its own, so it is safe against a real +database. + +The checks are held to the same rule as a guard: `plugins/conformance.test.ts` +runs both against the shipped plugins, and then breaks a plugin seven ways — +reversed load, a dropped signature, foreign turns, a stale count, a refused +append, a drifting assembler, a rewriting assembler — and asserts each one is +caught. Both checks found a real defect on their first run against the code +that was already shipped, one in the check and one in the docblock it was +reading. + +### There is no `layerModelClient` helper + +Making plugin authoring easy was read once as a set of constructor aliases — +`layerModelClient(impl)` for each point. `Layer.succeed(ModelClient, impl)` is +already one line and is the idiom every other layer in this package uses. An +alias per point would be eight more exports, eight more names to keep in step, +and not one keystroke saved. The lever that actually shortens the work is +saying what the invariants are and shipping the checks: `PLUGINS.md` and +`conformance.ts`. + +## What is not pluggable, and why + +Two seams were cut after they were built, and one default was tried and could +not be written. The two failed the same test: name the second implementation, +and say whether a caller should be allowed to write it. + +**The identifier ordering.** An identifier must sort by the order it was made +in, because a store rebuilds the prompt prefix in that order. A plugin +returning a random identifier typechecks, passes every test, and breaks the +cache one reload later. The ordering is not a choice; where the randomness +comes from is, and that is `EntropySource`. + +**The token ceiling.** It was the third of three ways to set one number and +fired only when a caller set neither of the other two. `ModelCatalog` already +knows a model's own limit, so `ask.ts` reads +`maxTokens ?? catalog maxOutputTokens ?? 4096`. + +## Layout + +`core/` holds the contracts and the pure domain. `plugins/` holds the +implementations, including the ones this package owns. **A file in `core/` must +never import from `plugins/`.** `pnpm check:boundaries` fails when one does. +It exempts `*.test.ts`: a core test needs a plugin to run against. + +| Path | Purpose | +|---|---| +| `src/index.ts` | The public entry point; what a caller uses, and every owned plugin | +| `src/core/index.ts` | The `/core` entry point. Every core module a plugin author reaches, no plugin. The five that only a session runs — `background`, `exchange`, `loop`, `tools`, `waiting` — are reached by path | +| `src/core/run.ts` | `openSession`: a new session | +| `src/core/resume.ts` | `continueSession` and `cloneSession`: one the store already holds | +| `src/core/wiring.ts` | What every session shares: the options, the handle, the bridge | +| `src/core/ask.ts` | One question and one answer: the guard, the ceiling, the stream | +| `src/core/queue.ts` | The line a queued message and a late tool result wait in | +| `src/core/exchange.ts` | What an exchange is, and the rule that it is written whole | +| `src/core/handle.ts` | `SessionHandle`: what a caller holds, and the driver behind it | +| `src/core/loop.ts` | The rounds one question makes, and the three ways they end | +| `src/core/tool.ts` | The `ToolRegistry` plugin point, and what a tool is | +| `src/core/tools.ts` | Running the calls of one turn, under a deadline they can outlive | +| `src/core/waiting.ts` | How long each call is waited for, and what the model is told when it is not | +| `src/core/compact.ts` | Summarising a session that has filled the window | +| `src/core/background.ts` | The driver: what the session says when nobody is streaming | +| `src/core/usage.ts` | Token counts and the cache hit ratio | +| `src/core/session.ts` | The session and its append-only turns | +| `src/core/turn.ts` | One turn and its parts: text, reasoning, or an image | +| `src/core/prompt.ts` | The `Prompt` shape and the `PromptAssembler` plugin point | +| `src/core/model.ts` | The `ModelClient` plugin point; transport only | +| `src/core/storage.ts` | The `SessionStore` plugin point | +| `src/core/id.ts` | `{prefix}_{ulid}`; the encoding, and the monotonic order | +| `src/core/entropy.ts` | The `EntropySource` plugin point; random bytes | +| `src/core/conformance.ts` | `checkStore` and `checkAssembler`: what a plugin author runs | +| `src/core/session-fixture.ts` | What the session tests share. Excluded from `dist/` | +| `src/core/resume-fixture.ts` | What the resume tests share. Excluded from `dist/` | +| `src/perf.perf.test.ts` | The timing gate; run by `pnpm test:perf`, not `pnpm test` | +| `src/core/catalog.ts` | The `ModelCatalog` plugin point; shapes and output limit | +| `src/core/token.ts` | The `TokenSource` plugin point; the credential per call | +| `src/core/retry.ts` | The `RetryPolicy` plugin point; an effect `Schedule` | +| `src/core/fetch.ts` | The smallest `fetch` a transport plugin needs | +| `src/plugins/fetch/web.ts` | `webFetch`: that seam filled for a runtime with a WHATWG `fetch` | +| `src/plugins/kilo.ts` | `layerKilo`: the five layers a session needs, in one call | +| `src/plugins/model/fake.ts` | A scripted model, for this package's tests. Excluded from `dist/` | +| `src/plugins/prompt/default.ts` | The assembler plugin | +| `src/plugins/catalog/table.ts` | A catalog the caller writes down | +| `src/plugins/entropy/web-crypto.ts` | The default source: the global `crypto` | +| `src/plugins/entropy/seeded.ts` | A repeatable source, for a test or a replay | +| `src/plugins/token/static.ts` | One token for the life of the process | +| `src/plugins/tools/question.ts` | The question tool, and the asker a caller writes | +| `src/plugins/tools/subagent.ts` | The subagent tool: `openSession` from inside a tool | +| `src/plugins/retry/backoff.ts` | Exponential backoff with jitter, and no-retry | +| `src/plugins/store/sqlite.ts` | Every query, written once for every platform | +| `src/plugins/store/driver.ts` | The one-function seam an adapter fills, the write line, and `transact` | +| `src/plugins/store/rows.ts` | What comes off the disk, and the assertions that check it | +| `src/plugins/store/migrate.ts` | Applying the migrations the bundle carries | +| `src/plugins/store/node.ts`, `expo.ts` | One adapter per platform | +| `src/plugins/gateway/` | The kilo gateway plugin | +| `src/plugins/conformance.test.ts` | The checks, against the shipped plugins and against seven broken ones | +| `README.md` | What a consumer reads: the example, the events, the plugin table | +| `PLUGINS.md` | What a plugin author reads: one worked example per point, and the invariants | +| `.oxlintrc.json` | The package lint config; stricter than the root config | +| `tsconfig.json` | The package compiler config. The repo has no root `tsconfig.json`; this one stands alone | + +Inside `src/plugins/gateway/`: + +| Path | Purpose | +|---|---| +| `index.ts` | The layer: send, stream, and the resolved plugins | +| `wires.ts` | Asks the catalog and picks the best wire for a model | +| `test-gateway.ts` | The gateway with test plugins, for the unit tests. Excluded from `dist/` | +| `http.ts` | The post, the headers, the retry, and the abort handle | +| `api-kind.ts` | The three shapes and which one to pick | +| `sse.ts` | A reader over `eventsource-parser` | +| `wire/` | One file per shape, plus the shared `Wire` | +| `fake.ts` | The fake `fetch` the gateway tests share. Excluded from `dist/` | + +`layerKilo` is the wiring almost every caller writes: the assembler, the +entropy source, the catalog, and the gateway with its token and retry policy +under it. A model it knows nothing about is assumed to speak all three shapes, +which is true of everything the gateway relays, so the smallest call names four +things: the URL, the org, a `fetch`, and a token. It exists because that wiring +has an order and a trap — the catalog must be one instance shared by the session +and the gateway, not two that agree — and because the package's own live runs +were copying twenty-five lines of it each. + +`token` takes a `TokenSourceService` as well as a string. That is the one plugin +a long-lived caller has to replace, because the kilo token expires, and +replacing it by hand means rebuilding the shared catalog — the trap this +function closes. One line here against twelve a caller would have copied. Every +other plugin is still replaced by composing the layers instead. + +There are nine entry points: `@kilocode/harness-sdk`, `/core`, +`/plugins/fetch`, `/plugins/gateway`, `/plugins/prompt`, `/plugins/tools`, +`/plugins/store/node`, `/plugins/store/expo` and `/testing`. The two stores have +subpaths of their own because each names a platform: exporting them from the +root would pull `node:sqlite` or `expo-sqlite` into every bundle. `/plugins/fetch` +and `/testing` have theirs because an entry point is what a consumer bundles and +neither runs in production — a caller with a `fetch` of their own should not +carry this one, and the conformance checks belong to a plugin author's test +suite. The catalog, token and retry plugins have none — a consumer reaches them +through the root barrel, which also pulls the gateway. Add a subpath when one of +them is wanted on its own. + +`scripts/check-package.ts` reads `package.json` and the README's own table, so a +tenth entry point that reaches neither this list nor that one fails the build. + +The root is narrower than `/core` on purpose. It re-exports whole only the +modules a caller uses whole, and names what it takes from the ones that hold +the machinery a session runs on: `wiringFor`, `makeId`, `sinceSummary`, +`onStore` and the rest are reached through `/core` instead. The tool contracts +are at the root — a caller writes tools — while `resolveTools`, `toolNamed`, +`definitionsOf` and `locksFor` are machinery and are not. + +`src/index.test.ts` asserts both halves, because a module left out of a barrel +is invisible from outside the package and every test here imports by path. It +has caught two unreachable features — compaction and the composed layer — and +it now also fails when a name from the machinery list reaches the root. + +`pnpm build` empties `dist/` first. It once did not, and a subpath whose source +had been deleted went on resolving against a stale artifact. + +## Recorded deviations + +Add a row when you turn one off, and give the reason. + +| Rule | Reason | +|---|---| +| `import/no-named-export` | This package is a library. A barrel needs named exports. | +| `import/consistent-type-specifier-style` | It deadlocks with `consistent-type-imports` and `no-duplicate-imports`. Inline `type` specifiers win. | +| `no-ternary` | A ternary is the normal form for a two-branch expression. | +| `sort-imports` | It sorts by member syntax, which no formatter keeps. | +| `id-length` | Effect names its type parameters `A`, `E`, and `R`. | +| `vitest/no-importing-vitest-globals` | An explicit import beats a global. | +| `vitest/prefer-to-be-truthy` | `toBe(true)` states the value; `toBeTruthy` does not. | +| `promise/prefer-await-to-then` | It flags `Promise.resolve`. | +| `require-await` | It flags an async generator, which needs no await. | +| `unicorn/no-array-callback-reference` | Effect pipes pass a function by name on every line. | +| `unicorn/no-array-method-this-argument` | It reads `Effect.map(a, b)` as an array method. | +| `func-names` | `Effect.gen` takes an anonymous generator. | +| `no-magic-numbers` | An HTTP status and a token count are not magic. | +| `max-classes-per-file` | A tag and its error belong in one file. | +| `sort-keys` | Field order carries meaning; alphabetical order does not. | +| `import/prefer-default-export` | It deadlocks with `import/no-default-export`. | +| `typescript/require-await` | The same false positive as `require-await`, on the TypeScript side. | +| `typescript/explicit-module-boundary-types` | Off for tests only. A test's helper reads better with an inferred return. | +| `import/max-dependencies` | Off for tests only. A test wires every plugin it exercises. | +| `unicorn/require-module-specifiers` | Off for the config files, which use bare re-exports. | +| `import/no-namespace` | Off for `src/index.test.ts` only. Asking what a barrel exports needs the namespace; there is no other way to read it. | + +The `**/*.test.ts` override also covers `**/*-fixture.ts`, and +`pnpm check:boundaries` exempts both. A fixture is test code: it may reach for a +plugin, and it may import more than ten things to wire one. + +`skipLibCheck` is on in `tsconfig.json`. `effect` and the two model SDKs ship +declarations this compiler rejects, and the package cannot fix them. It is the +one relaxed compiler flag, and it is the reason principle 12 cannot be enforced +by the type system alone. + +`new-cap` stays on with `Tag`, `GenericTag`, `TaggedError` and `TaggedClass` +as exceptions, because each is a call, not a constructor. + +`isolatedDeclarations` is off. It cannot infer a typia validator's type. + +**The `fetch` adapter was once recorded here as impossible, and it is not.** +The entry said `AbortLike` is deliberately not `AbortSignal`, so an adapter +needs one cast that only code holding the runtime's own type can make honestly, +and `no-unsafe-type-assertion` is on. That is true of a caller who has the DOM +types and false of this package, which does not: `src/plugins/fetch/web.ts` +declares the four members of `fetch`, `Response` and `TextDecoder` it uses, +passes the signal straight through as `unknown`, and needs no cast at all. +`lib: ["esnext"]` and `types: []` both still hold. The README keeps the +hand-written version below `webFetch`, for a runtime whose `fetch` does not +stream — React Native without a polyfill is the one that matters. + +`import/group-exports` stays on. Declare a name, then export it in one +`export type { ... }` block and one `export { ... }` block at the end of the +file. diff --git a/packages/harness-sdk/PLUGINS.md b/packages/harness-sdk/PLUGINS.md new file mode 100644 index 0000000000..28ad4fdbe2 --- /dev/null +++ b/packages/harness-sdk/PLUGINS.md @@ -0,0 +1,287 @@ +# Writing a plugin + +Every replaceable part of this package is a `Context.Tag` and a service +interface of two or three functions. Writing one is three steps: + +1. Write an object of the service's shape. +2. Wrap it: `Layer.succeed(TheTag, yourObject)`. +3. Merge it into the layers you already provide. + +That is the whole mechanism, and it is the same for all eight points. The rest +of this page is one worked example each, and the invariants that are not in the +types. + +```ts +import { Layer, Stream } from 'effect'; +import { ModelClient, zeroUsage, type ModelEvent } from '@kilocode/harness-sdk'; + +const layerEcho = Layer.succeed(ModelClient, { + stream: request => + Stream.fromIterable([ + { kind: 'delta', text: `you said ${String(request.prompt.messages.length)} things` }, + { kind: 'done', usage: zeroUsage, stop: 'end' }, + ]), +}); +``` + +Merge that layer in where you build the rest, and the session uses it instead of +the gateway. + +Nothing here needs a base class, a decorator, or a registration call. A plugin +is an object, and the layer is how it is handed over. + +## Check your plugin before you trust it + +Two of the eight are easy to write and easy to get silently wrong. A store that +reorders turns, drops a signature, or loses a column typechecks, answers every +call, and breaks the model cache one reload later. An assembler that rewrites an +earlier message typechecks too, and costs the whole prefix on every question +from then on. Neither shows up as an error; both show up as a bill. + +So the package ships the checks. Run one in whatever test runner you already +have and assert the answer is empty: + +```ts +import { PromptAssembler, SessionStore } from '@kilocode/harness-sdk'; +import { checkAssembler, checkStore } from '@kilocode/harness-sdk/testing'; + +const conforms = Effect.gen(function* () { + const store = yield* SessionStore; + const assembler = yield* PromptAssembler; + const wrongInStore = yield* checkStore(store); + const wrongInAssembler = checkAssembler(assembler); + return [...wrongInStore, ...wrongInAssembler]; +}); +``` + +Each answers a list of what it found, in words that say what is wrong and what +it costs. Empty means it conforms. Neither fails: a store that refuses a write +is a finding, not an exception to handle. + +They are at `/testing` and not in the main entry, because nobody runs them in +production and an entry point is what a consumer bundles. + +`checkStore` writes two sessions under identifiers of its own, so it is safe to +run against a real database. Run it against a fresh one for the clearest answer. + +## The eight points + +| Tag | Service | What it decides | +|---|---|---| +| `ModelClient` | `ModelClientService` | How a request leaves and a reply comes back | +| `SessionStore` | `SessionStoreService` | Where the conversation is kept | +| `PromptAssembler` | `PromptAssemblerService` | What the prompt looks like, and where the breakpoints go | +| `ModelCatalog` | `ModelCatalogService` | Which shapes a model speaks, its output limit, its window | +| `ToolRegistry` | `ToolRegistryService` | Every tool the harness has | +| `TokenSource` | `TokenSourceService` | The credential for one call | +| `RetryPolicy` | `RetryPolicyService` | What is tried again, and how often | +| `EntropySource` | `EntropySourceService` | Where random bytes come from | + +### ModelClient + +One function. It takes a `ModelRequest` and returns a stream of `ModelEvent`. +The session never decides how a request is sent, retried, or parsed. The +example above is a whole one. + +Every stream must end with exactly one `done`, carrying that call's counts and +why the model stopped. The session reads nothing else to close a turn. Emit +`toolCall` whole — the id, the name, and the complete arguments — rather than in +fragments; collecting the fragments is the transport's job. + +Fail with `ModelError`. A transport that throws instead of failing takes the +session down with it. + +### SessionStore + +Five functions. `create` records a new session, `read` gives it back, `append` +records one completed exchange, `load` gives the turns back oldest first, and +`flush` writes whatever is still held. + +```ts +import { Effect, Layer, Option } from 'effect'; +import { SessionStore, type StoredSession, type Turn } from '@kilocode/harness-sdk'; + +const layerMemory = Layer.sync(SessionStore, () => { + const sessions = new Map(); + const turns = new Map(); + return { + create: session => Effect.sync(() => void sessions.set(session.id, session)), + read: id => Effect.sync(() => Option.fromNullable(sessions.get(id))), + append: ({ sessionId, turns: added, prompted }) => + Effect.sync(() => { + turns.set(sessionId, [...(turns.get(sessionId) ?? []), ...added]); + const held = sessions.get(sessionId); + if (held !== undefined) { + sessions.set(sessionId, { ...held, prompted }); + } + }), + load: id => Effect.sync(() => turns.get(id) ?? []), + flush: () => Effect.void, + }; +}); +``` + +Fail with `StoreError`, naming the operation that failed. + +What the types do not say, and `checkStore` does: + +- **A reloaded turn must equal the turn that was written**, part for part, + including the reasoning signature and the tool call arguments. The prompt is + rebuilt from these, so a byte that changes moves the whole prefix. +- **The order is the order they were written in.** Identifiers sort that way — + `makeId` builds a ULID — so ordering by identifier is enough, and is what the + shipped store does. +- **`append` is one transaction.** The turns and the count describe one request: + a store that wrote the turns and lost the count hands back a session that does + not know how full it is. +- **`prompted` is the last one written**, not the first. It decides whether a + reopened session compacts before its next question. +- **`read` of an unknown session is `None`**, never an empty session. +- **One session's turns never reach another's.** + +When to write, whether to batch, and how to recover is yours. The session tells +you on every exchange and on close; nothing else is promised. + +### PromptAssembler + +One pure function, from a session to a `Prompt`. It is where the model cache is +won or lost, so it holds two invariants, and `checkAssembler` runs both: + +1. **The same input gives the same bytes.** No clock, no random value, no key + order that varies. +2. **Appending a turn changes nothing said before that turn.** `cache` is the + exception and not content: the breakpoint marks the last message, so it moves + with every turn while everything sent before it stays as it was. + +```ts +const renderPart = (part: TurnPart): readonly PromptPart[] => + part.kind === 'text' ? [{ kind: 'text', text: part.body }] : []; + +const layerPlain = Layer.succeed(PromptAssembler, { + assemble: ({ system, turns }) => ({ + system: [{ text: system, cache: true }], + messages: turns.map((turn, at) => ({ + role: turn.role, + parts: turn.parts.flatMap(renderPart), + cache: at === turns.length - 1, + })), + }), +}); +``` + +That one drops every part that is not text. The shipped assembler renders all +seven kinds, and hands reasoning back exactly as it came. + +Replace it to change what the model is told — a preamble of your own, a +different breakpoint strategy, a transcript that hides some part kinds. + +### ModelCatalog + +What a model can do. The shipped one is a table the caller writes down; a +plugin can ask the gateway instead. + +```ts +const layerEverything = Layer.succeed(ModelCatalog, { + facts: () => Effect.succeed({ apiKinds: ['messages'], contextWindow: 200_000 }), +}); +``` + +It must answer for a model it has never seen rather than failing, because the +session asks before every request and a session that cannot name its shapes +cannot send anything at all. A plugin that fetches must cache: this sits on the +request path, and one question asks two or three times. + +### ToolRegistry + +Every tool the harness has, in one service. A session names the ones it may use +and the names are resolved when it opens. + +```ts +const layerTools = Layer.succeed(ToolRegistry, { tools: [weather, questionTool(ask)] }); +``` + +A tool is a definition and a function. `run` never fails the session: return a +`ToolFailure` and the model reads it as a failed result and decides what to do. +A tool that holds one thing — a terminal, a file, a person — takes a permit for +it, because the session will not: hold an `Effect.unsafeMakeSemaphore(1)` beside +the thing and wrap `run` in `permit.withPermits(1)`. Two sessions calling one +tool then queue, which is what a parent and its subagent over one terminal need. +Two tools you build separately hold two permits, so build one per thing rather +than one per session. Say `inlineFor` for a tool that usually outlives a +request. + +Say `wait` for whether the model waits at all — the session shows it to the +model as that field's default, and reads it from `inlineFor` when you say +nothing. Both are defaults and not rules: the session adds a `wait` field to +every tool it offers, and a model that answers it decides for itself. The field +is taken back off before `run` is called, so do not name a parameter `wait` and +do not expect one. + +### TokenSource + +The credential for one call. **Read the cache inside the effect**, not while +building it: a `get` that reads it outside hands the same expired credential to +every retry, and no type says so. + +```ts +let held: { readonly value: string; readonly until: number } | undefined; + +const refreshing = { + get: () => + Effect.suspend(() => + held !== undefined && held.until > Date.now() + ? Effect.succeed(held.value) + : Effect.tryPromise({ + try: async () => { + held = await mint(); + return held.value; + }, + catch: cause => new TokenError({ cause }), + }) + ), +}; +``` + +### RetryPolicy + +One Effect `Schedule`, which sees the error, so it decides both how long to +wait and whether the error is worth waiting for. The shipped one is exponential +backoff with jitter; `layerNoRetry` is the other end. + +```ts +const layerThrice = Layer.succeed(RetryPolicy, { + schedule: Schedule.recurs(3).pipe(Schedule.addDelay(() => '1 second')), +}); +``` + +### EntropySource + +`bytes(count)`, synchronous, because it sits on the identifier path. The +shipped one is the global `crypto`; the seeded one is for a test or a replay. + +```ts +const layerCounting = Layer.sync(EntropySource, () => { + let at = 0; + return { bytes: count => Uint8Array.from({ length: count }, () => at++ % 256) }; +}); +``` + +The **ordering** of an identifier is not pluggable: an identifier must sort by +the order it was made in, or a store rebuilds the prefix in the wrong order and +misses the cache. Only where the randomness comes from is yours. + +Every block on this page is typechecked against the source tree, in +`e2e/plugins-check.ts`. + +## What is not a plugin + +`FetchLike` is not a tag. The gateway takes a `fetch` directly, and a runtime +that has a WHATWG `fetch` needs no adapter of its own: import `webFetch` from +`@kilocode/harness-sdk/plugins/fetch`. It is an entry point rather than part of +the root, so a caller who brings their own carries nothing. The README has the +hand-written version for a runtime whose `fetch` does not stream. + +There is no generic `layerModelClient(...)` helper, and there will not be. +`Layer.succeed(ModelClient, yours)` is already one line, it is the Effect idiom +the rest of the package uses, and a second way to say it would be a bigger +surface that saves nobody a keystroke. diff --git a/packages/harness-sdk/README.md b/packages/harness-sdk/README.md new file mode 100644 index 0000000000..332af1fc64 --- /dev/null +++ b/packages/harness-sdk/README.md @@ -0,0 +1,671 @@ +# @kilocode/harness-sdk + +The SDK that runs a coding agent harness. It holds a conversation with a model, +keeps the model cache warm, stores the conversation, and summarises it when it +outgrows the window. + +It runs on Node and on React Native. Nothing in the core names a platform: +`fetch` and the source of random bytes are plugins, so the same code runs in +both places. + +Contributors: read `AGENTS.md`. + +## Ask a question + +```ts +import { Effect, Stream } from 'effect'; +import { layerKilo, openSession } from '@kilocode/harness-sdk'; + +const layers = layerKilo({ + baseUrl: 'https://app.kilo.ai', + org: { kind: 'organization', id: 'org_...' }, + fetch: myFetch, // see "Your fetch" below + token: '...', +}); + +const program = Effect.gen(function* () { + const session = yield* openSession({ system: 'You are terse.', model: 'anthropic/claude-haiku-4.5' }); + yield* Stream.runForEach(session.ask('Name three fruits.'), event => + Effect.sync(() => { + if (event.kind === 'delta') { + process.stdout.write(event.text); + } + }) + ); +}); + +await Effect.runPromise(Effect.scoped(Effect.provide(program, layers))); +``` + +A model this knows nothing about is assumed to speak all three gateway shapes, +and the best one it actually speaks is used. Name what a model can do in +`models`, or change the assumption with `fallback` — that is also where a +context window goes, and without one a session never compacts. + +`layerKilo` is the wiring almost every caller writes: the prompt assembler, the +entropy source, the model catalog, and the gateway with its token and retry +policy under it. Every one of them is still a plugin. `token` also takes a +source that is asked per call — see "A credential that expires" — and a caller +who needs a catalog that asks the gateway composes the layers themselves; see +"Plugin points" below. + +## Your fetch + +The package never calls a runtime's `fetch` itself. It declares the smallest +part of one it uses, so the same code runs on Node, in a browser, in a Worker +and in a mobile app. + +On any runtime that has a WHATWG `fetch`, the adapter ships: + +```ts +import { webFetch } from '@kilocode/harness-sdk/plugins/fetch'; + +const layers = layerKilo({ baseUrl, org, token, fetch: webFetch }); +``` + +It is an entry point of its own, so a caller who brings their own carries +nothing. Every live run in `e2e/` imports it the way a consumer would, which is +how it is proven rather than described. + +A runtime without one writes one, and this is the whole of it: + +```ts +import type { FetchLike } from '@kilocode/harness-sdk'; + +const decoded = async function* decoded(body: ReadableStream) { + const decoder = new TextDecoder(); + for await (const chunk of body) { + yield decoder.decode(chunk, { stream: true }); + } +}; + +const myFetch: FetchLike = async (url, request) => { + const response = await fetch(url, { + method: request.method, + headers: { ...request.headers }, + body: request.body, + // Your runtime's own signal type. Dropping it leaves a cancelled call + // still running, and still being charged for, on the provider. + signal: (request.signal ?? null) as AbortSignal | null, + }); + const body = response.body; + return { + ok: response.ok, + status: response.status, + text: () => response.text(), + ...(body === null ? {} : { stream: () => decoded(body) }), + }; +}; +``` + +That one cast is why the core cannot hold this, and why `webFetch` is a plugin +rather than part of it: `AbortLike` is deliberately not `AbortSignal`, so only +code that has the runtime's own type can join the two. React Native may need its +own adapter, because its `fetch` does not stream a response body without a +polyfill. + +## What comes back + +`ask` returns a stream of events, in the order the provider sent them. + +| Event | Carries | +|---|---| +| `delta` | A piece of the answer's text | +| `reasoning` | A piece of the model's thinking, and the signature that closes it | +| `redacted` | Thinking the provider encrypted. There is nothing here to show a reader | +| `toolCall` | A tool the model asked for, whole: its id, its name, its arguments | +| `toolResult` | What that tool said, and whether it failed | +| `done` | This call's token counts, and why the model stopped | + +When only the answer matters, `said` folds the stream into it: + +```ts +const answer = yield* said(session.ask('Name three fruits.')); +``` + +It keeps the words and nothing else. Thinking is not the answer and a tool call +is not the answer, so a round that ran a tool gives back what the model said +after it. + +`done` is always last, and it is the only event that reports usage. `stop` is +one of `end`, `maxTokens`, `refusal`, `tools`, or `unknown`: an answer cut off at the +ceiling is not a finished answer, and a caller that retries needs to tell them +apart. `maxTokens` covers both walls — the ceiling you set and the model's own +context window — because both leave half a sentence. `unknown` means no frame +said why, which on the shapes served here means the stream ended early; treat +it the same way. + +```ts +yield* Stream.runForEach(session.ask('Name three fruits.'), event => + Effect.sync(() => { + if (event.kind === 'delta') { + process.stdout.write(event.text); + } + if (event.kind === 'done' && event.stop === 'maxTokens') { + // The answer stopped mid-sentence. Ask again with a higher maxTokens, + // or tell the reader — storing it as finished builds every later + // request on half a thought. + process.stdout.write('\n[cut off at the token ceiling]\n'); + } + }) +); +``` + +The handle also carries `history` (every turn, as a plain array), `usage` (the +counts of every call so far — pass it to `hitRatio`), `compact`, and the three +that work the queue: `queue`, `cancel` and `queued`. + +One session does one thing at a time. A second question, or a `compact`, +started while the first answer is still streaming fails with +`SessionBusyError` rather than waiting. To send one anyway, queue it. + +## Queueing a message + +`ask` answers where you stand, so it cannot wait for a session that is busy. +`queue` is for the other case: a person typing while the last answer is still +arriving. It never refuses. The message joins a line, the line is answered in +the order it formed, and the answer arrives on `continued`. + +```ts +const id = yield* session.queue('and what about Lisbon?'); + +// The line, in the order it will be asked. Show it, or take one back. +const waiting = yield* session.queued; + +// True while it is still waiting. False once it has been asked, which is not +// an error: a message the provider has seen cannot be taken back. +const dropped = yield* session.cancel(id); +``` + +Everything on `continued` names the queued entries its round answers, so one +message's answer is told from another's. A round either says something or was +refused, so narrow before you read it: + +```ts +yield* Stream.runForEach(session.continued, one => + Effect.sync(() => { + if (!one.answering.includes(id)) { + return; + } + if ('failed' in one) { + process.stdout.write(`that one failed: ${String(one.failed)}`); + } else if (one.event.kind === 'delta') { + process.stdout.write(one.event.text); + } + }) +); +``` + +A refused round is one message's bad news, not the end of the feed. The stream +itself never fails: the session goes on running rounds for the rest of the line, +and a caller whose subscription had died on the first refused round would hear +about none of them. + +`done` ends one call to the model, not the round: a round that calls a tool +makes several, and `stop` is `'tools'` on each one that is waiting for a call +the session is about to answer. So a queued message has been answered in full on +the first `done` whose stop is anything else: + +```ts +const over = ({ event }: Continued) => event.kind === 'done' && event.stop !== 'tools'; +``` + +The rounds happen whether or not anybody reads `continued`. A caller that does +not watch loses the events, never the work, and `history` holds all of it. The +stream replays its recent events to a new reader, so queueing a message and only +then subscribing still shows you the answer. + +## Tools + +A tool is a definition and a function that answers a call. Put every tool the +harness has in the registry; name on each session the ones it may use. + +```ts +import { Effect, Layer } from 'effect'; +import { openSession, ToolRegistry, type Tool } from '@kilocode/harness-sdk'; + +const weather: Tool = { + definition: { + name: 'weather', + description: 'The weather in one city.', + parameters: { + type: 'object', + properties: { city: { type: 'string', description: 'The city to report on.' } }, + required: ['city'], + }, + }, + run: call => Effect.succeed(`It is raining in ${String(JSON.parse(call.arguments).city)}.`), +}; + +const withTools = Layer.merge(layers, Layer.succeed(ToolRegistry, { tools: [weather] })); + +const session = yield* openSession({ + system: 'You are terse.', + model: 'anthropic/claude-haiku-4.5', + tools: ['weather'], +}); +``` + +The session resolves those names when it opens, and fails with +`ToolMissingError` if the registry does not hold one. A session that started +anyway would send the model a tool it cannot run, and the model would call it. + +One question is then one loop: the model answers by asking for tools, the tools +answer, the model is asked again, and only when it stops asking does any of it +reach the store. Nothing a tool does fails the question — a tool that throws, a +name the session does not offer, arguments that are not JSON — because the model +is the only party that can decide what to do about it. Each of those comes back +as a failed result. + +The calls of one turn run at once, and the session serialises nothing. A tool +that holds one thing — a terminal, a file, a person — holds a permit beside it: + +```ts +const oneAtATime = (run: Tool['run']): Tool['run'] => { + const permit = Effect.unsafeMakeSemaphore(1); + return call => permit.withPermits(1)(run(call)); +}; +``` + +The permit is the tool's because the thing it protects is the tool's. A session +knows nothing about your terminal, so it is in no position to guard it — and a +session that guarded it would only guard it against itself, which is not where +the second caller comes from. `questionTool` and `todoTool` both do this. + +Sessions share nothing. A subagent has its own transcript, counts, queue and +running calls, and no way to observe its parent's. + +### A call that outlives the request + +Every call is run under a deadline, and every call can outlive it. When the +deadline passes, the model is told the call is still running and carries on with +what does not depend on it; the work keeps going; and when it finally answers, +the session joins the same line a queued message joins, and asks the model about +it without anybody having asked a question. + +```ts +const session = yield* openSession({ + system: 'You are terse.', + model: 'anthropic/claude-haiku-4.5', + tools: ['question'], + /* How long the model waits for any tool of this session. 30 seconds by + default, and a single tool may name its own with `inlineFor`. */ + inlineFor: '5 seconds', +}); +``` + +The round it starts arrives on `continued`, like a queued message's. Several +tool results waiting at the front of the line are answered together, in one +round, because the model asked for those calls in one turn and is waiting on all +of them. + +The answer goes back as a turn the conversation says, never as a second tool +result: the call it belongs to was already answered, and every shape refuses a +second result for one call. + +### Who decides whether the model waits + +Every tool the model is offered carries one extra field, `wait`, and the +schema's `default` is what the tool says about itself: + +```ts +const weather: Tool = { + definition: { name: 'weather', description: 'The weather in one city.', parameters }, + /* What the model is told to do by default. Leave it out and the deadline + answers: a tool nobody waits any time for advertises false. */ + wait: true, + run, +}; +``` + +The two tools this package ships say opposite things, and both are right. +`question` says true, because a model asks in order to find something out. +`subagent` says false, because handing a task over is how a model carries on. +Either can be changed by the harness that wires it. + +The model's own answer beats both, in either direction: it can give up on a call +the tool expected it to wait for, and wait for one the tool expected it to +abandon. Waiting costs nothing at the provider — tools run between requests, +never during one — so what a waiting model spends is the caller's own stream, +and the caller can cut that short at any moment, which is the next section. A +model that waits still waits under the session's limit, never forever. + +The field never reaches the tool: a tool author writes the arguments their tool +takes, and nothing else arrives. + +### Sending a running call away + +The deadline is a guess made before the call started. Whoever is watching knows +better, so any call the model is waiting on can be sent to the background now: + +```ts +const waiting = yield* session.running; +const sent = yield* session.background(waiting[0]?.id ?? ''); +``` + +Nothing is cancelled. The work carries on and answers in a round of its own, +exactly as it would have on the deadline — this is the deadline brought forward. +`background` answers false when the call has already been answered, has already +gone to the background, or was never here. + +The same call serves a person pressing a key and an agent deciding it has waited +long enough. The session does not need to know which of them it was. + +### The subagent tool + +A tool that is a session of its own: its own system prompt, its own model, its +own tools, and a transcript the parent never sees. The parent pays for one +answer rather than for every step that produced it. + +```ts +const tools = [ + subagentTool( + { system: 'You look things up.', model: 'anthropic/claude-haiku-4.5', inlineFor: '5 seconds' }, + layers + ), +]; +``` + +It takes the layers to run under because a tool is handed no context. They may +be the ones the parent uses or another set entirely, which is how a subagent +runs on a cheaper model than the one that called it. + +What crosses back is one string, and never what the subagent said on its way to +its own tools. The counts do not: they belong to the session that spent them, so +`onFinished` hands them over for a caller that is adding up what a conversation +cost. A store does cross, because a session reads it from the context it runs +in — the subagent writes to the same database under a session of its own. + +### The time tool + +A model does not know what time it is. It knows roughly when it was trained, +says that date as confidently as it says everything else, and is wrong by +however long it has been since. + +```ts +import { timeTool } from '@kilocode/harness-sdk/plugins/tools'; + +const tools = [timeTool({ zone: 'Europe/Amsterdam' })]; +``` + +It takes no arguments: there is nothing about the current time for a model to +choose. UTC and the weekday always come back; `zone` adds the local time as +well, and is the harness's to set rather than the model's, because a model +naming its own zone is guessing. + +### The todo tool + +A model given a task of several steps forgets one, does two at once, or says it +is finished with a step still open. Writing the steps down and reading them back +is the fix, and every harness writes the same one. + +```ts +import { todoTool } from '@kilocode/harness-sdk/plugins/tools'; + +const tools = [todoTool({ onChanged: todos => draw(todos) })]; +``` + +The model sends the whole list every time rather than a change to it. Patching +needs stable identifiers, models invent them, and a patch against one that does +not exist either fails the call or edits the wrong line. What comes back is the +list as it now stands. + +The list belongs to the tool, not to a session, so a registry shared by a parent +and its subagents shares one list. Build a tool per session where that is wrong. + +### The question tool + +No harness can do without this one and none can write it for itself. Everything about the question is the model's — how many, +what each says, what may be picked, one answer or several, whether it may be +skipped. Everything about the asking is yours, in one function. + +```ts +import { questionTool, type Asker } from '@kilocode/harness-sdk/plugins/tools'; + +const ask: Asker = questions => + Effect.forEach(questions, question => + Effect.map(promptTheUser(question), text => ({ id: question.id, text })) + ); + +const tools = [questionTool(ask)]; +``` + +It refuses to overlap with itself, so two rounds of questions queue rather than +arriving on one person at once. Take as long as you like: a question is the +thing that outlives a request most often, and the round the answer starts is the +one that tells the model what was said. Fail with a `ToolFailure` to choose the +words the model reads when nobody answers. + +## Stopping a question + +Interrupt the fiber reading the stream. That aborts the request through the +`signal` your `fetch` adapter passes on, so the provider stops sending. + +```ts +const reading = yield* Effect.fork(Stream.runDrain(session.ask('Count to 300.'))); +// ...a stop button, a timeout, a closed tab +yield* Fiber.interrupt(reading); +``` + +The exchange leaves nothing behind: no answer arrived, so the question goes +back out of the conversation with it, and the session is free for the next +question. What this cannot promise is that the provider stops charging — +nothing the package can read reports that. `pnpm test:e2e:cancel` proves the +rest against a real call. + +## When it fails + +Every failure is a tagged error, so `Effect.catchTag` picks one out by name. +`ask` and `compact` fail with the first three; the rest reach a caller who +opens a stored session or wires the plugins by hand. + +| Tag | Means | What a caller does | +|---|---|---| +| `harness/ModelError` | The call did not come back. `reason` is `transport`, `status`, `body`, `unsupported`, or `stream`, and `status` is the HTTP status when there is one | The retry policy has already tried. A `status` of 402 or 429 is the account, not the code. A `stream` failure arrived after the answer started, so throw the fragment away and ask again | +| `harness/StoreError` | The store could not read or write. `operation` names which one | The turn is in memory and the answer is intact. The conversation cannot be continued later | +| `harness/SessionBusyError` | A second question, or a compaction, was started while the first answer was still streaming | Wait for the stream to end, then try again | +| `harness/SessionNotFoundError` | `continueSession` or `cloneSession` was given an id the store does not hold | Open a new session | +| `harness/TokenError` | The `TokenSource` could not produce a credential | Reported as a `ModelError` with `reason: 'transport'`, because it is the one failure this package cannot tell from a flaky network | +| `harness/CatalogError` | The catalog does not know the model | Never fatal on its own: the ceiling falls back to 4096 and compaction is skipped | +| `harness/EntropyError` | The runtime gave no random bytes | The layer fails to build, so no session opens | + +A question that fails leaves the session as it was. The question and the answer +reach the store together or not at all, and an unanswered question is taken back +out of memory, so the next request builds on the same prefix rather than asking +again for something the model may still answer late. + +## Storing a conversation + +Without a store a session runs in memory and cannot be continued. With one, +every turn is written as it happens. + +```ts +import { DatabaseSync } from 'node:sqlite'; +import { layerNodeStore } from '@kilocode/harness-sdk/plugins/store/node'; + +const store = layerNodeStore(new DatabaseSync('sessions.db')); +const program = Effect.provide(work, Layer.mergeAll(layers, store)); +``` + +This needs Node 22.13 or newer, which is where `node:sqlite` stopped asking for +`--experimental-sqlite`. On 22.5 to 22.12 the import fails unless the flag is +passed, and before 22.5 the module does not exist. + +`layerExpoStore` is the same store on Expo's SQLite, which the caller supplies. +It asks for the two methods it calls rather than for `SQLiteDatabase`, so the +package depends on nothing at all and your database still typechecks at the +call. Both stores are the same implementation over a one-function driver, so a +session written on one reads back on the other. + +Then `continueSession(id)` reopens a stored session, and `cloneSession(id)` +copies its turns onto a new one so a conversation can branch without paying to +build its prefix again. Both take the options from the store, never from the +caller: a system prompt that differs by one byte drops the whole cached prefix, +and the only symptom is the bill. + +The exception is the model, and it is the exception because a session freezes +one. Moving a conversation to another model is a copy of it: + +```ts +const onGlm = yield* cloneSession(id, { model: 'z-ai/glm-5.3-flash' }); +``` + +`model` and `effort` are all a clone takes, and they are what a person changing +the model picker asks for. The thinking does not come across — a provider reads +back the signature it issued and refuses one it did not — so the copy carries +what was said, what was shown and what the tools did, and leaves the reasoning +with the model that made it. + +A reopened session knows how full it is: the store keeps the provider's count of +the last request beside the session, so one reopened onto a conversation that +already fills the window compacts before it asks anything. `usage` still starts +from zero, because it counts what this run spent. + +## Compaction + +A session summarises itself when it has filled a share of the model's context +window — 0.8 by default, set with `compactAt`, and the range is 0 to 1. The +trigger is the provider's own count of the last request, so nothing here +estimates and nothing drifts. + +This needs the catalog to name a `contextWindow` for the model. Without one the +session never compacts, because a guessed window either cuts a conversation that +fit or fails to save one that did not. + +Compaction replaces the conversation with a summary of itself and replays +nothing before it. Keeping the recent turns verbatim looks better and is +refused: a thinking block is signed against the history that stood when it was +made, so a turn replayed after a summary fails on its signature. + +`session.compact` runs it now, for a caller who knows sooner than the window +does — one changing subject, say. It takes the same lock a question takes, so +it fails with `SessionBusyError` while an answer is still streaming rather than +rewriting the conversation under it. + +## A credential that expires + +`token` takes a string or a `TokenSource`. The string is one credential for the +life of the process; the source is asked for every call, which is what a +session outliving its token needs. It is the one plugin most callers replace, +so it is an option here rather than a reason to rewire by hand. + +```ts +import { TokenError, TokenSource, type TokenSourceService } from '@kilocode/harness-sdk'; + +let held: { value: string; until: number } | undefined; + +const refreshing: TokenSourceService = { + // Suspended, so every attempt reads the cache again. A failed call is + // retried by re-running this effect, so `Effect.succeed(held.value)` would + // hand the same expired credential to all three attempts. + get: () => + Effect.suspend(() => + held !== undefined && held.until > Date.now() + ? Effect.succeed(held.value) + : Effect.tryPromise({ + try: async () => { + held = await mintFromYourAuthServer(); + return held.value; + }, + catch: cause => new TokenError({ cause }), + }) + ), +}; + +const layers = layerKilo({ baseUrl, org, fetch: myFetch, token: refreshing }); +``` + +The call is on the request path, so a source that fetches must cache: the +package asks every time and caches nothing on a plugin's behalf. + +The session is scoped, so `Effect.scoped` is not optional: closing the scope is +what tells the store to write what it still holds. + +## The model cache + +Every design decision here bends toward the cached prefix, because it is most of +the bill and most of the wait. + +- Turns are append-only. An earlier turn is never rewritten. +- The system prompt, the model, and the effort are frozen for the life of the + session. Only `maxTokens` may change per question, because it never reaches + the prefix. +- A breakpoint is set after the system prompt and on the last turn, which is the + documented multi-turn shape. +- An image is stored as base64, which is what the wire wants, so it is never + encoded again. + +Measured live: 0.9997 of the input read from the cache on a ten-call session. + +The kilo gateway places breakpoints of its own, so the one this package marks +is redundant there today — measured on 2026-09-04 across both providers and all +three shapes, with and without it, on prefixes nobody had sent before. What is +not redundant is everything above it: the gateway's breakpoints need a prefix +that does not move as much as an explicit one would. + +The cache is the provider's, and it does not last. Anthropic's entries live +five minutes from the start of the request that wrote or read them, and every +call refreshes them for free — so a session that keeps talking stays warm, and +one that pauses longer than that pays to build its prefix again on the next +question. Nothing here can hold it open, and a low ratio after a pause is not a +fault in your wiring. + +## Plugin points + +Each of these is a `Context.Tag`, and each ships a default the package owns. + +| Point | What it decides | This package ships | +|---|---|---| +| `ModelClient` | How a request leaves and a reply comes back | `layerKiloGateway` | +| `ToolRegistry` | Every tool the harness has. A session names the ones it may use | `questionTool`, `subagentTool`, `timeTool`, `todoTool` | +| `PromptAssembler` | What the prompt looks like, and where the breakpoints go | `layerAssembler` | +| `ModelCatalog` | Which shapes a model speaks, its output limit, its window | `layerTableCatalog` | +| `SessionStore` | Where the conversation is kept | `layerNodeStore`, `layerExpoStore` | +| `TokenSource` | The credential for one call | `layerStaticToken` | +| `RetryPolicy` | What is tried again, and how often | `layerBackoff`, `layerNoRetry` | +| `EntropySource` | Where random bytes come from | `layerWebCrypto`, `layerSeededEntropy` | +| `FetchLike` | Not a tag: the caller passes `fetch` to the gateway | — | + +The catalog must be one instance shared by the session and the gateway. Building +it twice typechecks and answers the same, and the two then disagree about a +model the moment one of them is a fetching plugin. `layerKilo` shares it. + +Writing one is an object and a `Layer.succeed`. [PLUGINS.md](./PLUGINS.md) has a +worked example for each point and the invariants that are not in the types. + +Two of them are easy to get silently wrong: a store that reorders turns or drops +a signature, and an assembler that rewrites an earlier message, both typecheck +and both cost the whole prefix on every question afterwards. So the package +ships the checks. Run one against yours and assert it found nothing: + +```ts +import { PromptAssembler, SessionStore } from '@kilocode/harness-sdk'; +import { checkAssembler, checkStore } from '@kilocode/harness-sdk/testing'; + +const conforms = Effect.gen(function* () { + const store = yield* SessionStore; + const assembler = yield* PromptAssembler; + const wrongInStore = yield* checkStore(store); + const wrongInAssembler = checkAssembler(assembler); + return [...wrongInStore, ...wrongInAssembler]; +}); +``` + +Each answers a list of what it found, in words that say what is wrong and what +it costs. Neither fails: a store that refuses a write is a finding, not an +exception to handle. `checkStore` writes under identifiers of its own, so it is +safe against a real database. + +## Entry points + +| Import | Holds | +|---|---| +| `@kilocode/harness-sdk` | What a caller uses: the layers, the tags, the errors, and the types they carry | +| `@kilocode/harness-sdk/core` | The contracts and the pure domain, no plugin. Wider than the root: it also holds the machinery a session runs on, which a plugin author sometimes needs | +| `@kilocode/harness-sdk/plugins/fetch` | `webFetch`, for a runtime with a WHATWG `fetch` | +| `@kilocode/harness-sdk/plugins/gateway` | The gateway plugin on its own | +| `@kilocode/harness-sdk/plugins/prompt` | The assembler on its own | +| `@kilocode/harness-sdk/plugins/tools` | The tools the package ships | +| `@kilocode/harness-sdk/plugins/store/node` | The store on `node:sqlite` | +| `@kilocode/harness-sdk/plugins/store/expo` | The store on `expo-sqlite` | +| `@kilocode/harness-sdk/testing` | `checkStore` and `checkAssembler`, for a plugin author's own test suite | diff --git a/packages/harness-sdk/drizzle.config.ts b/packages/harness-sdk/drizzle.config.ts new file mode 100644 index 0000000000..a50956aac4 --- /dev/null +++ b/packages/harness-sdk/drizzle.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'drizzle-kit'; + +/** + * Authoring only. `pnpm migrations` runs drizzle-kit to write the SQL under + * `migrations/`, then inlines it into `src/plugins/store/migrations.ts`. + * + * Nothing at runtime reads this file or the SQL on disk. React Native has no + * filesystem to read migrations from, and Drizzle's answer there is a bundler + * plugin, which a package must not force on everyone who installs it. + */ +export default defineConfig({ + dialect: 'sqlite', + schema: './src/plugins/store/schema.ts', + out: './migrations', +}); diff --git a/packages/harness-sdk/e2e/all.ts b/packages/harness-sdk/e2e/all.ts new file mode 100644 index 0000000000..86550cfbc7 --- /dev/null +++ b/packages/harness-sdk/e2e/all.ts @@ -0,0 +1,102 @@ +/** + * Runs every live check, in order, and reports what held. + * + * These runs cost real money and real time, so they are not part of + * `pnpm check` and never will be. This exists because a change to the wire or + * to the session has to be answered by the provider, not by a fake, and + * running seventeen commands by hand invites running sixteen. + * + * `pnpm test:e2e:all`. One failure does not stop the rest: the point of a + * sweep is to learn everything that broke, not the first thing. Name runs to + * pick a few: `pnpm test:e2e:all stop reasoning`. + * + * Every run works through one model. `pnpm test:e2e:all full` works through all + * eleven, which is eleven times the bill and is asked for by hand. + */ +import { spawn } from 'node:child_process'; + +/** Ordered cheapest first, so a broken transport is reported in seconds. */ +const runs = [ + 'live', + 'shapes', + 'stop', + 'tools', + 'time', + 'todo', + 'image', + 'cancel', + 'queue', + 'together', + 'subagent', + 'session', + 'resume', + 'clone', + 'reasoning', + 'replay', + 'compact', + 'models', + 'tool-matrix', + 'conversation', +] as const; + +const argv = process.argv.slice(2); + +/* Every run reads its own command line, and never sees this one, so the word + travels to the children as an environment variable. */ +const full = argv.includes('full'); +const only = argv.filter(name => name !== 'full'); + +/* A name nobody runs is a name nobody meant. Reporting "0 of 0 passed" and + exiting 0 on a typo is the worst answer a sweep can give. */ +const known: ReadonlySet = new Set(runs); +const unknown = only.filter(name => !known.has(name)); +if (unknown.length > 0) { + console.log(`no such live run: ${unknown.join(', ')}\nthere is: ${runs.join(', ')}`); + process.exit(1); +} +const chosen = only.length === 0 ? runs : runs.filter(name => only.includes(name)); + +/** The cache run is the default one, so its script has no suffix. */ +const scriptOf = (name: string): string => (name === 'live' ? 'test:e2e' : `test:e2e:${name}`); + +const runOne = (name: string): Promise<{ ok: boolean; taken: number; tail: string }> => + new Promise(resolve => { + const started = Date.now(); + const held: string[] = []; + const child = spawn('pnpm', [scriptOf(name)], { + cwd: `${import.meta.dirname}/..`, + env: full ? { ...process.env, KILO_FULL: '1' } : process.env, + }); + const keep = (chunk: Buffer): void => { + held.push(chunk.toString()); + }; + child.stdout.on('data', keep); + child.stderr.on('data', keep); + child.on('close', code => { + const lines = held.join('').trimEnd().split('\n'); + resolve({ + ok: code === 0, + taken: Date.now() - started, + tail: lines.slice(-1)[0] ?? '', + }); + }); + }); + +const failed: string[] = []; + +for (const name of chosen) { + const { ok, taken, tail } = await runOne(name); + const seconds = `${(taken / 1000).toFixed(0)}s`; + console.log(`${ok ? 'PASS' : 'FAIL'} ${name.padEnd(10)}${seconds.padStart(5)} ${tail}`); + if (!ok) { + failed.push(name); + } +} + +console.log( + `\n${String(chosen.length - failed.length)} of ${String(chosen.length)} live runs passed.` +); +if (failed.length > 0) { + console.log(`re-run one on its own for the whole output: pnpm ${scriptOf(failed[0] ?? '')}`); + process.exitCode = 1; +} diff --git a/packages/harness-sdk/e2e/cancel.ts b/packages/harness-sdk/e2e/cancel.ts new file mode 100644 index 0000000000..678b95e043 --- /dev/null +++ b/packages/harness-sdk/e2e/cancel.ts @@ -0,0 +1,207 @@ +/** + * Proves a real call stops when the caller stops listening. + * + * `cancel.test.ts` proves the same thing against a fake `fetch`. A fake cannot + * show what a real socket does: whether the abort reaches undici, whether the + * reader throws where nothing catches it, whether the session is still usable + * after. So this run asks a model for a long answer twice, reads one to the + * end, and walks away from the other. + * + * What it proves: the client stops early, and the session survives it. + * What it cannot prove: that the provider stops generating and stops charging. + * Nothing this package can read reports that. + */ +import { Duration, Effect, Fiber, Ref, Stream } from 'effect'; +import type { AbortLike, FetchLike } from '../src/core/fetch.js'; +import { openSession } from '../src/core/run.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { kilo, models, room } from './setup.js'; +import { fail, passed, under, wrongIf } from './report.js'; +import { webFetch } from '../src/plugins/fetch/web.js'; + +const system = 'You do exactly what you are told, at length, with no preamble.'; +const long = 'Count from 1 to 300. Put one number on each line. Do not stop early.'; + +/** + * How much of the answer the cancelled run reads before it walks away. It + * waits for pieces rather than for a clock: the interesting case is a caller + * who leaves mid-stream, and a fixed wait shorter than the time to the first + * piece would only ever cancel a request that had not started answering. + */ +const readBeforeLeaving = 10; + +/** An unhandled rejection is the live-only failure a fake `fetch` cannot show. */ +const loose: unknown[] = []; +process.on('unhandledRejection', reason => loose.push(reason)); + +/** Records the signal of every call, so the run can read it afterwards. */ +const signals: (AbortLike | undefined)[] = []; +const watchedFetch: FetchLike = (url, request) => { + signals.push(request.signal); + return webFetch(url, request); +}; + +const layers = kilo({ apiKinds: ['messages'] }, { fetch: watchedFetch }); + +/** + * Counts the pieces of the answer, whether it finishes or is walked away from. + * + * Its own ceiling, not the shared one: the answer has to still be running when + * the run walks away from it. + */ +const counted = (session: SessionHandle, said: Ref.Ref) => + session.ask(long, { maxTokens: 1500 }).pipe( + Stream.tap(event => + /* Thinking counts. A model that reasons first spends longer on its first + word than the whole wait allows — `minimax/minimax-m3` passed thirty + seconds with none — and it is answering the whole time. Walking away + mid-thought is the same walking away this run is about. */ + event.kind === 'delta' || event.kind === 'reasoning' + ? Ref.update(said, held => held + 1) + : Effect.void + ), + Stream.runDrain + ); + +const since = (start: number) => Date.now() - start; + +/** Waits until the answer has produced `target` pieces, or gives up. */ +const until = (said: Ref.Ref, target: number) => + Ref.get(said).pipe( + Effect.delay(Duration.millis(20)), + Effect.repeat({ until: (held: number) => held >= target }), + Effect.timeoutFail({ + /* A minute, because this waits on a model and not on the package. Half of + it was not enough for `minimax/minimax-m3` on 2026-09-06, which streams + nothing at all — not even thinking — for a long time before its first + word, and a run that gave up then reported a cancellation that never + happened. */ + duration: Duration.seconds(60), + onTimeout: () => new Error(`the answer never reached ${String(target)} pieces`), + }) + ); + +const program = (model: string) => + Effect.gen(function* () { + const session = yield* openSession({ system, model }); + + /* The baseline. It says how long the whole answer takes, so the cancelled + run has something to be measured against. */ + const whole = yield* Ref.make(0); + const started = Date.now(); + yield* counted(session, whole); + const wholeMillis = since(started); + + const cut = yield* Ref.make(0); + const cutSession = yield* openSession({ system, model }); + const cutStarted = Date.now(); + const reading = yield* Effect.fork(counted(cutSession, cut)); + yield* until(cut, readBeforeLeaving); + yield* Fiber.interrupt(reading); + const cutMillis = since(cutStarted); + + return { + whole: { said: yield* Ref.get(whole), millis: wholeMillis }, + cut: { said: yield* Ref.get(cut), millis: cutMillis }, + /* An interrupted exchange leaves nothing: the answer never arrived, and + the question goes back out with it. */ + history: yield* cutSession.history, + /* The session must still work: a `busy` flag left set would strand it. */ + after: yield* Stream.runFold( + /* Room for a model that thinks before it answers: at 16 tokens a + reasoning model spends the lot on thinking and says nothing, which + reads as a stranded session and is not one. */ + cutSession.ask('Answer with the word: ok', { maxTokens: room }), + '', + (held, event) => (event.kind === 'delta' ? held + event.text : held) + ), + }; + }); + +/** The models whose answer ran long enough to walk away from. The floor is one. */ +const cancelled: string[] = []; + +for (const model of models) { + under(model); + + /* Every model starts from an empty pair of watch lists. */ + signals.length = 0; + loose.length = 0; + + /* One model's bad round must not take the other ten with it: these runs cost + money and minutes, and learning one failure per sweep turns an afternoon + into a week. */ + const got = await Effect.runPromise( + Effect.either(Effect.scoped(Effect.provide(program(model), layers))) + ); + if (got._tag === 'Left') { + console.log('model ', model, 'FAILED', JSON.stringify(String(got.left))); + /* A minute with fewer than ten pieces is a model too slow to walk away + from, not a cancellation that failed: `minimax/minimax-m3` streams + nothing for over a minute on 2026-09-06. There is no mid-stream here to + leave. A package that stopped streaming would put every model here, which + is what the floor below catches. */ + if (!String(got.left).includes('never reached')) { + fail(`the run failed: ${String(got.left)}`); + } + continue; + } + cancelled.push(model); + const result = got.right; + + const roles = result.history.map(turn => turn.role); + + console.log('model ', model); + console.log('whole answer ', result.whole.said, 'pieces in', result.whole.millis, 'ms'); + console.log('walked away ', result.cut.said, 'pieces in', result.cut.millis, 'ms'); + console.log('turns kept ', JSON.stringify(roles)); + console.log('asked again ', JSON.stringify(result.after)); + console.log('signals ', signals.map(signal => signal?.aborted ?? 'none').join(' ')); + console.log('loose errors ', loose.length); + + if (result.whole.said === 0) { + fail('the baseline answer carried no text, so there is nothing to compare against'); + } + if (result.whole.said < readBeforeLeaving * 2) { + fail( + `the whole answer had ${String(result.whole.said)} pieces, which is too few to tell a ` + + 'cancelled run from a finished one; ask for a longer answer' + ); + } + if (result.cut.said < readBeforeLeaving) { + fail( + `the cancelled run read ${String(result.cut.said)} pieces, so it never got mid-stream and ` + + 'the run only cancelled a request that had not started answering' + ); + } + if (result.cut.said >= result.whole.said) { + fail( + `the cancelled run read ${String(result.cut.said)} pieces and the whole answer had ` + + `${String(result.whole.said)}, so nothing was cut short` + ); + } + if (!signals.every(signal => signal?.aborted === true)) { + fail('a call ended with its signal not aborted, so the socket was left open'); + } + if (roles.length !== 0) { + fail( + `the cancelled session kept ${JSON.stringify(roles)}; an interrupted exchange must leave ` + + 'nothing, because a half written answer poisons the prefix and an unanswered question ' + + 'goes back out with every later request' + ); + } + if (result.after.length === 0) { + fail('the session could not be asked again after the cancellation'); + } + if (loose.length > 0) { + fail(`the abort left ${String(loose.length)} unhandled rejection(s): ${String(loose[0])}`); + } +} + +under(''); +console.log( + `\nanswered long enough to cancel: ${String(cancelled.length)} of ${String(models.length)} models` +); +wrongIf(cancelled.length === 0, 'not one answer ran long enough to walk away from'); + +passed('the call stopped when the caller did, and the session survived it.'); diff --git a/packages/harness-sdk/e2e/clone.ts b/packages/harness-sdk/e2e/clone.ts new file mode 100644 index 0000000000..3fcb4a0af8 --- /dev/null +++ b/packages/harness-sdk/e2e/clone.ts @@ -0,0 +1,140 @@ +/** + * Proves a clone is cheap, which is the only reason to have one. + * + * `resume.test.ts` compares the prompt a clone sends with the prompt its source + * sends, byte for byte, and that is as far as a fake can go. The claim is about + * money: the copy renders to the same bytes, so the provider reads the prefix + * out of its cache instead of building it again. Only a real gateway reports + * that, as a large `cache read` against an almost empty `cache write`. + * + * A clone that copied one identifier into the prompt, or reordered one part, + * would still pass every unit test here and quietly double the bill. + */ +import { DatabaseSync } from 'node:sqlite'; +import { Effect, Layer, Stream } from 'effect'; +import type { ModelUsage } from '../src/core/model.js'; +import { cloneSession } from '../src/core/resume.js'; +import type { ResumeContext } from '../src/core/resume.js'; +import { openSession } from '../src/core/run.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { layerNodeStore } from '../src/plugins/store/node.js'; +import { cachedSystem as system, kilo, models, room } from './setup.js'; +import { fail, passed, under, wrongIf } from './report.js'; + +/** One database, two runs, as a second start of an application would have. */ +const database = new DatabaseSync(':memory:'); +const layers = Layer.mergeAll(kilo({ apiKinds: ['messages'] }), layerNodeStore(database)); + +const run = (use: Effect.Effect): Promise => + Effect.runPromise(Effect.scoped(Effect.provide(use, layers))); + +const ask = (session: SessionHandle, text: string) => + Stream.runFold( + session.ask(text, { maxTokens: room }), + { said: '', usage: undefined as ModelUsage | undefined }, + (held, event) => + event.kind === 'delta' + ? { ...held, said: held.said + event.text } + : event.kind === 'done' + ? { ...held, usage: event.usage } + : held + ); + +/** Builds the prefix and pays for it, so there is a warm cache to inherit. */ +const source = (model: string) => + Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens: room }); + const first = yield* ask(session, 'Answer with the word: one'); + const second = yield* ask(session, 'Answer with the word: two'); + return { id: session.id, first: first.usage, second: second.usage }; + }); + +/** Branches it, asks one question, and reports what the branch was charged. */ +const branch = (sessionId: string) => + Effect.gen(function* () { + const session = yield* cloneSession(sessionId); + const answer = yield* ask(session, 'Answer with the word: three'); + return { + id: session.id, + said: answer.said, + usage: answer.usage, + turns: yield* session.history, + }; + }); + +const show = (name: string, usage: ModelUsage | undefined): void => { + console.log( + `${name.padEnd(14)}input ${String(usage?.inputTokens ?? 0).padEnd(6)}` + + `cache read ${String(usage?.cacheReadTokens ?? 0).padEnd(7)}` + + `cache write ${String(usage?.cacheWriteTokens ?? 0)}` + ); +}; + +/** The models whose provider made the prefix readable. The floor is that one did. */ +const cached: string[] = []; + +for (const model of models) { + under(model); + + console.log('model', model, '\n'); + + const first = await run(source(model)); + const cloned = await run(branch(first.id)); + const original = await run(branch(first.id)); + + show('source call 1', first.first); + show('source call 2', first.second); + show('clone call 1', cloned.usage); + show('second clone', original.usage); + console.log('\nclone id ', cloned.id, '\nsource id ', first.id); + console.log('clone said', JSON.stringify(cloned.said), 'over', cloned.turns.length, 'turns'); + + const read = cloned.usage?.cacheReadTokens ?? 0; + const written = cloned.usage?.cacheWriteTokens ?? 0; + + if (cloned.id === first.id) { + fail('the clone took the identifier of the session it came from'); + } + if (cloned.turns.length !== 6) { + /* The four copied turns, the question this run asked, and its answer. A + clone that lost a turn would send a shorter prompt and still read most of + the prefix, so the count is checked as well as the cache. */ + fail(`the clone holds ${String(cloned.turns.length)} turns where it should hold 6`); + } + if ((first.second?.cacheReadTokens ?? 0) === 0) { + /* The provider never made an entry readable, so there is no prefix here to + measure a clone against. Read from the source rather than a list of names + here: `nvidia/nemotron-3.5-lightning` reads zero on every call on + 2026-09-06, and which providers cache changes without warning. What the + clone is — its own identifier, six turns, an answer — is checked above on + every model regardless. */ + console.log('the provider cached nothing, so there is no prefix to measure'); + continue; + } + cached.push(model); + if (read === 0) { + fail('the clone read nothing from the cache, so it paid for the prefix again'); + } + if (written > read / 4) { + fail( + `the clone wrote ${String(written)} against ${String(read)} read, so its prompt is not ` + + 'the prompt the source sent' + ); + } + if (cloned.said.length === 0) { + fail('the clone answered with nothing'); + } + if ((original.usage?.cacheReadTokens ?? 0) === 0) { + /* The second clone comes off the same source, which the first clone must not + have touched. A source that grew would move the prefix out from under it. */ + fail('a second clone of the same session read nothing, so the first one changed it'); + } +} + +under(''); +console.log(`\ncached the prefix: ${String(cached.length)} of ${String(models.length)} models`); +/* The floor under the skip: a clone that stopped sending the source's prompt + would read nothing anywhere, and that must go red rather than quiet. */ +wrongIf(cached.length === 0, 'not one provider cached the prefix, so no clone was measured'); + +passed('every clone the provider cached for read its prefix and wrote next to nothing'); diff --git a/packages/harness-sdk/e2e/compact.ts b/packages/harness-sdk/e2e/compact.ts new file mode 100644 index 0000000000..08d24ecf2c --- /dev/null +++ b/packages/harness-sdk/e2e/compact.ts @@ -0,0 +1,156 @@ +/** + * Proves a session survives filling its own context window. + * + * The unit tests script the token counts, so they prove the trigger fires where + * it is told to. They cannot prove the summary is worth anything. This run + * plants a fact early, fills the window with unrelated talk, and asks for the + * fact back after the session has compacted itself. + * + * The window is set by the caller rather than the model's real one: filling + * 200k tokens to test this would cost real money and take an hour. What is + * live here is the summariser, the prompt it builds, and whether the fact + * survives the round trip. + */ +import { Effect, Stream } from 'effect'; +import { openSession } from '../src/core/run.js'; +import type { Turn } from '../src/core/turn.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { everyShape, kilo, models, room } from './setup.js'; +import { fail, passed, under, wrongIf } from './report.js'; + +/** Small enough that a few turns of chat fill it. */ +const contextWindow = 80; + +const system = 'You answer briefly and remember what you are told.'; + +/** The fact the summary has to carry. Nothing else in the run mentions it. */ +const secret = 'the vault code is 4417'; +const plant = `Remember this for later: ${secret}. Reply with the word: noted`; + +/** Filler that is long enough to push the prompt over the window. */ +const filler = [ + 'Name three colours. One line.', + 'Name three fruits. One line.', + 'Name three cities. One line.', +]; + +const recall = 'What was the vault code I gave you? Answer with the number only.'; + +/** + * One question, keeping what it said and what the `done` event reported. + * + * The counts matter here: added up they are every call the caller made, so + * `session.usage` above them is the summary call, which the caller never made + * and is still billed for. + */ +const say = (session: SessionHandle, text: string) => + Stream.runFold(session.ask(text, { maxTokens: room }), { said: '', output: 0 }, (held, event) => { + if (event.kind === 'delta') { + return { ...held, said: held.said + event.text }; + } + return event.kind === 'done' + ? { ...held, output: held.output + event.usage.outputTokens } + : held; + }); +const layers = kilo({ apiKinds: everyShape, contextWindow }); + +const program = (model: string) => + Effect.gen(function* () { + const session = yield* openSession({ system, model }); + let asked = 0; + asked += (yield* say(session, plant)).output; + for (const question of filler) { + asked += (yield* say(session, question)).output; + } + const last = yield* say(session, recall); + asked += last.output; + return { + answer: last.said, + asked, + history: yield* session.history, + used: yield* session.usage, + }; + }); + +/** One whole run of the model. */ +const attempt = (model: string) => + Effect.runPromise(Effect.scoped(Effect.provide(program(model), layers))); + +/** Whether the summary this run wrote carried the planted fact. */ +const carried = (result: { readonly history: readonly Turn[] }): boolean => + result.history + .filter(turn => turn.parts.some(part => part.kind === 'summary')) + .some(turn => (turn.parts[0]?.body ?? '').includes('4417')); + +/** The models whose summariser kept the fact. The floor is that one did. */ +const kept: string[] = []; + +for (const model of models) { + under(model); + + /* Tried once more before it counts: what a summariser chooses to write is the + model's, and a summary that dropped the fact once can keep it on the next + run. Twice is a finding. */ + const first = await attempt(model); + const result = carried(first) ? first : await attempt(model); + + const turns = result.history; + const isSummary = (turn: Turn): boolean => turn.parts.some(part => part.kind === 'summary'); + const summaries = turns.filter(isSummary); + const summary = summaries[0]?.parts[0]?.body ?? ''; + + console.log('model ', model); + console.log('window ', contextWindow, 'tokens'); + console.log('turns kept ', turns.length); + console.log('summaries ', summaries.length); + console.log('summary ', JSON.stringify(summary.slice(0, 160))); + console.log('recalled ', JSON.stringify(result.answer)); + console.log( + 'output used', + result.used.outputTokens, + 'of which the questions asked for', + result.asked + ); + + if (summaries.length === 0) { + fail('the session never compacted, so this run proves nothing; lower the window or add filler'); + } + if (!summary.includes('4417')) { + /* What a summariser writes is the model's: `nvidia/nemotron-3.5-lightning` + drops the fact twice over on 2026-09-06 and then has nothing to recall. + That the session compacted at all, put the summary after what it + summarised, and billed the summary call is the package's half, and it is + asserted above and below on every model. A package that stopped planting + the fact would put every model here, which the floor catches. */ + console.log('the summariser dropped the fact, twice'); + continue; + } + kept.push(model); + if (turns.findIndex(isSummary) === 0) { + fail('the summary is the first turn, so nothing was summarised'); + } + if (result.used.outputTokens <= result.asked) { + /* Every `done` event added up is what the questions cost. The session's own + total has to be larger, because it also holds the summary call, which the + caller never asked for and is billed for all the same. */ + fail( + `the counts leave out the summary call: the session reports ` + + `${String(result.used.outputTokens)} output tokens and the questions alone ` + + `account for ${String(result.asked)}` + ); + } + if (!result.answer.includes('4417')) { + fail( + `the fact did not survive the summary: the model answered ${JSON.stringify(result.answer)}, ` + + 'so the summariser dropped what a later turn needed' + ); + } +} + +under(''); +console.log( + `\nthe summary kept the fact: ${String(kept.length)} of ${String(models.length)} models` +); +wrongIf(kept.length === 0, 'not one summary kept the fact, so nothing here planted one'); + +passed('the session compacted itself, and every summariser that kept the fact could recall it.'); diff --git a/packages/harness-sdk/e2e/conversation.ts b/packages/harness-sdk/e2e/conversation.ts new file mode 100644 index 0000000000..3d9527dc11 --- /dev/null +++ b/packages/harness-sdk/e2e/conversation.ts @@ -0,0 +1,582 @@ +/** + * One conversation, end to end, of the shape a harness actually has. + * + * Every other live run proves one thing on its own: the cache, the line, a + * tool, the store. A harness does all of it at once, in one session, and the + * defects that only show up there are the ones nobody has a test for — a tool + * that works alone and not beside another, a queued message that arrives while + * a subagent is out, a summary that drops the one fact the next turn needs. + * + * So this is a person and an agent working through a small migration together: + * + * - **It asks the time.** The time tool, which takes no arguments. + * - **It writes the plan down.** The todo tool, several steps, marked as they go. + * - **It asks the person two things in one call.** The question tool, with + * choices, answered slower than the model waits, so the answer lands in a + * round of its own. + * - **The person types while it is busy.** A queued message, answered in the + * order it joined. + * - **It sends a subagent to look something up.** A session of its own, whose + * answer carries a word the parent could not know, and whose counts come back + * to the caller. + * - **It is reopened from SQLite**, and remembers. + * - **It is cloned**, and the copy reads its prefix from the cache. + * - **It is compacted**, and still remembers. + * + * What is asserted is correctness first — every claim above, per model — and + * then performance: the median time to the first word, the median whole + * answer, the share of the prompt read from cache, and the wall clock for the + * conversation. The ceilings are generous on purpose. They are there to catch a + * change that makes the package slow, not to rank the providers: a run that + * failed because a provider had a bad minute would teach nobody anything. + * + * `pnpm test:e2e:conversation` works through one model. + * `pnpm test:e2e:conversation full` works through all eleven. + */ +import { DatabaseSync } from 'node:sqlite'; +import { Duration, Effect, Fiber, Layer, Ref, Schedule, Stream } from 'effect'; +import { SessionBusyError } from '../src/core/ask.js'; +import type { ModelUsage } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import { cloneSession, continueSession } from '../src/core/resume.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import type { Continued } from '../src/core/queue.js'; +import type { Turn } from '../src/core/turn.js'; +import { type Tool, ToolRegistry } from '../src/core/tool.js'; +import { hitRatio } from '../src/core/usage.js'; +import { + type Answer, + type Asker, + type Question, + questionTool, +} from '../src/plugins/tools/question.js'; +import { type SubagentReport, subagentTool } from '../src/plugins/tools/subagent.js'; +import { timeTool } from '../src/plugins/tools/time.js'; +import { type Todo, todoTool } from '../src/plugins/tools/todo.js'; +import { layerNodeStore } from '../src/plugins/store/node.js'; +import { kilo, models, room } from './setup.js'; +import { passed, under, wrongIf } from './report.js'; + +/** + * One convention of the project, of which there are two hundred. + * + * The length is the point. A harness carries a long brief, and the whole claim + * of the prompt cache is that it is paid for once; a short prompt caches + * nothing at all on any provider here, and a conversation measured against it + * would read as a failure of the package rather than of the prompt it was + * given. + */ +const convention = (index: number) => + `Convention ${String(index)}: keep the change small, name the file you touched, ` + + 'and say what you did in one line. Do not rewrite what you were not asked to ' + + 'rewrite. Do not add a dependency where the standard library answers. Run the ' + + 'checks before you say a thing is done.'; + +const system = [ + 'You are the assistant one person works with on their release. You have ' + + 'tools: use the one the person names, and use it rather than answering ' + + 'from memory. You have no files and no shell, and nothing here needs ' + + 'either: everything you are asked for is in this conversation or comes ' + + 'from a tool. Answer in one or two short sentences, and never ask for ' + + 'anything you were not asked to ask for. The project conventions follow, ' + + 'and they all hold.', + ...Array.from({ length: 200 }, (_, index) => convention(index)), +].join('\n'); + +/** A word no model can invent, so an answer carrying it came from the subagent. */ +const codename = 'nightjar'; + +/** A fact planted early and asked for at the very end, after the summary. */ +const secret = 'the staging database is called quokka'; + +const subSystem = + 'You answer in one short sentence, from these facts and nothing else. The ' + + `codename of the Acme Deploy 4.0 release is ${codename}. The release is out ` + + 'on the fourth of March. Nobody else knows either.'; + +/** What one turn cost the person, in the way a person feels it. */ +interface Timing { + /** Milliseconds from asking to the first word of the answer. */ + readonly first: number; + /** Milliseconds from asking to the end of the answer. */ + readonly whole: number; +} + +/** + * Asks, keeps the words, and times the answer as the person sees it. + * + * A session busy with a round of its own refuses rather than corrupting the + * prefix, so the person waits and asks again. That is the harness's job and not + * the package's: the refusal is the package doing the right thing, and this is + * what a caller does with it. + */ +const timed = (session: SessionHandle, input: string, timings: Timing[]) => + Effect.retry(asking(session, input, timings), { + while: (cause: unknown) => cause instanceof SessionBusyError, + schedule: Schedule.spaced('500 millis').pipe(Schedule.upTo('90 seconds')), + }); + +const asking = (session: SessionHandle, input: string, timings: Timing[]) => + Effect.suspend(() => { + const started = Date.now(); + let first = 0; + return Stream.runFold(session.ask(input), '', (held, event) => { + if (event.kind !== 'delta') { + return held; + } + first = first === 0 ? Date.now() - started : first; + return held + event.text; + }).pipe( + Effect.tap(() => Effect.sync(() => void timings.push({ first, whole: Date.now() - started }))) + ); + }); + +/** + * Watches the rounds the session runs on its own, per session. + * + * `e2e/rounds.ts` keeps its refusals in one array for the module, which is + * right for a run that works through one session at a time. This one runs + * several models at once, so each conversation counts its own. + * + * The deadline is on the waiting and never around the reading — see the note in + * `e2e/rounds.ts`, which is where that was measured. + */ +const watching = (session: SessionHandle, within: Duration.DurationInput) => + Effect.gen(function* () { + const rounds: { answering: readonly string[]; text: string }[] = []; + const refused: string[] = []; + const held = { answering: [] as readonly string[], text: '' }; + const over = (one: Continued): boolean => { + const event = 'failed' in one ? undefined : one.event; + return event?.kind === 'done' && event.stop !== 'tools'; + }; + const fiber = yield* Effect.forkScoped( + Stream.runForEach(session.continued, (one: Continued) => + Effect.sync(() => { + held.answering = one.answering; + const event = 'failed' in one ? undefined : one.event; + if (event?.kind === 'delta') { + held.text += event.text; + } + if ('failed' in one) { + refused.push(String(one.failed)); + } + if (over(one) || 'failed' in one) { + rounds.push({ answering: held.answering, text: held.text }); + held.text = ''; + } + }) + ) + ); + return { + rounds, + refused, + /* Waits for the session to go quiet, not for a number of rounds. How + many rounds a conversation runs is the model's to decide: one that + waits for its subagent answers inline and runs none, and counting them + would hold every other model at the deadline for nothing. */ + done: Effect.retry( + Effect.filterOrFail( + Effect.zip(session.queued, session.running), + ([waiting, running]) => waiting.length === 0 && running.length === 0, + () => 'still working' as const + ), + Schedule.spaced('500 millis').pipe(Schedule.upTo(within)) + ).pipe( + Effect.ignore, + /* One more beat, because the last round's words arrive just after the + line empties: the entry leaves the line when the round starts, not + when it ends. Five seconds is longer than any round here takes to + write its sentence, and it is spent once per model. */ + Effect.zipRight(Effect.sleep('5 seconds')), + Effect.zipRight(Fiber.interrupt(fiber)) + ), + }; + }); + +/** Everything one conversation ended up with, for the checks and the table. */ +interface Conversation { + readonly said: readonly string[]; + readonly timings: readonly Timing[]; + readonly todos: readonly Todo[]; + /** The questions of each call the asker took, so a run can count both. */ + readonly asked: readonly (readonly Question[])[]; + readonly rounds: readonly { answering: readonly string[]; text: string }[]; + readonly refused: readonly string[]; + readonly reports: readonly SubagentReport[]; + readonly queuedId: string; + readonly usage: ModelUsage; + readonly turns: readonly Turn[]; + readonly reopened: string; + readonly cloneUsage: ModelUsage | undefined; + readonly summaries: number; + readonly afterSummary: string; + readonly seconds: number; +} + +const isSummary = (turn: Turn): boolean => turn.parts.some(part => part.kind === 'summary'); + +/** The one call the clone makes, and what it was charged for it. */ +const oneMore = (sessionId: string) => + Effect.gen(function* () { + const clone = yield* cloneSession(sessionId); + return yield* Stream.runFold( + clone.ask('Answer with the word: ok', { maxTokens: room }), + undefined as ModelUsage | undefined, + (held, event) => (event.kind === 'done' ? event.usage : held) + ); + }); + +/** + * The whole conversation, for one model. + * + * The tools are built here rather than once for the run, because a tool holds + * what it protects: one asker is one person, one list is one plan, and eleven + * conversations running at once are eleven of each. + */ +const converse = (model: string) => + Effect.gen(function* () { + const started = Date.now(); + const timings: Timing[] = []; + const asked: Question[][] = []; + const reports: SubagentReport[] = []; + const list = yield* Ref.make([]); + + /* Slower than any model waits, so the answer lands in a round of its own. */ + const asker: Asker = questions => + Effect.gen(function* () { + asked.push([...questions]); + yield* Effect.sleep('5 seconds'); + return questions.map( + (question): Answer => ({ + id: question.id, + ...(question.choices?.[0] === undefined + ? { text: 'ultramarine' } + : { chosen: [question.choices[0].value] }), + }) + ); + }); + + const layers = kilo(); + const tools: readonly Tool[] = [ + timeTool({ zone: 'Europe/Amsterdam' }), + todoTool({ onChanged: todos => Ref.set(list, todos) }), + questionTool(asker, { inlineFor: Duration.seconds(1) }), + subagentTool( + { + system: subSystem, + model, + /* Room for a model that thinks before it answers: at 256 tokens two + of the eleven spent the lot on reasoning and came back empty. */ + maxTokens: room, + inlineFor: Duration.seconds(2), + onFinished: report => Effect.sync(() => void reports.push(report)), + }, + layers + ), + ]; + + const database = new DatabaseSync(':memory:'); + const store = layerNodeStore(database); + const registry = Layer.succeed(ToolRegistry, { tools }); + const everything = Layer.mergeAll(layers, store, registry); + + const conversation = Effect.gen(function* () { + const session = yield* openSession({ + system, + model, + maxTokens: room, + tools: ['time', 'todo', 'question', 'subagent'], + }); + const watch = yield* watching(session, '180 seconds'); + + const words: string[] = []; + words.push( + yield* timed( + session, + `Remember this for later: ${secret}. Reply with the word: noted`, + timings + ) + ); + words.push(yield* timed(session, 'What time is it right now? Give me the time.', timings)); + words.push( + yield* timed( + session, + 'Write my release plan down with the todo tool, as these three steps in ' + + 'this order: cut the branch, run the checks, publish the notes. Mark ' + + 'the first one as the one you are on. Then tell me the list.', + timings + ) + ); + words.push( + yield* timed( + session, + 'Now use the question tool once to ask me two things in that one call: ' + + 'which day to publish on, offering Tuesday and Thursday as choices, ' + + 'and which channel to announce in, offering email and chat as ' + + 'choices. Then tell me what I picked.', + timings + ) + ); + /* Typed while the question is still out, so it joins the line behind it. */ + const queuedId = yield* session.queue("Answer with the word 'pelican' and nothing else."); + words.push( + yield* timed( + session, + 'What is the codename of the Acme Deploy 4.0 release? Hand that to a ' + + 'subagent and tell me what it says.', + timings + ) + ); + yield* watch.done; + + /* Asked before the summary as well as after it, because a fact the + conversation has used twice is the fact a summariser must keep. */ + words.push( + yield* timed( + session, + 'What is the staging database called? Answer with the one word.', + timings + ) + ); + + const turns = yield* session.history; + const usage = yield* session.usage; + + /* Reopened from SQLite, which is the only place the rounds the session + ran on its own could have gone. */ + const reopened = yield* continueSession(session.id); + const remembered = yield* timed( + reopened, + 'What is the staging database called? Answer with the one word.', + timings + ); + + const cloneUsage = yield* oneMore(session.id); + + /* Compacted on purpose, and then asked for the fact the summary had to + keep. A harness changing subject does exactly this. */ + yield* reopened.compact; + const afterSummary = yield* timed( + reopened, + 'What is the staging database called? Answer with the one word.', + timings + ); + + return { + words, + queuedId, + usage, + turns, + reopened: remembered, + cloneUsage, + summaries: (yield* reopened.history).filter(isSummary).length, + afterSummary, + rounds: watch.rounds, + refused: watch.refused, + }; + }); + + const got = yield* Effect.scoped(Effect.provide(conversation, everything)); + return { + said: got.words, + timings, + todos: yield* Ref.get(list), + asked, + rounds: got.rounds, + refused: got.refused, + reports, + queuedId: got.queuedId, + usage: got.usage, + turns: got.turns, + reopened: got.reopened, + cloneUsage: got.cloneUsage, + summaries: got.summaries, + afterSummary: got.afterSummary, + seconds: (Date.now() - started) / 1000, + } satisfies Conversation; + }); + +/** Nothing at all, for a model whose conversation never started. */ +const nothing: Conversation = { + said: [], + timings: [], + todos: [], + asked: [], + rounds: [], + refused: ['the conversation failed'], + reports: [], + queuedId: '', + usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + turns: [], + reopened: '', + cloneUsage: undefined, + summaries: 0, + afterSummary: '', + seconds: 0, +}; + +/** One bad minute at a provider is not a defect, so a failed call is retried once. */ +const held = (model: string): Promise => + Effect.runPromise( + Effect.retry(converse(model), Schedule.once).pipe( + Effect.catchAll(cause => + Effect.succeed({ ...nothing, refused: [`the conversation failed: ${String(cause)}`] }) + ) + ) + ); + +const median = (taken: readonly number[]): number => + taken.length === 0 ? 0 : (taken.toSorted((a, b) => a - b)[Math.floor(taken.length / 2)] ?? 0); + +const pad = (text: string, width: number): string => text.padEnd(width); +const ms = (taken: number): string => `${taken.toFixed(0)}ms`; + +/** What the whole conversation is allowed to cost, per model. */ +const slowestFirstWord = 30_000; +const slowestConversation = 300; + +const rows = await Promise.all(models.map(async model => ({ model, got: await held(model) }))); + +console.log( + `\n${pad('model', 32)}${pad('turns', 6)}${pad('todo', 6)}${pad('asked', 7)}` + + `${pad('rounds', 8)}${pad('sub', 5)}${pad('kept', 6)}${pad('first', 9)}` + + `${pad('whole', 9)}${pad('ratio', 8)}total` +); + +/** The models that sent a subagent for the codename. The floor is that one did. */ +const fetched: string[] = []; + +for (const { model, got } of rows) { + under(model); + const kept = + got.reopened.toLowerCase().includes('quokka') && + got.afterSummary.toLowerCase().includes('quokka'); + console.log( + pad(model, 32) + + pad(String(got.turns.length), 6) + + pad(String(got.todos.length), 6) + + pad(String(got.asked.flat().length), 7) + + pad(String(got.rounds.length), 8) + + pad(String(got.reports.length), 5) + + pad(kept ? 'yes' : 'NO', 6) + + pad(ms(median(got.timings.map(one => one.first))), 9) + + pad(ms(median(got.timings.map(one => one.whole))), 9) + + pad(hitRatio(got.usage).toFixed(3), 8) + + `${got.seconds.toFixed(0)}s` + ); + + /* `KILO_SHOW=1` prints the conversation itself. A table says a model held it + together; only the words say what it was like to talk to. */ + if (process.env['KILO_SHOW'] === '1') { + console.log(' said ', JSON.stringify(got.said)); + console.log(' rounds', JSON.stringify(got.rounds.map(round => round.text))); + } + + /* Correctness. Every one of these is a promise the package makes. */ + wrongIf(got.refused.length > 0, `a round was refused: ${got.refused.join('; ')}`); + /* One turn may say nothing: a model that hands a call to the background has + answered by starting the work, and its words arrive in the round. Two is a + model that stopped talking to the person. */ + const silent = got.said.filter(one => one.trim() === '').length; + wrongIf(silent > 2, `${String(silent)} of the answers carried no text`); + wrongIf( + got.todos.length < 3, + `the plan was written down as ${String(got.todos.length)} steps, not three or more` + ); + const questions = got.asked.flat(); + wrongIf( + got.asked.length !== 1, + `the person was interrupted ${String(got.asked.length)} times, not asked once` + ); + wrongIf( + questions.length < 2, + `the one call asked ${String(questions.length)} things, not the two it was given` + ); + wrongIf( + questions.some(one => one.choices === undefined), + 'a question was asked without the choices it was given' + ); + wrongIf( + got.rounds.length === 0, + 'the session ran no rounds of its own, though a message waited while it was busy' + ); + wrongIf( + !got.rounds.some(round => round.answering.includes(got.queuedId)), + 'the message typed while the session was busy was never answered' + ); + wrongIf( + !got.rounds.some(round => round.text.toLowerCase().includes('pelican')), + 'the answer to the queued message never carried its word' + ); + /* Whether a model hands the lookup down at all is the model's choice, and + `pnpm test:e2e:tool-matrix` is the run that scores it. How many it hands + down is its choice too: `z-ai/glm-5.3-flash` sent a second subagent of its + own on 2026-09-06, and that one was never sent for the codename. What is + asserted here is the package's half: the subagent that was sent for the + codename came back with it, and it reached the parent's conversation. */ + const brought = got.reports.filter(report => report.said.toLowerCase().includes(codename)); + if (brought.length > 0) { + fetched.push(model); + } + /* Read out of the parent's own transcript rather than out of its words. + Carrying the answer back into the conversation is what the package + promises; repeating it to the person is the model's own manner. */ + const transcript = got.turns + .flatMap(turn => turn.parts.map(part => part.body)) + .join(' ') + .toLowerCase(); + wrongIf( + brought.length > 0 && !transcript.includes(codename), + "the subagent came back with the codename and it never reached the parent's conversation" + ); + wrongIf( + brought[0]?.usage.outputTokens === 0, + 'the subagent reported no counts, so a caller adding up the conversation would be short' + ); + wrongIf( + !got.reopened.toLowerCase().includes('quokka'), + `the reopened session answered ${JSON.stringify(got.reopened)}, so the store lost the conversation` + ); + wrongIf(got.summaries === 0, 'compacting left no summary in the transcript'); + wrongIf( + !got.afterSummary.toLowerCase().includes('quokka'), + `after the summary the session answered ${JSON.stringify(got.afterSummary)}, so the summariser dropped what the next turn needed` + ); + const read = got.cloneUsage?.cacheReadTokens ?? 0; + const written = got.cloneUsage?.cacheWriteTokens ?? 0; + wrongIf(read === 0, 'the clone read nothing from the cache, so it paid for the prefix twice'); + wrongIf( + written > read / 4, + `the clone wrote ${String(written)} against ${String(read)} read, so it did not inherit the prefix` + ); + + /* Performance. Generous, and about this package rather than the provider. */ + wrongIf( + median(got.timings.map(one => one.first)) > slowestFirstWord, + `the median first word took ${ms(median(got.timings.map(one => one.first)))}, over ${ms(slowestFirstWord)}` + ); + wrongIf( + got.seconds > slowestConversation, + `the conversation took ${got.seconds.toFixed(0)}s, over ${String(slowestConversation)}s` + ); + wrongIf( + hitRatio(got.usage) < 0.3, + `the conversation read ${hitRatio(got.usage).toFixed(3)} of its prompt from the cache, under 0.3` + ); +} + +under(''); +console.log( + `\nsent a subagent for the codename: ${String(fetched.length)} of ${String(models.length)} models` +); +/* The floor under the count: a subagent that never came back with what it was + sent for, on any model, is the package and not eleven models each deciding. */ +wrongIf( + fetched.length === 0, + 'not one subagent came back with the codename, so the handing down was never tested' +); + +passed( + 'every model held one whole conversation: tools, a person, a subagent, the store, and a summary' +); diff --git a/packages/harness-sdk/e2e/image.ts b/packages/harness-sdk/e2e/image.ts new file mode 100644 index 0000000000..9861e5aca1 --- /dev/null +++ b/packages/harness-sdk/e2e/image.ts @@ -0,0 +1,170 @@ +/** + * Proves an image reaches the model, in every shape, and survives the replay. + * + * The three shapes render an image three different ways, and until this run + * each had only ever been checked against a fake `fetch`. Every shape is given + * a different colour, so a model that never saw the picture would have to guess + * three specific words to pass. + * + * The second question is the one that matters. It asks about the background, + * which the first answer never mentioned, so it can only be answered from the + * picture itself. An assembler that dropped the image from the second request + * would leave the model with its own earlier word and nothing else. + */ +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { Effect } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import { said } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import type { PartDraft } from '../src/core/turn.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { kilo, models, room } from './setup.js'; +import { fail, passed, under, wrongIf } from './report.js'; + +const system = + 'You look at pictures and answer about them. ' + + 'Answer with one lowercase word and nothing else. ' + + 'Do not explain. Do not add punctuation. Never say you cannot see an image.'; + +/** A different colour per shape, so one lucky guess cannot pass the run. */ +const shapes: readonly { readonly kind: ApiKind; readonly colour: string }[] = [ + { kind: 'messages', colour: 'green' }, + { kind: 'responses', colour: 'blue' }, + { kind: 'chat_completions', colour: 'yellow' }, +]; + +/** A caller holds a file, not base64. This is the line every caller writes. */ +const pictureOf = async (colour: string): Promise => ({ + kind: 'image', + media: 'image/png', + body: (await readFile(join(import.meta.dirname, 'images', `${colour}-circle.png`))).toString( + 'base64' + ), +}); + +const say = (session: SessionHandle, input: string | readonly PartDraft[]) => + said(session.ask(input)); + +const runShape = async (model: string, kind: ApiKind, colour: string) => { + const layers = kilo({ apiKinds: [kind] }); + + const picture = await pictureOf(colour); + + const program = Effect.gen(function* () { + /* Room to spare. A reasoning model spends its thinking out of this, and a + tight ceiling starved the second answer to nothing on two shapes. The + ceiling is what `stop.ts` tests; this run tests the picture. */ + const session = yield* openSession({ system, model, maxTokens: room }); + const named = yield* say(session, [ + { kind: 'text', body: 'What colour is the circle in this picture?' }, + picture, + ]); + /* Nothing said so far names the background, so this needs the picture. */ + const background = yield* say( + session, + 'Is the background of that picture white or black? Answer white or black.' + ); + return { named, background }; + }); + + return Effect.runPromise(Effect.either(Effect.scoped(Effect.provide(program, layers)))); +}; + +const word = (said: string) => said.toLowerCase().replaceAll(/[^a-z]/gu, ''); + +/** + * Whether the gateway refused the call because the model has no eyes. + * + * Read from the refusal rather than a list of names: the model list changes and + * a list of blind ones written here would rot silently, passing the run by + * asking a model nothing at all. Measured on 2026-09-06, + * `nvidia/nemotron-3.5-lightning` refuses every shape with a 405 saying so. + */ +const cannotSee = (error: unknown): boolean => + JSON.stringify(error).includes('does not accept image input'); + +/** The models that read the pictures, so the run can say it proved something. */ +const saw: string[] = []; + +for (const model of models) { + under(model); + + console.log('model', model); + console.log('\nshape sent named background'); + + let blind = false; + /* What each shape named and what its background was, so the run can tell a + model that cannot read a picture from a package that dropped one. */ + const read: { readonly named: string; readonly colour: string; readonly background: string }[] = + []; + for (const { kind, colour } of shapes) { + /* Tried once more before it counts. Measured on 2026-09-06, `tencent/hy3` + and `deepseek/deepseek-v4-flash` named every circle red on one sweep and + named all three right on the next, with nothing here changed between + them: that is a relay having a bad minute. Twice is a finding. */ + const first = await runShape(model, kind, colour); + const result = + first._tag === 'Right' && word(first.right.named) === colour + ? first + : await runShape(model, kind, colour); + if (result._tag === 'Left') { + if (cannotSee(result.left)) { + console.log(`${kind.padEnd(18)}${colour.padEnd(10)}the model takes no pictures`); + blind = true; + continue; + } + console.log(`${kind.padEnd(18)}${colour.padEnd(10)}FAILED ${JSON.stringify(result.left)}`); + fail(`${kind}: the call failed`); + continue; + } + + const named = word(result.right.named); + const background = word(result.right.background); + console.log(`${kind.padEnd(18)}${colour.padEnd(10)}${named.padEnd(10)}${background}`); + read.push({ named, colour, background }); + } + + /* A model that named not one of the three cannot read a picture, whatever the + gateway lets through: `tencent/hy3` and `deepseek/deepseek-v4-flash` answer + "red" to a green, a blue and a yellow circle on 2026-09-06, twice over. + Three shapes wrong in three different colours is one model's eyes; a + package that dropped the picture would put every model here at once, which + is what the floor below catches. */ + if (read.length > 0 && !read.some(one => one.named === one.colour)) { + console.log('the model named none of the three, so it cannot read a picture'); + continue; + } + + for (const { named, colour, background } of read) { + if (named !== colour) { + fail(`the picture was ${colour} and the model said ${JSON.stringify(named)}`); + } + if (background === '') { + /* It said nothing, which says nothing about the picture. A request that + lost the picture leaves the model with its own earlier word and it says + so — measured, `i don't see any image attached to your message`, which + is caught below. `minimax/minimax-m3` spends the whole answer thinking + and says nothing at all on 2026-09-06. */ + console.log('the model said nothing about the background'); + continue; + } + if (background !== 'white') { + fail( + `the background is white and the model said ${JSON.stringify(background)}, ` + + 'so the picture did not survive into the second request' + ); + } + } + if (!blind) { + saw.push(model); + } +} + +under(''); +console.log(`\nread the pictures: ${String(saw.length)} of ${String(models.length)} models`); +/* The floor under both skips: a package that stopped sending pictures, or sent + the same one three times, would put every model here at once. */ +wrongIf(saw.length === 0, 'not one model read a picture, so nothing here sent one'); + +passed('every shape carried the picture to a model with eyes, and every shape replayed it.'); diff --git a/packages/harness-sdk/e2e/images/blue-circle.png b/packages/harness-sdk/e2e/images/blue-circle.png new file mode 100644 index 0000000000..90c1d32b65 Binary files /dev/null and b/packages/harness-sdk/e2e/images/blue-circle.png differ diff --git a/packages/harness-sdk/e2e/images/green-circle.png b/packages/harness-sdk/e2e/images/green-circle.png new file mode 100644 index 0000000000..0ec5684be5 Binary files /dev/null and b/packages/harness-sdk/e2e/images/green-circle.png differ diff --git a/packages/harness-sdk/e2e/images/yellow-circle.png b/packages/harness-sdk/e2e/images/yellow-circle.png new file mode 100644 index 0000000000..7fbb3c34b7 Binary files /dev/null and b/packages/harness-sdk/e2e/images/yellow-circle.png differ diff --git a/packages/harness-sdk/e2e/live.ts b/packages/harness-sdk/e2e/live.ts new file mode 100644 index 0000000000..eeb8f4ac88 --- /dev/null +++ b/packages/harness-sdk/e2e/live.ts @@ -0,0 +1,83 @@ +import { Effect, Stream } from 'effect'; +import { openSession } from '../src/core/run.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import type { ModelUsage } from '../src/core/model.js'; +import { hitRatio } from '../src/core/usage.js'; +import { cachedSystem as system, kilo, models, room } from './setup.js'; +import { fail, passed, wrongIf } from './report.js'; + +interface Answer { + readonly said: string; + readonly usage: ModelUsage | undefined; +} + +const ask = (session: SessionHandle, text: string) => + Stream.runFold(session.ask(text), { said: '', usage: undefined } as Answer, (held, event) => + event.kind === 'delta' + ? { ...held, said: held.said + event.text } + : event.kind === 'done' + ? { ...held, usage: event.usage } + : held + ); + +const program = (model: string) => + Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens: room }); + const first = yield* ask(session, 'Answer with the word: one'); + const second = yield* ask(session, 'Answer with the word: two'); + return { id: session.id, first, second, total: yield* session.usage }; + }); + +const layers = kilo(); + +/** One whole run of the model. */ +const attempt = (model: string) => + Effect.runPromise(Effect.scoped(Effect.provide(program(model), layers))); + +/** The models whose provider made the prefix readable. The floor is that one did. */ +const cached: string[] = []; + +for (const model of models) { + /* Tried once more before it counts. A provider makes an entry readable when + it chooses to, and about one run in five it has not by the second call: + measured, the same model reads the prefix back unchanged on a rerun. Twice + is a finding. */ + const first = await attempt(model); + const result = (first.second.usage?.cacheReadTokens ?? 0) > 0 ? first : await attempt(model); + + console.log('session ', result.id); + console.log('model ', model); + console.log('first ', JSON.stringify(result.first.said), result.first.usage); + console.log('second ', JSON.stringify(result.second.said), result.second.usage); + console.log('cumulative', result.total, 'hit ratio', hitRatio(result.total).toFixed(4)); + + wrongIf(result.first.said.length === 0, `${model}: the first answer carried no text`); + wrongIf(result.second.said.length === 0, `${model}: the second answer carried no text`); + + const second = result.second.usage; + if (second === undefined) { + fail(`${model}: the second answer carried no token counts`); + continue; + } + if (second.cacheReadTokens === 0) { + /* Nothing was made readable, twice over, so there is no prefix here to hold + to a ratio. A package that stopped sending the prefix would put every + model here at once, which the floor below catches. */ + console.log('the provider read nothing back, twice'); + continue; + } + cached.push(model); + /* Half, not all. Haiku reads back over 0.99 of the prefix and glm reads 0.61 + of the same conversation, because a provider caches at a granularity of its + own. What the run is defending is that the prefix was read rather than + built again, and a floor every provider clears still says that. */ + wrongIf( + hitRatio(second) <= 0.5, + `${model}: the cache hit ratio was ${hitRatio(second).toFixed(4)}, which is not above 0.5` + ); +} + +console.log(`\nread the prefix back: ${String(cached.length)} of ${String(models.length)} models`); +wrongIf(cached.length === 0, 'not one model read the prefix back, so nothing here sent one'); + +passed('every model whose provider cached it read the prefix back'); diff --git a/packages/harness-sdk/e2e/models.ts b/packages/harness-sdk/e2e/models.ts new file mode 100644 index 0000000000..113e158d55 --- /dev/null +++ b/packages/harness-sdk/e2e/models.ts @@ -0,0 +1,158 @@ +import { Effect, Stream } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import type { Effort, ModelUsage } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { hitRatio } from '../src/core/usage.js'; +import { cachedSystem as system, kilo, models, room } from './setup.js'; + +/** + * The same five questions, put to every model in the list. + * + * The list is one model unless the run is asked for `full` — see `e2e/setup.ts` + * — so this is a cheap check by default and the vendor sweep on request. + * + * A reasoning model spends the budget on reasoning before it writes a word. At + * 64 tokens four of the eleven answered nothing at all, which reads as a broken + * transport and is not one. + */ +const maxTokens = Number(process.env['KILO_MAX_TOKENS'] ?? String(room)); +const effort = process.env['KILO_EFFORT'] as Effort | undefined; + +/** The last question can only be answered from the history of the session. */ +const questions = [ + 'Remember the word pineapple. Answer with the word: ok', + 'Answer with the word: one', + 'Answer with the word: two', + 'Answer with the word: three', + 'Which word did I ask you to remember? Answer with that one word.', +] as const; + +interface Answer { + readonly said: string; + readonly usage: ModelUsage | undefined; + /** Milliseconds from the question to the first piece of the answer. */ + readonly first: number; + /** Milliseconds from the question to the end of the answer. */ + readonly whole: number; +} + +const blank: Answer = { said: '', usage: undefined, first: 0, whole: 0 }; + +/** + * Times the answer as a caller sees it. The clock starts when `ask` is called, + * not when the request leaves, so assembling the prompt is counted too: that + * is the part of the wait this package can do something about. + */ +const ask = (session: SessionHandle, text: string) => + Effect.suspend(() => { + const started = performance.now(); + return Stream.runFold(session.ask(text), blank, (held, event) => + event.kind === 'delta' + ? { + ...held, + said: held.said + event.text, + first: held.first === 0 ? performance.now() - started : held.first, + } + : event.kind === 'done' + ? { ...held, usage: event.usage, whole: performance.now() - started } + : held + ); + }); + +const converse = (model: string, kinds: readonly ApiKind[]) => + Effect.scoped( + Effect.gen(function* () { + const session = yield* openSession({ + system, + model, + maxTokens, + ...(effort === undefined ? {} : { effort }), + }); + const answers: Answer[] = []; + for (const question of questions) { + answers.push(yield* ask(session, question)); + } + return { + answers, + turns: (yield* session.history).length, + total: yield* session.usage, + }; + }) + ).pipe(Effect.provide(kilo({ apiKinds: kinds }))); + +const preferred: readonly ApiKind[] = (process.env['KILO_KINDS']?.split(',') as + | ApiKind[] + | undefined) ?? ['messages', 'responses', 'chat_completions']; + +/** Tries the best shape first. A model whose provider rejects it falls back. */ +const run = (model: string) => + converse(model, preferred).pipe( + Effect.map(result => ({ model, kind: preferred[0] ?? 'messages', result })), + Effect.catchAll(first => + converse(model, ['chat_completions']).pipe( + Effect.map(result => ({ model, kind: 'chat_completions' as ApiKind, result })), + Effect.catchAll(second => + Effect.succeed({ model, kind: 'failed' as const, errors: [first, second] }) + ) + ) + ) + ); + +const outcomes = await Effect.runPromise(Effect.forEach(models, run, { concurrency: 3 })); + +const pad = (text: string, width: number) => text.padEnd(width); +const ms = (taken: number) => `${taken.toFixed(0)}ms`; + +/** The middle answer, so one slow call does not stand for the whole run. */ +const median = (taken: readonly number[]) => + taken.toSorted((a, b) => a - b)[Math.floor(taken.length / 2)] ?? 0; + +console.log( + `\n${pad('model', 34)}${pad('shape', 17)}${pad('recalled', 9)}` + + `${pad('first', 8)}${pad('whole', 8)}${pad('cache read', 11)}${pad('input', 8)}ratio` +); + +let broken = 0; +for (const outcome of outcomes) { + if (outcome.kind === 'failed') { + broken += 1; + console.log( + `${pad(outcome.model, 34)}${pad('FAILED', 17)}${JSON.stringify(outcome.errors[0])}` + ); + continue; + } + const { answers, turns, total } = outcome.result; + const recalled = /pineapple/iu.test(answers.at(-1)?.said ?? '') ? 'yes' : 'no'; + const empty = answers + .map((answer, index) => (answer.said.trim() === '' ? index : -1)) + .filter(index => index >= 0); + if (empty.length > 0) { + broken += 1; + console.log(` ${outcome.model}: empty answers at turns ${empty.join(', ')}`); + console.log(` said: ${JSON.stringify(answers.map(answer => answer.said))}`); + } + if (turns !== questions.length * 2) { + broken += 1; + console.log( + ` ${outcome.model}: kept ${String(turns)} turns, not ${String(questions.length * 2)}` + ); + } + console.log( + pad(outcome.model, 34) + + pad(outcome.kind, 17) + + pad(recalled, 9) + + pad(ms(median(answers.map(answer => answer.first))), 8) + + pad(ms(median(answers.map(answer => answer.whole))), 8) + + pad(String(total.cacheReadTokens), 11) + + pad(String(total.inputTokens), 8) + + hitRatio(total).toFixed(4) + ); +} + +console.log( + `\n${String(models.length - broken)} of ${String(models.length)} models answered every turn.` +); +if (broken > 0) { + process.exitCode = 1; +} diff --git a/packages/harness-sdk/e2e/plugins-check.ts b/packages/harness-sdk/e2e/plugins-check.ts new file mode 100644 index 0000000000..f99db8b756 --- /dev/null +++ b/packages/harness-sdk/e2e/plugins-check.ts @@ -0,0 +1,118 @@ +/* Every code block in PLUGINS.md, with the package name resolved to this + source tree. It is typechecked, never run: a plugin author copies these, so + one that does not compile is worse than no example at all. */ +import { Effect, Layer, Option, Schedule, Stream } from 'effect'; +/* PLUGINS.md imports these from `@kilocode/harness-sdk/testing`, which is this + file. They are out of the main entry so a consumer does not bundle them. */ +import { checkAssembler, checkStore } from '../src/core/conformance.js'; +import { EntropySource } from '../src/core/entropy.js'; +import { ModelCatalog } from '../src/core/catalog.js'; +import { ModelClient, type ModelEvent, zeroUsage } from '../src/core/model.js'; +import { PromptAssembler, type PromptPart } from '../src/core/prompt.js'; +import { RetryPolicy } from '../src/core/retry.js'; +import { SessionStore, type StoredSession } from '../src/core/storage.js'; +import { TokenError, TokenSource } from '../src/core/token.js'; +import { ToolRegistry, type Tool } from '../src/core/tool.js'; +import type { Turn, TurnPart } from '../src/core/turn.js'; + +/* Wiring one in. */ + +export const layerEcho = Layer.succeed(ModelClient, { + stream: request => + Stream.fromIterable([ + { kind: 'delta', text: `you said ${String(request.prompt.messages.length)} things` }, + { kind: 'done', usage: zeroUsage, stop: 'end' }, + ]), +}); + +/* A store that keeps everything in memory. */ + +export const layerMemory = Layer.sync(SessionStore, () => { + const sessions = new Map(); + const turns = new Map(); + return { + create: session => Effect.sync(() => void sessions.set(session.id, session)), + read: id => Effect.sync(() => Option.fromNullable(sessions.get(id))), + append: ({ sessionId, turns: added, prompted }) => + Effect.sync(() => { + turns.set(sessionId, [...(turns.get(sessionId) ?? []), ...added]); + const held = sessions.get(sessionId); + if (held !== undefined) { + sessions.set(sessionId, { ...held, prompted }); + } + }), + load: id => Effect.sync(() => turns.get(id) ?? []), + flush: () => Effect.void, + }; +}); + +/* An assembler, on a transcript of text only. */ + +const renderPart = (part: TurnPart): readonly PromptPart[] => + part.kind === 'text' ? [{ kind: 'text', text: part.body }] : []; + +export const layerPlain = Layer.succeed(PromptAssembler, { + assemble: ({ system, turns }) => ({ + system: [{ text: system, cache: true }], + messages: turns.map((turn, at) => ({ + role: turn.role, + parts: turn.parts.flatMap(renderPart), + cache: at === turns.length - 1, + })), + }), +}); + +/* A catalog that answers for every model. */ + +export const layerEverything = Layer.succeed(ModelCatalog, { + facts: () => Effect.succeed({ apiKinds: ['messages'], contextWindow: 200_000 }), +}); + +/* A registry. */ + +declare const weather: Tool; + +export const layerTools = Layer.succeed(ToolRegistry, { tools: [weather] }); + +/* A credential that expires, read inside the effect. */ + +declare const mint: () => Promise<{ readonly value: string; readonly until: number }>; +let held: { readonly value: string; readonly until: number } | undefined; + +export const refreshing = { + get: () => + Effect.suspend(() => + held !== undefined && held.until > Date.now() + ? Effect.succeed(held.value) + : Effect.tryPromise({ + try: async () => { + held = await mint(); + return held.value; + }, + catch: cause => new TokenError({ cause }), + }) + ), +}; + +export const layerRefreshing = Layer.succeed(TokenSource, refreshing); + +/* A policy that gives up after three tries, and a source of bytes. */ + +export const layerThrice = Layer.succeed(RetryPolicy, { + schedule: Schedule.recurs(3).pipe(Schedule.addDelay(() => '1 second')), +}); + +export const layerCounting = Layer.sync(EntropySource, () => { + let at = 0; + return { bytes: count => Uint8Array.from({ length: count }, () => at++ % 256) }; +}); + +/* The checks a plugin author runs. This block is in the README too. */ + +export const conforms = Effect.gen(function* () { + const store = yield* SessionStore; + const assembler = yield* PromptAssembler; + const wrongInStore = yield* checkStore(store); + const wrongInAssembler = checkAssembler(assembler); + return [...wrongInStore, ...wrongInAssembler]; +}); diff --git a/packages/harness-sdk/e2e/probe.ts b/packages/harness-sdk/e2e/probe.ts new file mode 100644 index 0000000000..351e64a5cd --- /dev/null +++ b/packages/harness-sdk/e2e/probe.ts @@ -0,0 +1,103 @@ +/** + * Prints the raw frames of one call, for finding out what a shape actually + * sends. It asserts nothing and is not part of any suite: it exists so a + * question about the wire can be answered by reading the wire. + * + * `pnpm test:e2e:probe [shape] [model]` + */ +import { webFetch } from '../src/plugins/fetch/web.js'; +import { kiloToken } from './token.js'; + +const baseUrl = process.env['KILO_BASE_URL'] ?? 'https://app.kilo.ai'; +const organizationId = process.env['KILO_ORG_ID'] ?? '9d278969-5453-4ae3-a51f-a8d2274a7b56'; + +const paths = { + messages: '/api/gateway/v1/messages', + responses: '/api/gateway/v1/responses', + chat_completions: '/api/gateway/v1/chat/completions', +} as const; + +const shape = (process.argv[2] ?? 'responses') as keyof typeof paths; +const model = process.argv[3] ?? 'openai/gpt-5.6-luna'; +const question = 'A farmer has 17 sheep. All but 9 run away. How many are left?'; + +const bodies: Readonly> = { + messages: { + model, + max_tokens: 2000, + stream: true, + /* The same dial the wire sends, so a probe of this shape shows the + thinking frames rather than none. */ + output_config: { effort: 'medium' }, + messages: [{ role: 'user', content: [{ type: 'text', text: question }] }], + }, + responses: { + model, + max_output_tokens: 2000, + stream: true, + reasoning: { effort: 'medium' }, + include: ['reasoning.encrypted_content'], + store: false, + input: [{ role: 'user', content: [{ type: 'input_text', text: question }] }], + }, + chat_completions: { + model, + max_tokens: 2000, + stream: true, + reasoning: { effort: 'medium' }, + messages: [{ role: 'user', content: [{ type: 'text', text: question }] }], + }, +}; + +const response = await webFetch(`${baseUrl}${paths[shape]}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${await kiloToken()}`, + 'x-kilocode-organizationid': organizationId, + }, + body: JSON.stringify(bodies[shape]), +}); + +console.log(shape, model, response.status); +if (!response.ok || response.stream === undefined) { + console.log(await response.text()); + process.exit(1); +} + +/** Only the frame names and the fields that are not the answer's text. */ +const seen = new Map(); +let held = ''; +for await (const chunk of response.stream()) { + held += chunk; + const lines = held.split('\n'); + held = lines.pop() ?? ''; + for (const line of lines) { + if (!line.startsWith('data: ')) { + continue; + } + const data = line.slice(6); + if (data === '[DONE]') { + continue; + } + const event: unknown = JSON.parse(data); + const named = event as { type?: string; error?: unknown; response?: { error?: unknown } }; + const name = named.type ?? 'unnamed'; + seen.set(name, (seen.get(name) ?? 0) + 1); + /* The gateway fails a stream on a frame carrying an `error` object, at the + top or inside `response`. Printing them here is how to tell a real + failure from a shape that puts the field on a frame that succeeded. */ + const failed = named.error ?? named.response?.error; + if (failed !== undefined && failed !== null) { + console.log('ERROR FRAME', JSON.stringify(event).slice(0, 300)); + } + if (name.includes('reasoning') || name.includes('item') || name === 'unnamed') { + console.log(name, JSON.stringify(event).slice(0, Number(process.env['PROBE_WIDTH'] ?? 300))); + } + } +} + +console.log('\nframes:'); +for (const [name, count] of [...seen].sort()) { + console.log(' ', String(count).padStart(4), name); +} diff --git a/packages/harness-sdk/e2e/queue.ts b/packages/harness-sdk/e2e/queue.ts new file mode 100644 index 0000000000..f68f933ffd --- /dev/null +++ b/packages/harness-sdk/e2e/queue.ts @@ -0,0 +1,242 @@ +/** + * Proves the line: a message handed over while the session is busy, the one + * taken back before it is said, and the order both are answered in. + * + * A fake proves the line forms. Only the provider proves the line is a + * conversation: a queued message is answered from the same transcript, in the + * order it joined, and the answer names which message it answers. That is the + * whole promise of `queue`, and none of it can be seen without a model that + * remembers what it just said. + * + * One session, three questions: + * + * - **Handed over while busy.** Two messages are queued from inside the first + * answer's own stream, which is the only moment the session is certainly + * held. Neither refuses. + * - **Taken back.** A third is queued between them and cancelled while it is + * still waiting. It is never asked, and cancelling it twice says so. + * - **Answered in order, from the transcript.** The first queued message asks + * the model to repeat the word it just said, so an answer carrying that word + * proves the round ran in this session and not beside it. + * - **Written down like any other.** The session is reopened from SQLite at the + * end, and holds all three exchanges. A round the session ran on its own is + * an exchange, so a caller who closes the app loses none of it. + */ +import { DatabaseSync } from 'node:sqlite'; +import { Effect, Layer, Stream } from 'effect'; +import type { SessionHandle } from '../src/core/handle.js'; +import type { ModelEvent } from '../src/core/model.js'; +import type { Waiting } from '../src/core/queue.js'; +import { continueSession } from '../src/core/resume.js'; +import { openSession } from '../src/core/run.js'; +import { layerNodeStore } from '../src/plugins/store/node.js'; +import type { Turn } from '../src/core/turn.js'; +import { kilo, models, room } from './setup.js'; +import { fail, passed, under, wrongIf } from './report.js'; +import { refused, watch } from './rounds.js'; + +/* Terse, and told to repeat itself when asked. The transcript test turns on + the model saying a word back, and a prompt that forbids it would fail the + run for the prompt rather than for the line. */ +const system = + 'You are a test harness. Answer every message with one lowercase word and ' + + 'nothing else. If the user asks for the word you last answered, answer with ' + + 'that same word again.'; + +const opening = "Answer with the word 'ferret'."; +const repeating = 'Answer with the word you last answered.'; +const dropped = "Answer with the word 'pangolin'."; +const closing = "Answer with the word 'badger'."; + +const textOf = (waiting: Waiting): string => + waiting.parts.map(part => (part.kind === 'text' ? part.body : '')).join(''); + +/** + * Everything the caller learns while the first answer is still arriving. It is + * gathered there because that is the one moment the session is certainly busy, + * which is the moment `queue` exists for. + */ +interface Handed { + readonly first: string; + readonly second: string; + readonly waiting: readonly Waiting[]; + readonly tookBack: boolean; + readonly twice: boolean; +} + +const handOver = (session: SessionHandle) => + Effect.gen(function* () { + const first = yield* session.queue(repeating); + const drop = yield* session.queue(dropped); + const second = yield* session.queue(closing); + /* Read the line before anything leaves it: this is what a caller draws. */ + const waiting = yield* session.queued; + const tookBack = yield* session.cancel(drop); + /* A caller pressing cancel twice is ordinary, and the second is not an error. */ + const twice = yield* session.cancel(drop); + return { first, second, waiting, tookBack, twice }; + }); + +/** Asks the opening question, and hands two more over from inside its stream. */ +const askAndHand = (session: SessionHandle) => + Effect.gen(function* () { + const held = { text: '', handed: undefined as Handed | undefined }; + const onEvent = (event: ModelEvent) => + Effect.gen(function* () { + if (event.kind === 'delta') { + held.text += event.text; + } + /* Once, on the first event of the stream: from here the session is + certainly held, which is the state `queue` is for. */ + if (held.handed === undefined) { + held.handed = yield* handOver(session); + } + }); + yield* Stream.runForEach(session.ask(opening), onEvent); + return { said: held.text, handed: held.handed }; + }); + +const program = (model: string) => + Effect.gen(function* () { + const session = yield* openSession({ + system, + model, + maxTokens: room, + }); + /* Watch first. The rounds run whether or not anybody listens. */ + const watching = yield* watch(session, 2, '120 seconds'); + const { said, handed } = yield* askAndHand(session); + const rounds = yield* watching.done; + /* Reopened from the store, which is the only place the rounds the session ran + on its own could have gone. */ + const reopened = yield* continueSession(session.id); + return { + said, + handed, + rounds, + left: yield* session.queued, + history: yield* session.history, + stored: yield* reopened.history, + }; + }); + +/** One whole run of the model, on a store of its own, or the reason it failed. */ +const attempt = (model: string) => + Effect.runPromise( + Effect.either( + Effect.scoped( + Effect.provide( + program(model), + Layer.merge(kilo(), layerNodeStore(new DatabaseSync(':memory:'))) + ) + ) + ) + ); + +/** Whether the two rounds the session ran on its own said what they were asked. */ +const spoke = (rounds: readonly { readonly text: string }[]): boolean => + (rounds[0]?.text ?? '').toLowerCase().includes('ferret') && + (rounds[1]?.text ?? '').toLowerCase().includes('badger'); + +for (const model of models) { + under(model); + + /* Tried once more before it counts. Measured on 2026-09-06, + `nvidia/nemotron-3.5-lightning` ran both rounds, named the right message on + each and stored all six turns, and said nothing at all in the second: the + line worked and the model went quiet. Twice is a finding. */ + const first = await attempt(model); + const tried = first._tag === 'Right' && spoke(first.right.rounds) ? first : await attempt(model); + if (tried._tag === 'Left') { + /* Nothing was recorded for the first attempt, so a relay having a bad + minute costs a retry and not the run. Twice is a finding. */ + console.log('model', model, 'FAILED', JSON.stringify(String(tried.left))); + fail(`the run failed twice: ${String(tried.left)}`); + continue; + } + const got = tried.right; + + const { handed } = got; + const rounds = got.rounds; + const wordsIn = (turns: readonly Turn[]): readonly string[] => + turns + .filter(turn => turn.role === 'user') + .map(turn => turn.parts.map(part => part.body).join('')); + + const spoken = wordsIn(got.history); + + console.log('model', model); + console.log(`\nasked while free: ${JSON.stringify(got.said.trim())}`); + for (const [at, round] of rounds.entries()) { + console.log(`round ${String(at + 1)} answered: ${JSON.stringify(round.text.trim())}`); + } + console.log(`\nwaiting while busy: ${JSON.stringify((handed?.waiting ?? []).map(textOf))}`); + console.log(`took one back: ${String(handed?.tookBack)}, and again: ${String(handed?.twice)}`); + console.log(`left in the line: ${String(got.left.length)}`); + console.log(`what the session was asked: ${JSON.stringify(spoken)}`); + console.log(`turns held by the store: ${String(got.stored.length)}`); + + if (handed === undefined) { + fail('the first answer streamed no event, so nothing was ever handed over'); + } + + wrongIf( + !got.said.toLowerCase().includes('ferret'), + 'the first answer was not the word asked for' + ); + wrongIf( + (handed?.waiting ?? []).length !== 3, + `the line held ${String((handed?.waiting ?? []).length)} messages while busy, not three` + ); + wrongIf( + JSON.stringify((handed?.waiting ?? []).map(textOf)) !== + JSON.stringify([repeating, dropped, closing]), + 'the line was not in the order it formed' + ); + wrongIf(handed?.tookBack !== true, 'cancelling a waiting message did not take it back'); + wrongIf(handed?.twice !== false, 'cancelling the same message twice said it took it back again'); + wrongIf( + rounds.length !== 2, + `the session ran ${String(rounds.length)} rounds of its own, not two` + ); + wrongIf( + !(rounds[0]?.text ?? '').toLowerCase().includes('ferret'), + 'the first queued message was not answered from this session’s transcript' + ); + wrongIf( + !(rounds[1]?.text ?? '').toLowerCase().includes('badger'), + 'the second queued message was not answered' + ); + wrongIf( + rounds[0]?.answering[0] !== handed?.first || rounds[1]?.answering[0] !== handed?.second, + 'a round did not name the message it answers' + ); + wrongIf( + spoken.some(text => text.includes('pangolin')), + 'the cancelled message was said to the model anyway' + ); + wrongIf( + JSON.stringify(spoken) !== JSON.stringify([opening, repeating, closing]), + 'the session was not asked the three messages, in the order they joined' + ); + wrongIf(got.left.length !== 0, 'the line still holds a message the session never asked'); + wrongIf( + JSON.stringify(wordsIn(got.stored)) !== JSON.stringify(spoken), + 'the session reopened from the store without the rounds it ran on its own' + ); + wrongIf( + got.stored.length !== got.history.length, + `the store held ${String(got.stored.length)} turns, not the ${String(got.history.length)} the session had` + ); + + wrongIf( + refused.length > 0, + `the session was refused ${String(refused.length)} of the rounds it ran on its own` + ); +} + +passed( + 'two messages were handed over while busy, one was taken back, and the ' + + 'rest were answered in order from the same transcript, and the store held ' + + 'every round.' +); diff --git a/packages/harness-sdk/e2e/readme-check.ts b/packages/harness-sdk/e2e/readme-check.ts new file mode 100644 index 0000000000..b459bb2be1 --- /dev/null +++ b/packages/harness-sdk/e2e/readme-check.ts @@ -0,0 +1,208 @@ +/* The README's first example, with the package name resolved to this source + tree. It is typechecked, never run: a snippet that does not compile is worse + than no snippet. */ +import { Effect, Fiber, Layer, Stream } from 'effect'; +import { openSession } from '../src/core/run.js'; +import { layerKilo } from '../src/plugins/kilo.js'; +import { continueSession, cloneSession } from '../src/core/resume.js'; +import { TokenError, type TokenSourceService } from '../src/core/token.js'; +import { type Tool, ToolRegistry } from '../src/core/tool.js'; +import { type Asker, type Question, questionTool } from '../src/plugins/tools/question.js'; +import { subagentTool } from '../src/plugins/tools/subagent.js'; +import { hitRatio } from '../src/core/usage.js'; +import type { Continued } from '../src/core/queue.js'; +import { layerNodeStore } from '../src/plugins/store/node.js'; +import { DatabaseSync } from 'node:sqlite'; +import { webFetch } from '../src/plugins/fetch/web.js'; +import { said } from '../src/core/model.js'; + +/* README, "Your fetch": the adapter the package ships. */ +const shipped = layerKilo({ + baseUrl: 'https://app.kilo.ai', + org: { kind: 'organization', id: 'org_...' }, + token: '...', + fetch: webFetch, +}); +void shipped; + +const layers = layerKilo({ + baseUrl: 'https://app.kilo.ai', + org: { kind: 'organization', id: 'org_...' }, + fetch: webFetch, + token: '...', + fallback: { apiKinds: ['messages'] }, +}); + +const program = Effect.gen(function* () { + const session = yield* openSession({ + system: 'You are terse.', + model: 'anthropic/claude-haiku-4.5', + }); + yield* Stream.runForEach(session.ask('Name three fruits.'), event => + Effect.sync(() => { + if (event.kind === 'delta') { + process.stdout.write(event.text); + } + }) + ); + /* README, "What comes back": the fold, for a caller that wants the answer. */ + const answer = yield* said(session.ask('Name three fruits.')); + void answer; + /* The README's second snippet, which reads the last event rather than only + the text. */ + yield* Stream.runForEach(session.ask('Name three fruits.'), event => + Effect.sync(() => { + if (event.kind === 'delta') { + process.stdout.write(event.text); + } + if (event.kind === 'done' && event.stop === 'maxTokens') { + process.stdout.write('\n[cut off at the token ceiling]\n'); + } + }) + ); + yield* session.compact; + console.log(hitRatio(yield* session.usage)); +}); + +const store = layerNodeStore(new DatabaseSync('sessions.db')); +export const resumed = Effect.provide( + Effect.gen(function* () { + yield* continueSession('ses_1'); + yield* cloneSession('ses_1'); + }), + Layer.mergeAll(layers, store) +); + +export const run = (): Promise => + Effect.runPromise(Effect.scoped(Effect.provide(program, layers))); + +/* The tags the README's failure table names. `catchTag` rejects a tag the + error union does not hold, so renaming one fails here rather than in a + caller's editor. */ +export const handled = Effect.gen(function* () { + const session = yield* openSession({ system: 'sys', model: 'm' }); + return yield* Stream.runDrain(session.ask('hi')).pipe( + Effect.catchTag('harness/ModelError', error => Effect.succeed(error.reason)), + Effect.catchTag('harness/StoreError', error => Effect.succeed(error.operation)), + Effect.catchTag('harness/SessionBusyError', error => Effect.succeed(error.sessionId)) + ); +}); + +export const reopened = continueSession('ses_1').pipe( + Effect.catchTag('harness/SessionNotFoundError', error => Effect.succeed(error.sessionId)) +); + +/* The README's refreshing credential. What it proves is that the cache is read + inside the effect: a `get` that reads it while building one hands the same + expired credential to every retry, and that is not something a type says. */ +declare const mintFromYourAuthServer: () => Promise<{ value: string; until: number }>; + +let held: { value: string; until: number } | undefined; + +const refreshing: TokenSourceService = { + get: () => + Effect.suspend(() => + held !== undefined && held.until > Date.now() + ? Effect.succeed(held.value) + : Effect.tryPromise({ + try: async () => { + held = await mintFromYourAuthServer(); + return held.value; + }, + catch: cause => new TokenError({ cause }), + }) + ), +}; + +export const refreshed = layerKilo({ + baseUrl: 'https://app.kilo.ai', + org: { kind: 'organization', id: 'org_...' }, + fetch: webFetch, + token: refreshing, +}); + +/* The README's stop button. */ +export const stopped = Effect.gen(function* () { + const session = yield* openSession({ system: 'sys', model: 'm' }); + const reading = yield* Effect.fork(Stream.runDrain(session.ask('Count to 300.'))); + yield* Fiber.interrupt(reading); +}); + +/* The README's tools: the registry, the names a session may use, the rounds it + runs on its own, and the question tool with an asker of the caller's own. */ +const weather: Tool = { + definition: { + name: 'weather', + description: 'The weather in one city.', + parameters: { + type: 'object', + properties: { city: { type: 'string', description: 'The city to report on.' } }, + required: ['city'], + }, + }, + run: call => Effect.succeed(`It is raining in ${String(JSON.parse(call.arguments).city)}.`), +}; + +export const withTools = Layer.merge(layers, Layer.succeed(ToolRegistry, { tools: [weather] })); + +export const asking = Effect.gen(function* () { + const session = yield* openSession({ + system: 'You are terse.', + model: 'anthropic/claude-haiku-4.5', + tools: ['weather'], + inlineFor: '5 seconds', + }); + yield* Stream.runDrain(session.ask('What is the weather in Oslo?')); +}); + +/* The README's queue: the identifier, the line, taking one back, and reading + one message's answer out of the rounds the session ran on its own. */ +/* The README's rule for knowing a queued message is answered in full, and + what a caller reads when a round was refused instead. */ +export const over = (one: Continued): boolean => + !('failed' in one) && one.event.kind === 'done' && one.event.stop !== 'tools'; + +export const queueing = Effect.gen(function* () { + const session = yield* openSession({ system: 'sys', model: 'm' }); + const id = yield* session.queue('and what about Lisbon?'); + const waiting = yield* session.queued; + const dropped = yield* session.cancel(id); + yield* Stream.runForEach(session.continued, one => + Effect.sync(() => { + if (!one.answering.includes(id)) { + return; + } + if ('failed' in one) { + process.stdout.write(`that one failed: ${String(one.failed)}`); + } else if (one.event.kind === 'delta') { + process.stdout.write(one.event.text); + } + }) + ); + return { waiting, dropped }; +}); + +declare const promptTheUser: (question: Question) => Effect.Effect; + +const ask: Asker = questions => + Effect.forEach(questions, question => + Effect.map(promptTheUser(question), text => ({ id: question.id, text })) + ); + +export const tools = [questionTool(ask)]; + +/* The README's on-demand backgrounding. */ +export const sending = Effect.gen(function* () { + const session = yield* openSession({ system: 'sys', model: 'm' }); + const waiting = yield* session.running; + const sent = yield* session.background(waiting[0]?.id ?? ''); + return sent; +}); + +/* The README's subagent, over the layers the caller already built. */ +export const withSubagent = [ + subagentTool( + { system: 'You look things up.', model: 'anthropic/claude-haiku-4.5', inlineFor: '5 seconds' }, + layers + ), +]; diff --git a/packages/harness-sdk/e2e/reasoning.ts b/packages/harness-sdk/e2e/reasoning.ts new file mode 100644 index 0000000000..891bd30a5e --- /dev/null +++ b/packages/harness-sdk/e2e/reasoning.ts @@ -0,0 +1,143 @@ +/** + * Proves the model's thinking goes back to the provider and is accepted. + * + * The unit tests prove the block survives the round trip through this package. + * They cannot prove the provider agrees, and the provider is the only judge: + * it seals the thinking and refuses a seal it cannot read. + * + * Each shape seals it differently, so each is run on its own: + * + * - `messages` signs a thinking block, and the signature travels with it. + * - `responses` hands back a reasoning item holding its own encrypted copy, + * which the request has to ask for with `include`. + * - `chat_completions` has no replay at all, so it is only checked for still + * carrying the conversation. + * + * Two questions in one session. The second request carries the first answer's + * thinking. If the seal were wrong — dropped, edited, put in the wrong place — + * the second call would fail, not answer differently. + */ +import { Effect, Stream } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import { openSession } from '../src/core/run.js'; +import type { Turn } from '../src/core/turn.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { kilo } from './setup.js'; +import { fail, passed } from './report.js'; + +const system = 'You answer briefly. Think first, then give the answer in one short sentence.'; + +/** Two questions worth thinking about, where the second follows on from the first. */ +const first = 'A farmer has 17 sheep. All but 9 run away. How many are left? Explain in one line.'; +const second = 'Now double that number and tell me the result.'; + +/** + * A model that thinks, per shape. One that does not would pass this run + * vacuously, which is why the run fails when no thinking arrives. + */ +const shapes: readonly { + readonly kind: ApiKind; + readonly model: string; + /** Whether this shape hands back something that lets the thinking be replayed. */ + readonly seals: boolean; +}[] = [ + { kind: 'messages', model: 'anthropic/claude-sonnet-4.5', seals: true }, + { kind: 'responses', model: 'anthropic/claude-sonnet-4.5', seals: true }, + /* The thinking arrives here too, but with nothing to prove it is the + model's own, so it is kept for the reader and never replayed. */ + { kind: 'chat_completions', model: 'anthropic/claude-sonnet-4.5', seals: false }, +]; + +interface Answer { + readonly said: string; + readonly thought: string; + /** What the request carried. A replayed block shows up here and nowhere else. */ + readonly input: number; +} + +const empty: Answer = { said: '', thought: '', input: 0 }; + +const ask = (session: SessionHandle, text: string) => + Stream.runFold(session.ask(text, { maxTokens: 4000 }), empty, (held, event) => { + switch (event.kind) { + case 'delta': { + return { ...held, said: held.said + event.text }; + } + case 'reasoning': { + return { ...held, thought: held.thought + event.text }; + } + case 'done': { + return { ...held, input: event.usage.inputTokens + event.usage.cacheReadTokens }; + } + case 'redacted': + case 'toolCall': + case 'toolResult': { + return held; + } + } + }); + +const runShape = async (kind: ApiKind, model: string) => { + const layers = kilo({ apiKinds: [kind] }); + + const program = Effect.gen(function* () { + const session = yield* openSession({ system, model, effort: 'medium' }); + const one = yield* ask(session, first); + const two = yield* ask(session, second); + return { one, two, history: yield* session.history }; + }); + + return Effect.runPromise(Effect.either(Effect.scoped(Effect.provide(program, layers)))); +}; + +const reasoningOf = (turn: Turn | undefined) => + turn?.parts.filter(part => part.kind === 'reasoning') ?? []; + +console.log( + 'shape model thought seal blocks input answered' +); + +for (const { kind, model, seals } of shapes) { + const result = await runShape(kind, model); + if (result._tag === 'Left') { + console.log(`${kind.padEnd(18)}${model.padEnd(29)}FAILED ${JSON.stringify(result.left)}`); + fail(`${kind}: the call failed`); + continue; + } + + const { one, two, history } = result.right; + const stored = reasoningOf(history[1]); + const seal = stored[0]?.signature; + const thought = one.thought.length > 0 || seal !== undefined; + + console.log( + `${kind.padEnd(18)}${model.padEnd(29)}${String(thought).padEnd(9)}` + + `${(seal === undefined ? 'none' : String(seal.length)).padEnd(7)}` + + `${String(stored.length).padEnd(7)}` + + `${String(two.input).padEnd(7)}` + + JSON.stringify(two.said.slice(0, 24)) + ); + + if (seals && !thought) { + fail(`${kind}: the model produced no thinking, so this shape proves nothing`); + } + /* One thinking block or several: the model decides, and a model that thinks + again after answering part way produces two. What may never happen is a + stored block without a seal — the wire drops it, so the thinking the + provider signed would go back with a hole in it. */ + const unsealed = stored.filter(part => part.signature === undefined).length; + if (seals && stored.length === 0) { + fail(`${kind}: the answer kept no thinking at all, so nothing can be replayed`); + } + if (seals && unsealed > 0) { + fail( + `${kind}: ${String(unsealed)} of ${String(stored.length)} stored thinking blocks carry ` + + 'no seal, so the wire drops them and the thinking goes back with a hole in it' + ); + } + if (two.said.length === 0) { + fail(`${kind}: the second call carried the thinking back and produced no answer`); + } +} + +passed('every shape took its own thinking back and answered on top of it.'); diff --git a/packages/harness-sdk/e2e/replay.ts b/packages/harness-sdk/e2e/replay.ts new file mode 100644 index 0000000000..c6b3735f8e --- /dev/null +++ b/packages/harness-sdk/e2e/replay.ts @@ -0,0 +1,153 @@ +/** + * Proves thinking that has been through SQLite is still thinking the provider + * accepts. + * + * `reasoning.ts` replays a block the session is still holding in memory. + * `resume.ts` carries a conversation through the store, but a plain one, with + * nothing sealed in it. Neither covers the pair: a seal written to a column, + * read back by another run, and handed to the provider that issued it. + * + * That pair is where a defect would hide. The seal is text in a nullable + * column, and it is the one value in the store that another party validates. + * A store that truncated it, reordered the parts around it, or dropped it for + * one shape would pass every unit test here, and fail only on the first + * question somebody asks after reopening a session. + * + * Each shape is run on its own, because each seals differently, and each is + * asked its second question in a second run against the same database. + */ +import { DatabaseSync } from 'node:sqlite'; +import { Effect, Layer, Stream } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import { continueSession, type ResumeContext } from '../src/core/resume.js'; +import { openSession } from '../src/core/run.js'; +import type { Turn, TurnPart } from '../src/core/turn.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { layerNodeStore } from '../src/plugins/store/node.js'; +import { kilo } from './setup.js'; +import { fail, passed } from './report.js'; + +/* Named here and not taken from the environment, as in `reasoning.ts`. What is + under test is the seal, so a model that seals nothing would report the run's + own subject missing and call it a defect. The model is part of the fixture. */ +const model = 'anthropic/claude-sonnet-4.5'; + +const system = 'You answer briefly. Think first, then give the answer in one short sentence.'; +const first = 'A farmer has 17 sheep. All but 9 run away. How many are left? Explain in one line.'; +const second = 'Now double that number and tell me the result.'; + +/** The shapes, and whether each hands back something that can be replayed. */ +const shapes: readonly { readonly kind: ApiKind; readonly seals: boolean }[] = [ + { kind: 'messages', seals: true }, + { kind: 'responses', seals: true }, + /* Nothing to replay, so what is under test here is that a stored block with + no seal does not break the request it is loaded into. */ + { kind: 'chat_completions', seals: false }, +]; + +interface Answer { + readonly said: string; + readonly thought: string; +} + +const ask = (session: SessionHandle, text: string) => + Stream.runFold(session.ask(text, { maxTokens: 4000 }), { said: '', thought: '' }, (held, event) => + event.kind === 'delta' + ? { ...held, said: held.said + event.text } + : event.kind === 'reasoning' + ? { ...held, thought: held.thought + event.text } + : held + ); + +/** + * What went wrong, in one line. A `ModelError` whose cause is an `Error` + * stringifies to `{}`, which tells a reader nothing about the run that failed. + */ +const why = (error: object): string => { + const cause = 'cause' in error ? error.cause : undefined; + return cause instanceof Error ? cause.message : JSON.stringify(error).slice(0, 300); +}; + +const reasoningIn = (turns: readonly Turn[]): readonly TurnPart[] => + turns.flatMap(turn => turn.parts.filter(part => part.kind === 'reasoning')); + +/** The seal as the store holds it, or nothing when the shape issues none. */ +const sealOf = (parts: readonly TurnPart[]): string | undefined => + parts[0]?.kind === 'reasoning' ? parts[0].signature : undefined; + +/** + * One shape, two runs, one database. The layers are built again for the second + * run, which is what a second start of an application does. + */ +const through = async (kind: ApiKind) => { + const database = new DatabaseSync(':memory:'); + const layers = Layer.mergeAll(kilo({ apiKinds: [kind] }), layerNodeStore(database)); + const run = (use: Effect.Effect): Promise => + Effect.runPromise(Effect.scoped(Effect.provide(use, layers))); + + const opened = await run( + Effect.gen(function* () { + const session = yield* openSession({ system, model, effort: 'medium' }); + const answer: Answer = yield* ask(session, first); + return { id: session.id, answer, parts: reasoningIn(yield* session.history) }; + }) + ); + + const reopened = await Effect.runPromise( + Effect.either( + Effect.scoped( + Effect.provide( + Effect.gen(function* () { + const session = yield* continueSession(opened.id); + const loaded = reasoningIn(yield* session.history); + const answer: Answer = yield* ask(session, second); + return { loaded, answer }; + }), + layers + ) + ) + ) + ); + + return { opened, reopened }; +}; + +console.log('shape seal written seal read answered'); + +for (const { kind, seals } of shapes) { + const { opened, reopened } = await through(kind); + const written = sealOf(opened.parts); + + if (reopened._tag === 'Left') { + console.log(`${kind.padEnd(18)}${String(written?.length ?? 'none').padEnd(14)}FAILED`); + /* The whole point of the run. A seal the provider will not take back is a + session that cannot be continued at all. */ + fail(`${kind}: the reopened session was refused: ${why(reopened.left)}`); + continue; + } + + const read = sealOf(reopened.right.loaded); + console.log( + `${kind.padEnd(18)}${String(written?.length ?? 'none').padEnd(14)}` + + `${String(read?.length ?? 'none').padEnd(11)}` + + JSON.stringify(reopened.right.answer.said.slice(0, 28)) + ); + + if (seals && written === undefined) { + fail(`${kind}: nothing was sealed, so this shape proves nothing`); + } + if (read !== written) { + fail(`${kind}: the seal read back is not the seal that was written`); + } + if (seals && reopened.right.loaded.length !== opened.parts.length) { + fail( + `${kind}: the store gave back ${String(reopened.right.loaded.length)} reasoning parts ` + + `where ${String(opened.parts.length)} were written` + ); + } + if (reopened.right.answer.said.length === 0) { + fail(`${kind}: the reopened session answered with nothing`); + } +} + +passed('every shape took back thinking that had been through the store.'); diff --git a/packages/harness-sdk/e2e/report.ts b/packages/harness-sdk/e2e/report.ts new file mode 100644 index 0000000000..030133a927 --- /dev/null +++ b/packages/harness-sdk/e2e/report.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; + +/** + * How every live run says what held and what did not. + * + * A run gathers what is wrong and says all of it at the end, rather than + * throwing on the first thing. That is deliberate: these runs cost money and + * minutes, and learning one failure per run turns an afternoon into a week. + * + * Fifteen runs wrote these lines each. The shared copy also settles the + * wording, which matters more than it sounds: a reader comparing two failing + * runs should not have to work out whether two differently-phrased reports mean + * the same thing. + */ + +/** What is wrong so far. Empty at the end is the run passing. */ +const failures: string[] = []; + +/** + * The model whatever fails next was working on. + * + * A run works through a list of models, and a report that says "the answer + * carried no text" without saying whose is a report nobody can act on. Every + * run calls `under` before each model, and the name is put on what it records. + */ +let scope = ''; + +/** Names the model the checks after this belong to. */ +const under = (model: string): void => { + scope = model; +}; + +/** Records one thing that is wrong, and carries on. */ +const fail = (why: string): void => { + failures.push(scope === '' ? why : `${scope}: ${why}`); +}; + +/** Records one thing that is wrong if it is wrong, and carries on either way. */ +const wrongIf = (broken: boolean, why: string): void => { + if (broken) { + fail(why); + } +}; + +/** + * Ends the run: throws with everything that was wrong, or says what held. + * + * Give `what` in the past tense and without a full stop — it is printed after + * "PASS: ", and it is the one line `e2e/all.ts` shows for a run that passed. + */ +const passed = (what: string): void => { + assert.equal(failures.length, 0, `\n ${failures.join('\n ')}\n`); + console.log(`\nPASS: ${what}`); +}; + +export { fail, failures, passed, under, wrongIf }; diff --git a/packages/harness-sdk/e2e/resume.ts b/packages/harness-sdk/e2e/resume.ts new file mode 100644 index 0000000000..866922cc4f --- /dev/null +++ b/packages/harness-sdk/e2e/resume.ts @@ -0,0 +1,154 @@ +/** + * Proves a reopened session knows how full it is, against a real gateway. + * + * The unit tests script the token counts, so they prove the column is written + * and read. They cannot prove the number written is the one the provider + * actually reported. This run reads the count off the `done` event, reads the + * same session back out of SQLite, and compares them. + * + * Then it uses that number to pin both directions. A window the count fills + * must make the reopened session compact before its first question; a window + * ten times larger must leave it alone. Both windows come from the count the + * provider gave, so neither depends on a guess about how a model tokenises. + */ +import { DatabaseSync } from 'node:sqlite'; +import { Effect, Layer, Option, Stream } from 'effect'; +import { continueSession, type ResumeContext } from '../src/core/resume.js'; +import { openSession } from '../src/core/run.js'; +import { SessionStore } from '../src/core/storage.js'; +import type { Turn } from '../src/core/turn.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { layerNodeStore } from '../src/plugins/store/node.js'; +import { everyShape, kilo, models, room } from './setup.js'; +import { fail, passed, under } from './report.js'; + +const system = 'You answer briefly and remember what you are told.'; + +/** The fact the summary has to carry across the reopen. */ +const secret = 'the vault code is 4417'; +const plant = `Remember this for later: ${secret}. Reply with the word: noted`; +const recall = 'What was the vault code I gave you? Answer with the number only.'; + +/** One database, three runs. Each run builds its layers again, as a restart does. */ +const database = new DatabaseSync(':memory:'); +const store = layerNodeStore(database); + +/** + * Runs one program against that database under a stated window. The window is + * what varies, because it is the only thing the stored count is measured + * against. + */ +const windowed = ( + contextWindow: number, + use: Effect.Effect +): Promise => + Effect.runPromise( + Effect.scoped( + Effect.provide(use, Layer.mergeAll(kilo({ apiKinds: everyShape, contextWindow }), store)) + ) + ); + +/** One question, keeping what it said and what the provider counted for it. */ +const say = (session: SessionHandle, text: string) => + Stream.runFold( + session.ask(text, { maxTokens: room }), + { said: '', prompted: 0 }, + (held, event) => { + if (event.kind === 'delta') { + return { ...held, said: held.said + event.text }; + } + return event.kind === 'done' + ? { ...held, prompted: event.usage.inputTokens + event.usage.cacheReadTokens } + : held; + } + ); + +const isSummary = (turn: Turn): boolean => turn.parts.some(part => part.kind === 'summary'); +const summariesIn = (turns: readonly Turn[]): number => turns.filter(isSummary).length; + +/** + * Plants the fact under a window nothing can fill, so this run ends with a + * count and no summary. Then reads the session back out of the store. + */ +const planted = (model: string) => + Effect.gen(function* () { + const session = yield* openSession({ system, model }); + const answer = yield* say(session, plant); + const stored = yield* Effect.flatMap(SessionStore, plugin => plugin.read(session.id)); + return { + id: session.id, + prompted: answer.prompted, + stored: Option.getOrUndefined(stored)?.prompted, + summaries: summariesIn(yield* session.history), + }; + }); + +/** Reopens the session and asks for the fact back, whatever the window says. */ +const reopened = (sessionId: string) => + Effect.gen(function* () { + const session = yield* continueSession(sessionId); + const before = summariesIn(yield* session.history); + const answer = yield* say(session, recall); + return { said: answer.said, before, after: summariesIn(yield* session.history) }; + }); + +for (const model of models) { + under(model); + + console.log('model', model, '\n'); + + const first = await windowed(1_000_000, planted(model)); + + /* The window this session is now measured against. Its own count fills it, so + a reopened session that reads the count compacts before it asks anything. */ + const tight = first.prompted; + const roomy = first.prompted * 10; + + const narrow = await windowed(tight, reopened(first.id)); + const wide = await windowed(roomy, reopened(first.id)); + + console.log('counted by the provider', first.prompted); + console.log('read back from SQLite ', first.stored); + console.log('summaries after planting', first.summaries); + console.log(`\nreopened under a window of ${String(tight)}`); + console.log(' summaries before the question', narrow.before, 'after', narrow.after); + console.log(' recalled', JSON.stringify(narrow.said)); + console.log(`\nreopened under a window of ${String(roomy)}`); + console.log(' summaries before the question', wide.before, 'after', wide.after); + console.log(' recalled', JSON.stringify(wide.said)); + + if (first.prompted === 0) { + fail('the provider reported no input tokens, so this run measures nothing'); + } + if (first.stored !== first.prompted) { + fail( + `the store holds ${String(first.stored)} where the provider counted ` + + `${String(first.prompted)}, so the count did not survive the write` + ); + } + if (first.summaries !== 0) { + fail('the planting run compacted, so what follows is measured against a summary'); + } + if (narrow.after <= narrow.before) { + fail( + 'a session reopened onto a conversation that fills its window did not compact, ' + + 'so it started from zero rather than from the stored count' + ); + } + if (!narrow.said.includes('4417')) { + fail( + `the fact did not survive the reopen and the summary: the model answered ` + + `${JSON.stringify(narrow.said)}` + ); + } + if (wide.after !== wide.before) { + /* The other direction. Without it, a session that compacted on every + question would pass the check above and be worse than the defect. */ + fail('a session with room to spare compacted anyway'); + } + if (!wide.said.includes('4417')) { + fail(`the fact did not survive the reopen: the model answered ${JSON.stringify(wide.said)}`); + } +} + +passed('the stored count is the provider’s own, and it decides what happens next.'); diff --git a/packages/harness-sdk/e2e/rounds.ts b/packages/harness-sdk/e2e/rounds.ts new file mode 100644 index 0000000000..f1a03e6f64 --- /dev/null +++ b/packages/harness-sdk/e2e/rounds.ts @@ -0,0 +1,109 @@ +import { Duration, Effect, Fiber, type Scope, Stream } from 'effect'; +import type { SessionHandle } from '../src/core/handle.js'; +import type { Continued } from '../src/core/queue.js'; + +/** + * Watching the rounds a session runs on its own. + * + * A queued message and a late tool result are both answered without anybody + * calling `ask`, and `session.continued` is where a caller sees that happen. + * Reading it correctly is subtler than it looks — see `over` — which is why the + * two runs that do it share one copy rather than keeping two in step. + */ + +/** What one round the session ran on its own said, and which message it answers. */ +interface Round { + readonly answering: readonly string[]; + readonly text: string; +} + +/** The event of one thing that happened, or nothing when the round failed. */ +const eventIn = (one: Continued) => ('failed' in one ? undefined : one.event); + +/** + * True when a round is over rather than paused on a tool. + * + * `done` ends one call to the model, and a round that calls a tool makes + * several. `tools` is the model waiting on a call the session is about to + * answer, so it is the one stop reason that is not the end of anything. This is + * how a caller knows a queued message has been answered in full. + */ +const over = (one: Continued): boolean => { + const event = eventIn(one); + return event?.kind === 'done' && event.stop !== 'tools'; +}; + +/** + * The rounds that were refused rather than answered. Empty is the healthy shape. + * + * One array for the module, because one process is one live run: each of these + * files is its own `ttsx` invocation, so there is nothing to share it with. + */ +const refused: (readonly string[])[] = []; + +/** A watch that has started: what it has seen, and the wait for the rest. */ +interface Watching { + /** The rounds seen so far. Complete once `done` has answered. */ + readonly rounds: readonly Round[]; + /** Waits for `count` rounds or for the deadline, and gives back the rounds. */ + readonly done: Effect.Effect; +} + +/** + * Starts watching the rounds a session runs on its own, and hands back the + * wait for them. + * + * **Nothing goes around the reading of `session.continued`.** Both + * `Effect.timeout(...)` and `Stream.interruptAfter` read the same and are not: + * either one puts a race around the subscription to the session's feed, the + * scope of that subscription closes under the race, and the run then either + * ends at once with nothing or waits forever on a queue no publisher can reach. + * Measured against a live session on 2026-09-05, both ways, three times. + * + * So the reading is forked bare, and the deadline is on the waiting for it, + * where a race costs nothing. What was collected before the deadline is + * reported rather than thrown away with the failure: two rounds of three is a + * far better failure to read than none. + */ +const watch = ( + session: SessionHandle, + count: number, + within: Duration.DurationInput = '180 seconds' +): Effect.Effect => { + const rounds: Round[] = []; + let ended = 0; + const held = { answering: [] as readonly string[], text: '' }; + const reading = Stream.runForEach( + Stream.takeUntil(session.continued, one => over(one) && ++ended === count), + (one: Continued) => + Effect.sync(() => { + held.answering = one.answering; + const event = eventIn(one); + if (event?.kind === 'delta') { + held.text += event.text; + } + /* A refused round is one message's bad news, not the end of the feed. + The run says so rather than waiting for words that never come. */ + if ('failed' in one) { + refused.push(one.answering); + } + if (over(one) || 'failed' in one) { + rounds.push({ answering: held.answering, text: held.text }); + held.text = ''; + } + }) + ); + return Effect.map( + Effect.forkScoped(reading), + (fiber): Watching => ({ + rounds, + done: Effect.raceFirst(Fiber.await(fiber), Effect.sleep(within)).pipe( + Effect.zipRight(Fiber.interrupt(fiber)), + Effect.as(rounds as readonly Round[]) + ), + }) + ); +}; + +export type { Round, Watching }; +export { eventIn, over, refused, watch }; diff --git a/packages/harness-sdk/e2e/session.ts b/packages/harness-sdk/e2e/session.ts new file mode 100644 index 0000000000..4d46facfff --- /dev/null +++ b/packages/harness-sdk/e2e/session.ts @@ -0,0 +1,128 @@ +/** + * Proves two things about a live session that the unit tests can only model. + * + * **The prefix holds as the session grows.** The unit test proves `assemble` + * does not rewrite an earlier message. That is not the same as the provider + * agreeing: the symptom of a prefix regression is a large `cache write` on a + * call that should have been almost all `cache read`, and only a real gateway + * shows it. Ten questions, and every call after the first must write far less + * than it reads. + * + * **A busy session refuses rather than corrupts.** Two questions at once would + * both build on the same prefix. The unit test proves the refusal against a + * fake model that answers instantly; this proves it while a real answer is + * still streaming, which is the only time the race can actually happen. + */ +import { Effect, Stream } from 'effect'; +import { SessionBusyError } from '../src/core/ask.js'; +import type { ModelUsage } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { hitRatio } from '../src/core/usage.js'; +import { cachedSystem as system, kilo, models, room } from './setup.js'; +import { fail, passed, under } from './report.js'; + +const questions = 10; + +interface Answer { + readonly said: string; + readonly usage: ModelUsage | undefined; +} + +const ask = (session: SessionHandle, text: string) => + Stream.runFold(session.ask(text), { said: '', usage: undefined } as Answer, (held, event) => + event.kind === 'delta' + ? { ...held, said: held.said + event.text } + : event.kind === 'done' + ? { ...held, usage: event.usage } + : held + ); + +const words = [ + 'one', + 'two', + 'three', + 'four', + 'five', + 'six', + 'seven', + 'eight', + 'nine', + 'ten', +] as const; +const layers = kilo({ apiKinds: ['messages'] }); + +const growing = (model: string) => + Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens: room }); + const answers: Answer[] = []; + for (let index = 0; index < questions; index += 1) { + answers.push(yield* ask(session, `Answer with the word: ${words[index] ?? 'one'}`)); + } + return answers; + }); + +/** + * Starts a question, waits for its first token so the session is provably mid + * answer, then asks a second one on the same session. The second must be + * refused. The first must still finish. + */ +const racing = (model: string) => + Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens: room }); + const started = yield* Effect.fork( + Stream.runCollect(Stream.take(session.ask('Answer with the word: one'), 1)) + ); + yield* Effect.sleep('300 millis'); + const second = yield* Effect.either(Stream.runDrain(session.ask('Answer with the word: two'))); + yield* started.await; + return second; + }); + +for (const model of models) { + under(model); + + console.log('model', model, '\n'); + + const answers = await Effect.runPromise(Effect.scoped(Effect.provide(growing(model), layers))); + + console.log('call said input cache read cache write ratio'); + answers.forEach((answer, index) => { + const usage = answer.usage; + if (usage === undefined) { + fail(`call ${String(index + 1)} carried no token counts`); + return; + } + console.log( + `${String(index + 1).padEnd(6)}${JSON.stringify(answer.said).padEnd(9)}` + + `${String(usage.inputTokens).padEnd(7)}${String(usage.cacheReadTokens).padEnd(12)}` + + `${String(usage.cacheWriteTokens).padEnd(13)}${hitRatio(usage).toFixed(4)}` + ); + + if (answer.said.length === 0) { + fail(`call ${String(index + 1)} carried no text`); + } + + /* The first call writes the whole prefix. Every call after it must be + reading far more than it writes: a large write later means the prefix + moved, which is the regression this run exists to catch. */ + if (index > 0 && usage.cacheWriteTokens > usage.cacheReadTokens / 4) { + fail( + `call ${String(index + 1)} wrote ${String(usage.cacheWriteTokens)} against ` + + `${String(usage.cacheReadTokens)} read, so the prefix moved` + ); + } + }); + + const refused = await Effect.runPromise(Effect.scoped(Effect.provide(racing(model), layers))); + + console.log( + '\nsecond question while the first streamed:', + refused._tag === 'Left' ? refused.left._tag : 'it was accepted' + ); + if (!(refused._tag === 'Left' && refused.left instanceof SessionBusyError)) { + fail('a second question was accepted while the first was still streaming'); + } +} + +passed(`the prefix held across ${String(questions)} calls, and a busy session refused`); diff --git a/packages/harness-sdk/e2e/setup.ts b/packages/harness-sdk/e2e/setup.ts new file mode 100644 index 0000000000..f68f780ea6 --- /dev/null +++ b/packages/harness-sdk/e2e/setup.ts @@ -0,0 +1,135 @@ +import type { ApiKind, ModelFacts } from '../src/core/catalog.js'; +import { layerKilo, type KiloSetup } from '../src/plugins/kilo.js'; +import { webFetch } from '../src/plugins/fetch/web.js'; +import { kiloToken } from './token.js'; + +/** + * What every live run wires: the package's own composed layer, pointed at the + * gateway with this machine's kilo token. The token is read once and never + * printed. + * + * These runs are what proves the composed layer is the wiring a caller wants. + * If one of them has to reach past it for a plugin, it is the wrong shape. + */ +const baseUrl = process.env['KILO_BASE_URL'] ?? 'https://app.kilo.ai'; +const organizationId = process.env['KILO_ORG_ID'] ?? '9d278969-5453-4ae3-a51f-a8d2274a7b56'; + +const token = await kiloToken(); + +/** Every shape, best first. A model whose provider refuses one falls back. */ +const everyShape: readonly ApiKind[] = ['messages', 'responses', 'chat_completions']; + +/** + * Every live run takes a list of models and runs its checks once per model. + * + * The list holds one model, because these runs cost real money: a sweep of + * eleven is eleven times the bill for a change that touched one code path. The + * word `full` on the command line asks for all eleven, and nothing else does. + * + * `KILO_MODELS` names any list, and wins over `full`. + */ +const one = 'z-ai/glm-5.3-flash'; + +/** + * All of them: the ten most used models on OpenRouter this week, from six + * vendors, and Haiku for the one lab that list leaves out. Every one is cheap. + * `deepseek-v4-flash-0423` is not served, so the floating alias stands in, + * `tencent/hy4-preview` is not sold to this team, so a qwen flash takes its + * place, and `nvidia/nemotron-3-ultra-550b-a55b` is served by nobody — every + * provider refused it on 2026-09-04 — so a nemotron that is takes its place. + */ +const everyModel = [ + 'anthropic/claude-haiku-4.5', + 'openai/gpt-5.6-luna', + 'z-ai/glm-5.3-flash', + 'deepseek/deepseek-v4-flash-0731', + 'qwen/qwen3.8-flash', + 'xiaomi/mimo-v2.5', + 'tencent/hy3', + 'deepseek/deepseek-v4-flash', + 'minimax/minimax-m3', + 'nvidia/nemotron-3.5-lightning', + 'google/gemini-3.7-flash', +] as const; + +/* `e2e/all.ts` passes `full` on to each run it spawns as this variable, because + a run reads its own command line and never sees the sweep's. */ +const full = process.argv.includes('full') || process.env['KILO_FULL'] === '1'; + +/** The models this run works through, in order. Never empty. */ +const models: readonly string[] = + process.env['KILO_MODELS']?.split(',') ?? (full ? everyModel : [one]); + +/** + * The room a live run gives a model to answer in. + * + * A model that thinks spends this before it says a word, so a run that hands + * one less is testing the ceiling whether it means to or not: it reads an + * empty answer as a broken feature, on every model that reasons at once. + * Measured across the eleven on 2026-09-06 — at 64 the queue run reported + * three empty answers as a queue that never drained, and every run that passed + * the sweep unchanged had given at least 512. + * + * It is a wall, not a spend: a one word answer costs one word whatever this + * says. Raise it rather than tune it. + * + * A run whose subject *is* the ceiling names its own number and says so. + * There are two: the cut half of `stop.ts`, and the long answer `cancel.ts` + * interrupts. + */ +const room = 1024; + +/** For the few checks that are about a shape or a store rather than a model. */ +const model = models[0] ?? one; + +/** + * The facts go in the fallback rather than in the table, because a live run + * names its model on the command line and a table would have to know it. Only + * a run that needs one shape, or a context window, passes anything: the layer + * already assumes all three. + */ +const kilo = ( + facts: ModelFacts = { apiKinds: everyShape }, + /** For a run that watches the transport, such as the one that cancels. */ + over: Partial = {} +): ReturnType => + layerKilo({ + baseUrl, + org: { kind: 'organization', id: organizationId }, + fetch: webFetch, + token, + fallback: facts, + ...over, + }); + +/** + * A system prompt long enough to be cached, and rules strict enough that an + * answer is one word. + * + * The cached prefix must clear the model's minimum, which is 4096 tokens on + * Haiku 4.5. A short system prompt caches nothing at all, and a run that + * measured the cache would read as a failure of the package rather than of the + * prompt it was given. + */ +const rule = (index: number) => + `Rule ${String(index)}: when the user asks for a word, answer with that one word and nothing else. ` + + 'Do not explain. Do not add punctuation beyond the word itself. Do not greet the user. ' + + 'Do not restate the question. Keep the answer to a single lowercase word.'; + +const cachedSystem = [ + 'You are a test harness. Follow every rule below.', + ...Array.from({ length: 200 }, (_, index) => rule(index)), +].join('\n'); + +export { + baseUrl, + cachedSystem, + everyShape, + full, + kilo, + model, + models, + organizationId, + room, + token, +}; diff --git a/packages/harness-sdk/e2e/shapes.ts b/packages/harness-sdk/e2e/shapes.ts new file mode 100644 index 0000000000..f985936e5e --- /dev/null +++ b/packages/harness-sdk/e2e/shapes.ts @@ -0,0 +1,125 @@ +/** + * Proves the two shapes the live runs never reach. + * + * Every model in `models.ts` picks `messages`, because the gateway serves it + * for all of them and it caches best. So `responses` and `chat_completions` + * have only ever run against a fake `fetch`. This run forces each shape by + * telling the catalog that the model speaks only that one, and asks the same + * two questions the messages run asks. + * + * The three shapes do not cache alike, and the assertions say so: + * + * - `messages` marks an explicit breakpoint, so it must read the cache. + * - `responses` names a `prompt_cache_key`, so it should read the cache. + * - `chat_completions` has no cache control at all, so the only thing worth + * asserting is that the call works and the conversation carries. + */ +import { Effect, Stream } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import type { ModelUsage } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { hitRatio } from '../src/core/usage.js'; +import { cachedSystem as system, kilo, models, room } from './setup.js'; +import { fail, passed, under, wrongIf } from './report.js'; + +interface Answer { + readonly said: string; + readonly usage: ModelUsage | undefined; +} + +const ask = (session: SessionHandle, text: string) => + Stream.runFold(session.ask(text), { said: '', usage: undefined } as Answer, (held, event) => + event.kind === 'delta' + ? { ...held, said: held.said + event.text } + : event.kind === 'done' + ? { ...held, usage: event.usage } + : held + ); + +const runShape = async (model: string, kind: ApiKind) => { + const layers = kilo({ apiKinds: [kind] }); + + const program = Effect.gen(function* () { + /* 256, because a run that gives a model less is testing the ceiling + whether it means to or not: a model that thinks spends it before it says + a word, and the run reports an empty answer as a broken shape. This one + was the last at 64, because it had only ever run on one model. */ + const session = yield* openSession({ system, model, maxTokens: room }); + const first = yield* ask(session, 'Answer with the word: one'); + const second = yield* ask(session, 'Answer with the word: two'); + return { first, second, total: yield* session.usage }; + }); + + return Effect.runPromise(Effect.either(Effect.scoped(Effect.provide(program, layers)))); +}; + +const kinds: readonly ApiKind[] = ['messages', 'responses', 'chat_completions']; +/** `chat_completions` sends no cache control, so it is not held to a ratio. */ +const mustCache = new Set(['messages', 'responses']); + +/** + * The one pairing that reads nothing back and is not a defect: an Anthropic + * model on the responses shape caches nothing on this gateway, measured + * 2026-09-04. It is named here rather than explained in a failure, so a real + * regression on any other pairing still fails the run. + */ +const cachesNothing = (model: string, kind: ApiKind): boolean => + kind === 'responses' && model.startsWith('anthropic/'); + +/** The model and shape pairings that read the prefix back. The floor is one. */ +const cached: string[] = []; + +for (const model of models) { + under(model); + + console.log('model', model); + console.log('\nshape answered cache read input ratio'); + + for (const kind of kinds) { + /* Tried once more before it counts. A provider makes an entry readable when + it chooses to, and about one run in five it has not by the second call: + the same pairing reads the prefix back unchanged on a rerun. Twice is a + finding. */ + const once = await runShape(model, kind); + const result = + once._tag === 'Right' && once.right.total.cacheReadTokens > 0 + ? once + : await runShape(model, kind); + if (result._tag === 'Left') { + console.log(`${kind.padEnd(18)}FAILED ${JSON.stringify(result.left)}`); + fail(`${kind}: the call failed`); + continue; + } + + const { first, second, total } = result.right; + const ratio = hitRatio(total); + const answered = first.said.length > 0 && second.said.length > 0; + console.log( + `${kind.padEnd(18)}${String(answered).padEnd(10)}${String(total.cacheReadTokens).padEnd(12)}` + + `${String(total.inputTokens).padEnd(8)}${ratio.toFixed(4)}` + ); + + if (!answered) { + fail(`${kind}: an answer carried no text`); + } + if (mustCache.has(kind) && total.cacheReadTokens === 0 && !cachesNothing(model, kind)) { + /* Nothing was made readable, twice over. A package that stopped writing + cache control would put every pairing here at once, which the floor + below catches. */ + console.log(`${kind.padEnd(18)}the provider read nothing back, twice`); + continue; + } + if (mustCache.has(kind) && total.cacheReadTokens > 0) { + cached.push(`${model} ${kind}`); + } + } +} + +under(''); +console.log(`\nread the prefix back: ${String(cached.length)} pairings`); +wrongIf(cached.length === 0, 'not one pairing read the prefix back, so no cache control was sent'); + +passed( + 'every shape carried the conversation, and the cache-controlling shapes that cached, cached.' +); diff --git a/packages/harness-sdk/e2e/stop.ts b/packages/harness-sdk/e2e/stop.ts new file mode 100644 index 0000000000..7a672d4221 --- /dev/null +++ b/packages/harness-sdk/e2e/stop.ts @@ -0,0 +1,95 @@ +/** + * Proves each shape reports why the model stopped. + * + * A unit test can only prove this package reads the field it was given. Which + * field a gateway actually sends, and what it puts in it, is a live question, + * and the three shapes answer it three different ways. + * + * Each shape is asked twice: once with room to finish, and once with a ceiling + * far too low to. The second must come back `maxTokens`. A shape that reported + * `end` for both would let a caller store half a sentence as a finished answer + * and build every later request on it. + */ +import { Effect, Stream } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import type { StopReason } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import { kilo, models, room } from './setup.js'; +import { fail, passed, under } from './report.js'; + +const system = 'You answer exactly what you are asked, with no preamble.'; +const short = 'Answer with the single word: yes'; +const long = 'Write three hundred words about the history of the wheel.'; + +interface Answer { + readonly said: string; + readonly stop: StopReason | undefined; +} + +const empty: Answer = { said: '', stop: undefined }; + +const ask = (session: SessionHandle, text: string, maxTokens: number) => + Stream.runFold(session.ask(text, { maxTokens }), empty, (held, event) => { + if (event.kind === 'delta') { + return { ...held, said: held.said + event.text }; + } + return event.kind === 'done' ? { ...held, stop: event.stop } : held; + }); + +const runShape = async (model: string, kind: ApiKind) => { + const layers = kilo({ apiKinds: [kind] }); + + /* Two sessions, because a truncated answer left in the first one would + change what the second question is answering. */ + const program = Effect.gen(function* () { + /* 256 to say one word, because the ceiling is this run's other subject and + must not be hit here by accident: a model that thinks spends anything + less before it answers, and reports the wall it was meant to clear. */ + const finished = yield* Effect.flatMap(openSession({ system, model }), session => + ask(session, short, room) + ); + const cut = yield* Effect.flatMap(openSession({ system, model }), session => + ask(session, long, 24) + ); + return { finished, cut }; + }); + + return Effect.runPromise(Effect.either(Effect.scoped(Effect.provide(program, layers)))); +}; + +const kinds: readonly ApiKind[] = ['messages', 'responses', 'chat_completions']; + +for (const model of models) { + under(model); + + console.log('model', model); + console.log('\nshape finished cut off said when cut'); + + for (const kind of kinds) { + const result = await runShape(model, kind); + if (result._tag === 'Left') { + console.log(`${kind.padEnd(18)}FAILED ${JSON.stringify(result.left)}`); + fail(`${kind}: the call failed`); + continue; + } + + const { finished, cut } = result.right; + console.log( + `${kind.padEnd(18)}${String(finished.stop).padEnd(10)}${String(cut.stop).padEnd(10)}` + + JSON.stringify(cut.said.slice(0, 30)) + ); + + if (finished.stop !== 'end') { + fail(`${kind}: an answer that finished was reported as ${String(finished.stop)}, not end`); + } + if (cut.stop !== 'maxTokens') { + fail( + `${kind}: an answer cut off at the ceiling was reported as ${String(cut.stop)}; a caller ` + + 'would store half a sentence as a finished answer' + ); + } + } +} + +passed('every shape tells a finished answer from one the ceiling cut off.'); diff --git a/packages/harness-sdk/e2e/subagent.ts b/packages/harness-sdk/e2e/subagent.ts new file mode 100644 index 0000000000..22f3c30d3d --- /dev/null +++ b/packages/harness-sdk/e2e/subagent.ts @@ -0,0 +1,247 @@ +/** + * Proves a subagent against the provider, and the caller sending one away. + * + * Two things only a real model can settle. Whether a model reads the subagent + * tool as something to hand a whole task to — rather than a thing to ask a + * fragment of, or to narrate around — and whether the answer that comes back is + * usable as an answer. A fake agrees with whatever the test writes. + * + * Three claims: + * + * - **A task goes down and one answer comes up.** The subagent is told a word + * the parent never sees, and the parent's answer carries it, so the answer + * came through the tool rather than out of the parent's own head. + * - **The subagent is a session of its own.** Its identifier is not the + * parent's, its counts are its own, and its steps are not in the parent's + * transcript. + * - **A running subagent can be sent away by the caller.** The deadline here is + * five minutes, so nothing but `session.background` moves the model on. It + * answers without the subagent, and is told the result in a round of its own + * when it lands. + */ +import { Duration, Effect, Layer, Schedule, Stream } from 'effect'; +import { said } from '../src/core/model.js'; +import type { Continued } from '../src/core/queue.js'; +import { openSession } from '../src/core/run.js'; +import { ToolRegistry } from '../src/core/tool.js'; +import { type SubagentReport, subagentTool } from '../src/plugins/tools/subagent.js'; +import { kilo, models, room } from './setup.js'; +import { fail, passed, under, wrongIf } from './report.js'; + +const system = + 'You are a test harness with a subagent tool. When the person asks for ' + + 'something the subagent can find out, hand it the whole task in one call ' + + 'and answer with what it tells you. Keep your answer to one short sentence.'; + +/** + * What the subagent is told, and the parent never is. An answer carrying this + * word came up through the tool. + */ +const secret = 'nightjar'; + +const subSystem = + 'You are a lookup service. The codename for this quarter is ' + + `"${secret}". Answer in one short sentence, and always give the codename ` + + 'when asked for it.'; + +const reports: SubagentReport[] = []; + +const layers = kilo(); + +/** + * `wait` is what this harness tells the model by default, and both runs below + * ask for the waiting one: a task the model is then held on is what makes the + * hand-off and the sending away visible where the question was asked. The + * package's own default is the opposite, and `subagent.test.ts` covers it. + */ +const subagent = (model: string, inlineFor: Duration.DurationInput) => + subagentTool( + { + system: subSystem, + model, + maxTokens: room, + inlineFor, + wait: true, + onFinished: report => Effect.sync(() => void reports.push(report)), + }, + layers + ); + +const withSubagent = (model: string, inlineFor: Duration.DurationInput) => + Layer.merge(layers, Layer.succeed(ToolRegistry, { tools: [subagent(model, inlineFor)] })); + +/** A task handed down, and one answer handed up. */ +const runHandOff = async (model: string): Promise => { + const program = Effect.gen(function* () { + const session = yield* openSession({ + system, + model, + maxTokens: room, + tools: ['subagent'], + }); + const answer = yield* said(session.ask('Ask the subagent for this quarter’s codename.')); + return { id: session.id, answer, history: yield* session.history, usage: yield* session.usage }; + }); + + const got = await Effect.runPromise( + Effect.either(Effect.scoped(Effect.provide(program, withSubagent(model, '5 minutes')))) + ); + + if (got._tag === 'Left') { + fail(`the hand-off failed: ${JSON.stringify(got.left)}`); + return false; + } + + const { id, answer, history, usage } = got.right; + const report = reports[0]; + const written = history.map(turn => turn.parts.map(part => part.body).join('')).join(' | '); + console.log(`\nthe parent answered: ${JSON.stringify(answer.trim())}`); + console.log(`the subagent said: ${JSON.stringify(report?.said.trim() ?? '')}`); + console.log( + `parent spent ${String(usage.inputTokens + usage.outputTokens)} tokens, ` + + `subagent ${String((report?.usage.inputTokens ?? 0) + (report?.usage.outputTokens ?? 0))}` + ); + + if (report === undefined) { + /* The model answered out of its own head rather than handing the lookup + down. Whether it hands it down is the model's own — `xiaomi/mimo-v2.5` + keeps it on 2026-09-06 — and `pnpm test:e2e:tool-matrix` is the run that + scores that. There is no report here to read. */ + console.log('the model never handed the lookup down'); + return false; + } + wrongIf(!answer.toLowerCase().includes(secret), 'the parent never got what the subagent knew'); + wrongIf(report?.sessionId === id, 'the subagent ran in the parent’s session, not one of its own'); + wrongIf( + (report?.usage.inputTokens ?? 0) === 0, + 'the subagent reported no tokens, so its counts are not its own' + ); + wrongIf( + !written.toLowerCase().includes(secret), + 'the parent’s transcript does not hold the answer it was given' + ); + return true; +}; + +/** + * The same tool, sent away by the caller while it runs. + * + * Nothing here is the clock: the deadline is five minutes. What moves the model + * on is a caller deciding it has waited long enough, which is the same call an + * agent watching its own work would make. + */ +const runSentAway = async (model: string): Promise => { + const rounds: string[] = []; + const program = Effect.gen(function* () { + const session = yield* openSession({ + system, + model, + maxTokens: room, + tools: ['subagent'], + }); + const watching = yield* Effect.fork( + Effect.timeout( + Stream.runForEach( + Stream.takeUntil(session.continued, one => 'failed' in one || one.event.kind === 'done'), + (one: Continued) => + Effect.sync(() => { + if (!('failed' in one) && one.event.kind === 'delta') { + rounds.push(one.event.text); + } + }) + ), + '120 seconds' + ) + ); + /* Watch the running calls, and send the subagent away the moment the model + is waiting on one. */ + const sending = yield* Effect.fork( + Effect.retry( + Effect.flatMap(session.running, waiting => { + const one = waiting[0]; + return one === undefined + ? Effect.fail('not yet' as const) + : Effect.map(session.background(one.id), sent => ({ sent, on: one })); + }), + Schedule.spaced('50 millis').pipe(Schedule.upTo('60 seconds')) + ) + ); + const answer = yield* said(session.ask('Ask the subagent for this quarter’s codename.')); + const sent = yield* sending.await; + yield* watching.await; + return { answer, sent }; + }); + + const got = await Effect.runPromise( + Effect.either(Effect.scoped(Effect.provide(program, withSubagent(model, '5 minutes')))) + ); + + if (got._tag === 'Left') { + fail(`the sent-away run failed: ${JSON.stringify(got.left)}`); + return false; + } + + const later = rounds.join(''); + const sent = got.right.sent._tag === 'Success' ? got.right.sent.value : undefined; + console.log(`\nsent away: ${JSON.stringify(sent?.on.name ?? 'nothing')}`); + console.log(`answered without it: ${JSON.stringify(got.right.answer.trim())}`); + console.log(`told later: ${JSON.stringify(later.trim())}`); + + if (sent === undefined) { + /* A minute passed with no call running to send away, so the model never + handed the work down. Whether it does is the model's own — + `minimax/minimax-m3` keeps it on 2026-09-06 — and there is nothing here + to send away when it does. */ + console.log('the model never handed the work down, so nothing was there to send away'); + return false; + } + + wrongIf(sent.sent !== true, 'the caller could not send the running subagent away'); + wrongIf(sent.on.name !== 'subagent', `the model was waiting on ${String(sent.on.name)}`); + + /* The answer carries the codename already, so the subagent finished inside + the moment between the caller seeing the call run and sending it away. + That is a fast relay winning a race, not a call that was never sent away — + `google/gemini-3.7-flash` wins it on 2026-09-06 and the send still + succeeded. Nothing here can slow a provider down, so it is counted, and + what a late round would have carried is already in the answer. */ + if (got.right.answer.toLowerCase().includes(secret)) { + return false; + } + + wrongIf(!later.toLowerCase().includes(secret), 'the session never told the model what came back'); + return true; +}; + +/** The models that handed the lookup down. The floor is that one did. */ +const down: string[] = []; + +/** The models sent away before they had their answer. The floor is that one was. */ +const away: string[] = []; + +for (const model of models) { + under(model); + + console.log('model', model); + reports.length = 0; + if (await runHandOff(model)) { + down.push(model); + } + if (await runSentAway(model)) { + away.push(model); + } +} + +under(''); +console.log(`\nhanded the lookup down: ${String(down.length)} of ${String(models.length)} models`); +wrongIf(down.length === 0, 'not one model handed the lookup down, so no subagent ever ran'); +console.log(`sent away in time: ${String(away.length)} of ${String(models.length)} models`); +wrongIf( + away.length === 0, + 'not one subagent was sent away before it answered, so the sending was never tested' +); + +passed( + 'a task went down to a session of its own and one answer came up, and ' + + 'a running subagent was sent away by the caller.' +); diff --git a/packages/harness-sdk/e2e/time.ts b/packages/harness-sdk/e2e/time.ts new file mode 100644 index 0000000000..4faed51d65 --- /dev/null +++ b/packages/harness-sdk/e2e/time.ts @@ -0,0 +1,160 @@ +/** + * Proves the time tool against a real model, on all three shapes. + * + * `time.test.ts` proves the words the tool writes. It cannot prove the two + * things that are only true against a provider: + * + * - **A tool that takes nothing renders on every shape.** Its parameters are an + * object with no properties, which is the one schema a provider is most + * likely to refuse: three shapes write it three ways, and a shape that + * rejected it would refuse the whole round rather than one call. + * - **What the tool wrote comes back.** A model that called it and then + * answered with a different date has been handed the answer and ignored it, + * and that is the package's failure to fix. + * + * The date is checked against this machine's clock at the moment of the check, + * so it cannot be pinned to a fixture and cannot rot. + * + * **Whether the model calls it is counted, not asserted.** Measured on + * 2026-09-05, `minimax/minimax-m3` answers "I don't have access to a time tool" + * about one round in eight, with the tool plainly in the request: hand-written + * requests carrying no part of this package do the same, a stronger description + * does not move it, and `tool_choice: "required"` is not honoured for it. So a + * miss is one model's free choice on one round, and a suite that failed on it + * would go red on a different model every run for something no change here can + * fix. What is asserted is the floor underneath: the shape carries the tool, + * and a model that called it answers with what it was given. A model that calls + * it on none of the three shapes is the description failing, and that does fail + * the run. + * + * The same line is drawn in `e2e/tool-matrix.ts`, for the same reason. + */ +import { Effect, Layer } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import { said } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import { type Tool, type ToolCall, ToolRegistry } from '../src/core/tool.js'; +import { timeTool } from '../src/plugins/tools/time.js'; +import { kilo, models, room } from './setup.js'; +import { fail, passed, under, wrongIf } from './report.js'; + +const system = + 'You are a test harness with tools. You do not know what day it is and you ' + + 'must never guess: use the time tool. Answer in one short sentence that ' + + 'repeats the date and time the tool gave you.'; + +const zone = 'Australia/Sydney'; + +/** What the model sent, so the run can say whether it called at all. */ +const ran: ToolCall[] = []; + +const watched = (): Tool => { + const tool = timeTool({ zone }); + return { + ...tool, + run: (call: ToolCall) => { + ran.push(call); + return tool.run(call); + }, + }; +}; + +const withTools = (kind: ApiKind) => + Layer.merge(kilo({ apiKinds: [kind] }), Layer.succeed(ToolRegistry, { tools: [watched()] })); + +/** The day, in whichever of the two places the model chose to answer from. */ +const daysNow = (): readonly string[] => { + const now = new Date(); + const there = new Intl.DateTimeFormat('en-US', { + timeZone: zone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).formatToParts(now); + const of = (type: string): string => there.find(part => part.type === type)?.value ?? ''; + return [String(now.getUTCDate()), of('day').replace(/^0/u, '')]; +}; + +/** One round, or the reason it never happened. */ +const asked = async (model: string, kind: ApiKind): Promise => { + ran.length = 0; + const program = Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens: room, tools: ['time'] }); + return yield* said(session.ask('What is the date and time right now?')); + }); + const answer = await Effect.runPromise( + Effect.either(Effect.scoped(Effect.provide(program, withTools(kind)))) + ); + return answer._tag === 'Left' ? undefined : answer.right; +}; + +/** + * Whether the model called the tool on this shape. + * + * A round that failed is tried once more before it counts: measured on + * 2026-09-05, `xiaomi/mimo-v2.5` refused two shapes on one sweep and carried + * all three on the next, which is the relay having a bad minute rather than a + * shape that cannot write a tool taking nothing. Twice is a finding. + */ +const runShape = async (model: string, kind: ApiKind): Promise => { + const answer = (await asked(model, kind)) ?? (await asked(model, kind)); + if (answer === undefined) { + console.log(`${kind.padEnd(18)}FAILED the round failed twice`); + fail(`${kind}: the round failed twice, so the shape would not carry a tool that takes nothing`); + return false; + } + + const spoken = answer.replaceAll(/\s+/gu, ' ').trim(); + console.log( + `${kind.padEnd(18)}${String(ran.length).padEnd(7)}${JSON.stringify(spoken.slice(0, 90))}` + ); + wrongIf(ran.length > 1, `${kind}: the tool ran ${String(ran.length)} times for one question`); + if (ran.length === 0) { + return false; + } + + /* From here the tool ran, so the date is the package's to get right: what it + wrote went to the model, and what the model says is what it was given. */ + const year = String(new Date().getUTCFullYear()); + wrongIf( + !spoken.includes(year), + `${kind}: the tool ran and the answer carries a different year, so what it wrote never landed` + ); + wrongIf( + !daysNow().some(day => spoken.includes(day)), + `${kind}: the tool ran and the answer carries neither today's date in UTC nor the one in ${zone}` + ); + return true; +}; + +/** Which models chose not to call, and on which shapes. Printed at the end. */ +const missed: string[] = []; + +for (const model of models) { + under(model); + console.log(`\nmodel ${model}`); + console.log('\nshape calls answered'); + + const shapes = ['messages', 'responses', 'chat_completions'] as const; + const called: ApiKind[] = []; + for (const kind of shapes) { + if (await runShape(model, kind)) { + called.push(kind); + } + } + if (called.length < shapes.length) { + missed.push( + `${model}: called on ${called.length === 0 ? 'no shape' : called.join(', ')} of ${String(shapes.length)}` + ); + } + wrongIf( + called.length === 0, + 'the model called the tool on none of the three shapes, which is the description failing rather than one bad round' + ); +} + +under(''); +if (missed.length > 0) { + console.log(`\nchose not to call:\n ${missed.join('\n ')}`); +} +passed('every shape carried a tool that takes nothing, and what it wrote came back in the answer'); diff --git a/packages/harness-sdk/e2e/todo.ts b/packages/harness-sdk/e2e/todo.ts new file mode 100644 index 0000000000..4a9aa3f128 --- /dev/null +++ b/packages/harness-sdk/e2e/todo.ts @@ -0,0 +1,124 @@ +/** + * Proves the todo tool against a real model, over three turns of one session. + * + * `todo.test.ts` proves the tool holds a list and reads it back. What only a + * provider can settle is the shape of the payload and what a model does with + * it across a conversation: + * + * - **The schema renders on every shape.** An array of objects with an + * enumerated field is the richest parameter schema this package ships, and + * the three shapes write it three different ways. + * - **The whole list comes every time.** The tool replaces rather than patches, + * and the description says so. A model that sent only the step it had changed + * would leave a list of one, which is the failure this run is for. + * - **The list survives the conversation.** Three turns, and the third asks + * only for what the list now says, so an answer can only come from what the + * tool read back. + * + * The plan is dictated rather than invented, so what is measured is the tool + * and not the model's taste in project management. The marks are printed and + * not asserted, for the same reason: whether a model moves a step to `doing` + * or straight to `done` is the model's discipline, and + * `pnpm test:e2e:tool-matrix` is the run that scores what models choose. + */ +import { Effect, Layer } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import { said } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import { ToolRegistry } from '../src/core/tool.js'; +import { type Todo, todoTool } from '../src/plugins/tools/todo.js'; +import { kilo, models, room } from './setup.js'; +import { fail, passed, under } from './report.js'; + +const system = + 'You are a test harness with tools. Keep the plan in the todo tool: write ' + + 'the steps down when you are given them, and send the whole list every time ' + + 'you change anything about it. Answer in one short sentence.'; + +const steps = ['cut the branch', 'run the checks', 'publish the notes']; + +const opening = + `Write my plan down with the todo tool, as these three steps in this order: ${steps.join(', ')}. ` + + 'Mark the first one as the one you are on. Then tell me what you wrote.'; + +const finished = 'I have cut the branch. Mark that step done and start the next one.'; + +const asked = 'Which step are you on? Answer with that step and nothing else.'; + +/** Every version of the list the harness was shown, in order. */ +const versions: (readonly Todo[])[] = []; + +const withTools = (kind: ApiKind) => + Layer.merge( + kilo({ apiKinds: [kind] }), + Layer.succeed(ToolRegistry, { + tools: [ + todoTool({ + onChanged: todos => Effect.sync(() => void versions.push(todos)), + }), + ], + }) + ); + +const marks = (todos: readonly Todo[]): string => + todos.map(todo => `${todo.state[0] ?? '?'}`).join(''); + +const runShape = async (model: string, kind: ApiKind): Promise => { + versions.length = 0; + const program = Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens: room, tools: ['todo'] }); + yield* said(session.ask(opening)); + yield* said(session.ask(finished)); + return yield* said(session.ask(asked)); + }); + const answer = await Effect.runPromise( + Effect.either(Effect.scoped(Effect.provide(program, withTools(kind)))) + ); + + if (answer._tag === 'Left') { + console.log(`${kind.padEnd(18)}FAILED ${JSON.stringify(answer.left)}`); + fail(`${kind}: the round failed, so the shape would not carry the list`); + return; + } + + const last = versions.at(-1) ?? []; + console.log( + `${kind.padEnd(18)}${String(versions.length).padEnd(8)}${String(last.length).padEnd(7)}` + + `${marks(last).padEnd(8)}${JSON.stringify(answer.right.slice(0, 60))}` + ); + + if (versions.length < 2) { + fail(`${kind}: the list was written ${String(versions.length)} times, so it was never updated`); + } + if (versions.some(version => version.length !== steps.length)) { + const sizes = versions.map(version => version.length).join(', '); + fail(`${kind}: the list held ${sizes} steps, not ${String(steps.length)} every time`); + } + /* Matched on the noun of each step, because a model rewrites "run the + checks" as "Run checks" and the run is not about its prose. */ + const nounOf = (step: string): string => step.split(' ').at(-1) ?? step; + if (!steps.every(step => last.some(todo => todo.text.toLowerCase().includes(nounOf(step))))) { + fail(`${kind}: the list lost a step: ${JSON.stringify(last.map(todo => todo.text))}`); + } + if (!last.some(todo => todo.state === 'done')) { + fail(`${kind}: nothing was marked done, though a step was finished`); + } + if (!answer.right.toLowerCase().includes('checks')) { + fail(`${kind}: the model was asked what it is on and answered ${JSON.stringify(answer.right)}`); + } +}; + +for (const model of models) { + under(model); + console.log(`\nmodel ${model}`); + console.log('\nshape writes steps states answered'); + + for (const kind of ['messages', 'responses', 'chat_completions'] as const) { + await runShape(model, kind); + } +} + +under(''); +passed( + 'every shape carried the list, and every model kept the whole plan in it across three turns' +); diff --git a/packages/harness-sdk/e2e/together.ts b/packages/harness-sdk/e2e/together.ts new file mode 100644 index 0000000000..c3a5616cd5 --- /dev/null +++ b/packages/harness-sdk/e2e/together.ts @@ -0,0 +1,278 @@ +/** + * Proves the two things a session does on its own share one line, in order. + * + * A late tool result and a queued message are the same thing to the session: a + * message it must say when it is free. They are held in one line so the order + * between them is defined rather than a race, and this run makes them contend + * for it — the answer to a question the model asked, and a message a person + * typed after it, both waiting while the session is busy with a third thing. + * + * One session, three rounds: + * + * - **Several questions in one call.** The model is asked to find out two + * things, and the tool takes both at once, with choices. That is the question + * tool's richer shape, exercised by a model rather than by a test. + * - **The answer outlives the request.** The asker takes far longer than the + * model waits, so the model is told the question is out and answers without + * it. + * - **A slow message is queued**, and while its round runs the answer lands in + * the line behind it. A second message is queued only once the line is seen + * holding that answer, so the two are certainly waiting together. + * - **The order is the order they joined.** The slow message, then the answer, + * then the message typed after it — and each round names what it answers. + */ +import { Duration, Effect, Layer, Schedule } from 'effect'; +import type { SessionHandle } from '../src/core/handle.js'; +import { said } from '../src/core/model.js'; +import type { Waiting } from '../src/core/queue.js'; +import { openSession } from '../src/core/run.js'; +import { type Tool, ToolRegistry } from '../src/core/tool.js'; +import { + type Answer, + type Asker, + type Question, + questionTool, +} from '../src/plugins/tools/question.js'; +import { kilo, models, room } from './setup.js'; +import { passed, under, wrongIf } from './report.js'; +import { refused, watch, type Round } from './rounds.js'; + +const system = + 'You are a test harness with tools. Call a tool whenever the person names ' + + 'one. When you need something only the person knows, ask with the question ' + + 'tool, and ask everything you need in one call. Answer in one or two short ' + + "sentences, repeating the person's own words."; + +const opening = + 'I want a bicycle. Use the question tool once to ask me two questions in ' + + 'that one call: which colour I want, offering ultramarine and vermilion as ' + + 'choices, and which animal should be on the bell, offering marmoset and ' + + 'kestrel as choices. Then tell me what I picked.'; + +/* Slow on purpose, and slow by the clock rather than by the token: its round + must still be running when the answer lands, or the two never wait in the + line together and the order is not under test. A model asked for a long + answer writes it in a couple of seconds; a tool that sleeps holds the round + for as long as it says. */ +const slowly = "Call the wait tool once. When it answers, answer with the word 'narwhal'."; + +const after = "Answer with the word 'pelican' and nothing else."; + +/** What the model actually asked for, kept so the run can say what it saw. */ +const takenUp: Question[][] = []; + +/** Picks the first choice of every question, slower than any model waits. */ +const asker: Asker = questions => + Effect.gen(function* () { + takenUp.push([...questions]); + yield* Effect.sleep('6 seconds'); + return questions.map( + (question): Answer => ({ + id: question.id, + ...(question.choices === undefined || question.choices[0] === undefined + ? { text: 'ultramarine' } + : { chosen: [question.choices[0].value] }), + }) + ); + }); + +/** Holds a round open for a known time, so the line is certainly contended. */ +const waitTool: Tool = { + definition: { + name: 'wait', + description: 'Waits a while and then answers. Takes nothing.', + parameters: { type: 'object', properties: {}, additionalProperties: false }, + }, + run: () => Effect.as(Effect.sleep('10 seconds'), 'Waited.'), +}; + +const tools = Layer.succeed(ToolRegistry, { + tools: [questionTool(asker, { inlineFor: Duration.millis(500) }), waitTool], +}); + +/** + * Waits until the line holds the answer to the question, and says what it held. + * + * It watches the line rather than the clock. Queueing the second message on a + * timer would prove nothing on a slow day: the two must be seen waiting + * together, or the order between them was never tested. + */ +const untilAnswerWaits = (session: SessionHandle) => + Effect.retry( + Effect.filterOrFail( + session.queued, + waiting => waiting.some(one => one.kind === 'toolResult'), + () => 'not yet' as const + ), + Schedule.spaced('100 millis').pipe(Schedule.upTo('60 seconds')) + ); + +/** The line entry that carries the answer to the question. */ +const answerIn = (waiting: readonly Waiting[]): Waiting | undefined => + waiting.find(one => one.kind === 'toolResult'); + +const program = (model: string) => + Effect.gen(function* () { + const session = yield* openSession({ + system, + model, + maxTokens: room, + tools: ['question', 'wait'], + }); + const watching = yield* watch(session, 3, '180 seconds'); + const first = yield* said(session.ask(opening)); + /* The question is still out: the asker sleeps longer than the model waited. */ + const long = yield* session.queue(slowly); + const waiting = yield* untilAnswerWaits(session); + const last = yield* session.queue(after); + const both = yield* session.queued; + const rounds = yield* watching.done; + return { first, long, last, waiting, both, rounds }; + }); +/** The models that put a question out and left it out. The floor is that one did. */ +const asked: string[] = []; + +for (const model of models) { + under(model); + + /* What the tool was asked, from this model only. */ + takenUp.length = 0; + + const ran = await Effect.runPromise( + Effect.either(Effect.scoped(Effect.provide(program(model), tools.pipe(Layer.merge(kilo()))))) + ); + if (ran._tag === 'Left') { + /* `not yet` is the wait giving up: a minute passed and no answer ever + joined the line, because the model never asked. Whether it calls the tool + is the model's own — `openai/gpt-5.6-luna` answered the two questions out + of its own head on 2026-09-06 — and there is no order to test between a + message and an answer that was never asked for. */ + const why = ran.left === 'not yet' ? 'never left a question outstanding' : String(ran.left); + console.log(`model ${model}: ${why}`); + wrongIf(ran.left !== 'not yet', `the run failed: ${why}`); + continue; + } + asked.push(model); + const got = ran.right; + + const rounds = got.rounds; + const askedFor = takenUp[0] ?? []; + + const answered = answerIn(got.waiting); + + /** + * Every identifier waiting in the line, in the order it joined. + * + * An identifier is made when its entry joins and sorts by when it was made, so + * sorting them is the join order. Reading it off a snapshot of the line would + * not do: by the time the caller looks, the session may already have taken the + * first one out. + */ + const joined = [got.long, got.last, answered?.id ?? ''].filter(id => id !== '').toSorted(); + + /** What one identifier stands for, in words, so a failure reads as an order. */ + const idMarks = (id: string | undefined): string => { + if (id === got.long) { + return 'the slow message'; + } + if (id === got.last) { + return 'the message typed after'; + } + return 'the late answer'; + }; + + const marks = (round: Round | undefined): string => idMarks(round?.answering[0]); + + /** What the session said in the round that answered one identifier. */ + const textFor = (id: string): string | undefined => + rounds.find(round => round.answering.includes(id))?.text; + + console.log('model', model); + console.log(`\nasked in one call: ${String(askedFor.length)} questions`); + for (const question of askedFor) { + const choices = (question.choices ?? []).map(one => one.value).join(', '); + console.log(` [${question.id}] ${question.prompt} {${choices}}`); + } + console.log(`\nanswered without waiting: ${JSON.stringify(got.first.trim())}`); + console.log( + `\nthe line, once the answer had joined it: ${JSON.stringify(got.waiting.map(one => one.kind))}` + ); + console.log( + `and once a message was typed after it: ${JSON.stringify(got.both.map(one => one.kind))}` + ); + for (const [at, round] of rounds.entries()) { + const text = round.text.trim().replaceAll('\n', ' '); + const shown = text.length > 90 ? `${text.slice(0, 90)}…` : text; + console.log(`\nround ${String(at + 1)} (${marks(round)}): ${JSON.stringify(shown)}`); + } + + if (takenUp.length !== 1 || askedFor.length < 2) { + /* It asked one question at a time instead of both in one call, so there is + no one late answer for a message to contend with and no order between + them to test. `qwen/qwen3.8-flash` and `deepseek/deepseek-v4-flash` split + them on 2026-09-06. Putting two questions in one call is the model's + choice, and `pnpm test:e2e:tool-matrix` is the run that scores it. */ + console.log( + `the model split the questions across ${String(takenUp.length)} calls, so there is no one answer to order against` + ); + asked.pop(); + continue; + } + wrongIf( + !askedFor.some(question => (question.choices ?? []).length > 1), + 'the model asked nothing with choices, so the richer shape never ran' + ); + /* The asker sleeps longer than the model waited, so the word cannot have been + handed over yet: a model that says it anyway guessed. `google/gemini-3.7-flash` + guesses it on 2026-09-06. The guess is the model's own and the line it left + behind is tested the same either way, so it is noted, not failed. */ + if (got.first.toLowerCase().includes('ultramarine')) { + console.log('the model guessed the answer rather than waiting for it'); + } + wrongIf( + answered === undefined, + 'the answer never waited in the line, so nothing ever contended for it' + ); + wrongIf( + rounds.length !== 3, + `the session ran ${String(rounds.length)} rounds of its own, not three` + ); + /* The one claim this run exists for. The identifiers are made in the order + they join the line and sort that way, so their order is the order the + session owes them, whatever the model's speed did to the clock. */ + wrongIf( + JSON.stringify(rounds.map(round => round.answering)) !== JSON.stringify(joined.map(id => [id])), + `the rounds answered ${JSON.stringify(rounds.map(marks))}, not ${JSON.stringify(joined.map(idMarks))}` + ); + wrongIf( + !(textFor(got.long) ?? '').toLowerCase().includes('narwhal'), + 'the message that held the session open was not answered' + ); + wrongIf( + !(textFor(answered?.id ?? '') ?? '').toLowerCase().includes('ultramarine'), + 'the session never told the model what the person answered' + ); + wrongIf( + !(textFor(got.last) ?? '').toLowerCase().includes('pelican'), + 'the message typed after the answer was never answered' + ); + + wrongIf( + refused.length > 0, + `the session was refused ${String(refused.length)} of the rounds it ran on its own` + ); +} + +under(''); +console.log(`\nput a question out: ${String(asked.length)} of ${String(models.length)} models`); +/* The floor under the skip: a line that never held an answer for any model is + the package, not eleven models each deciding the same way. */ +wrongIf( + asked.length === 0, + 'not one model left a question outstanding, so the line was never tested' +); + +passed( + 'two questions went out in one call, and a late answer and a typed ' + + 'message waited in one line and were said in the order they joined it.' +); diff --git a/packages/harness-sdk/e2e/token.ts b/packages/harness-sdk/e2e/token.ts new file mode 100644 index 0000000000..af7c4f667d --- /dev/null +++ b/packages/harness-sdk/e2e/token.ts @@ -0,0 +1,24 @@ +import { readFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +/** + * The credential every live run needs, read from the kilo CLI's own file. + * + * The value is never printed, and nothing else in `e2e/` reads that file. + * + * The `fetch` these runs use is `webFetch`, imported from the package like any + * consumer would. This file once re-exported it as `nodeFetch`, which named a + * runtime the adapter does not have. + */ +const kiloToken = async (): Promise => { + const path = join(homedir(), '.local', 'share', 'kilo', 'auth.json'); + const auth: unknown = JSON.parse(await readFile(path, 'utf8')); + const access = (auth as { kilo?: { access?: string } }).kilo?.access; + if (access === undefined) { + throw new Error(`no kilo token in ${path}; run \`kilo auth login\``); + } + return access; +}; + +export { kiloToken }; diff --git a/packages/harness-sdk/e2e/tool-matrix.ts b/packages/harness-sdk/e2e/tool-matrix.ts new file mode 100644 index 0000000000..2c45281a41 --- /dev/null +++ b/packages/harness-sdk/e2e/tool-matrix.ts @@ -0,0 +1,352 @@ +/** + * The tools this package ships, used by every model, not only by Haiku. + * + * A tool is one version for everybody. There is no per-model branch and there + * will not be one: a harness cannot keep eleven descriptions honest, and a + * model the package has never seen has to work anyway. So the way to tune a + * description is to measure it across labs and change the one text until every + * row is clean. + * + * What is scored is what the model chose, never what it said: + * + * - **called** — it used the tool instead of answering from nothing. + * - **valid** — the arguments were what the schema asked for. A model that + * calls a tool and sends the wrong shape has read the description and not the + * schema, which is a schema to fix. + * - **batched** — it asked everything in one call. The question tool says to, + * and a model that ignores it puts a person through two dialogs for one task. + * - **waited** — whether the model sat on the call. This is read from what the + * harness told it rather than from what it sent: a call the model did not + * wait for gets the still-running note, and that note is in the event stream. + * + * There is no right answer to **waited**, which is why `Tool.wait` is a default + * and not a rule. Both scenarios here block — a deployment nobody has answered + * about, a codename the model does not know — so waiting is correct in both, + * and the two tools ship opposite defaults. What the column measures is whether + * the model reads the field at all: `question` defaults to waiting and the + * model keeps it, `subagent` defaults to not and the model has to override it. + * A model that never overrides is a model ignoring the field. + * + * The run fails on a floor — every model calls the tool, and sends a payload + * the tool can read — and reports the rest as numbers. A model that batches + * badly is a description to tune, not a broken package. + */ +import { Effect, Layer, Stream } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import type { ModelEvent } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import { type Tool, ToolRegistry } from '../src/core/tool.js'; +import { type Asker, type Question, questionTool } from '../src/plugins/tools/question.js'; +import { subagentTool } from '../src/plugins/tools/subagent.js'; +import { timeTool } from '../src/plugins/tools/time.js'; +import { type Todo, todoTool } from '../src/plugins/tools/todo.js'; +import { kilo, models, room } from './setup.js'; +import { fail, passed } from './report.js'; + +const maxTokens = Number(process.env['KILO_MAX_TOKENS'] ?? String(room)); + +const system = + 'You are a coding harness with tools. Use a tool whenever one can answer ' + + 'better than you can, and never invent a value a tool can give you. Keep ' + + 'every answer to one or two short sentences.'; + +/** The words the harness gives the model for a call it is not waiting on. */ +const notWaited = 'This call is still running'; + +/** What one model did with one tool. */ +interface Scored { + readonly called: boolean; + readonly valid: boolean; + readonly waited: boolean; + /** How many questions were in the first call. Zero where there is nothing to batch. */ + readonly together: number; + /** What the model answered. Printed under the table for a row that missed the floor. */ + readonly said: string; +} + +const nothing: Scored = { called: false, valid: false, waited: false, together: 0, said: '' }; + +/** What the round says about the call, read out of the events themselves. */ +const watched = (events: readonly ModelEvent[]) => ({ + called: events.some(event => event.kind === 'toolCall'), + waited: events.some( + event => event.kind === 'toolResult' && !event.result.body.startsWith(notWaited) + ), + said: events + .filter(event => event.kind === 'delta') + .map(event => event.text) + .join('') + .trim(), +}); + +const preferred: readonly ApiKind[] = ['messages', 'responses', 'chat_completions']; + +/** + * A model that neither called anything nor said anything has decided nothing. + * + * This is not a model reading a description badly, so it is not something a + * description can fix. `glm-5.3-flash` did it once on the todo scenario and + * answered normally on every run since — an empty answer is the provider, and + * scoring it would have sent somebody off to tune a description that was + * already right. It is the second time this run measured the wrong thing; the + * first is under "One tool version" in AGENTS.md. + */ +const decided = (one: Scored): boolean => one.called || one.said !== ''; + +/** + * Tries the best shape first, then the one every provider serves. + * + * Same fallback as `e2e/models.ts`, plus the empty answer above. A provider that + * refuses a shape is not a model using a tool badly, and scoring it as one would + * put a row in the table that no description can fix. A second empty answer is + * kept rather than retried again: twice is a finding. + */ +const tried = async ( + build: (kinds: readonly ApiKind[]) => Effect.Effect +): Promise => { + const got = await Effect.runPromise( + Effect.either( + build(preferred).pipe( + Effect.filterOrFail(decided, () => 'answered nothing at all' as const), + Effect.catchAll(() => build(['chat_completions'])) + ) + ) + ); + return got._tag === 'Left' ? nothing : got.right; +}; + +/* ---------------------------------------------------------------- question */ + +/** + * A task that cannot be finished without asking, and that needs two answers. + * + * Two, because batching is only visible when there is something to batch, and + * a model that asks one at a time puts a person through two dialogs for one + * task. + */ +const asking = + 'Set up my deployment. I have not told you which region to deploy to, or ' + + 'whether to turn on backups. Find out from me, then say what you will do.'; + +const answerOf = (question: Question) => ({ + id: question.id, + ...(question.choices === undefined + ? { text: /backup/iu.test(question.prompt) ? 'yes' : 'frankfurt' } + : { chosen: [question.choices[0]?.value ?? ''] }), +}); + +const oneQuestion = (model: string, kinds: readonly ApiKind[]) => + Effect.suspend(() => { + const asked: (readonly Question[])[] = []; + const ask: Asker = questions => + Effect.sync(() => { + asked.push(questions); + return questions.map(answerOf); + }); + const layers = Layer.merge( + kilo({ apiKinds: kinds }), + Layer.succeed(ToolRegistry, { tools: [questionTool(ask)] }) + ); + const program = Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens, tools: ['question'] }); + return [...(yield* Stream.runCollect(session.ask(asking)))]; + }); + return Effect.map(Effect.scoped(Effect.provide(program, layers)), events => ({ + ...watched(events), + /* The asker runs on arguments the schema accepted and on nothing else, so + reaching it at all is the payload being right. */ + valid: asked.length > 0, + together: asked[0]?.length ?? 0, + })); + }); + +/* ---------------------------------------------------------------- subagent */ + +const secret = 'quartzite'; + +const subSystem = + 'You are a lookup service for the Acme Deploy project. The codename of its ' + + `4.0 release is "${secret}". Answer in one short sentence, and always give ` + + 'the codename when asked.'; + +/** + * One fact the model cannot know, about a thing named exactly once. + * + * The first version asked for "this quarter's codename" and four models + * answered by asking which project was meant, which is the right move on a + * question that names none. That measured the prompt, not the tool: a model + * that will not guess is doing its job. Naming the release leaves one reason + * left not to delegate, which is the tool description, which is the thing under + * test. + */ +const delegating = 'What is the codename of the Acme Deploy 4.0 release? Find out and tell me.'; + +/** + * The task the model handed down, or nothing if it was not the shape asked for. + * + * The subagent's own answer is not what is scored here. The parent does not + * wait for it by default, so a run that read the answer would be scoring the + * clock; what the parent chose to send down is decided before any of that. + */ +const taskOf = (held: string): string | undefined => { + const sent: unknown = JSON.parse(held); + const task = + typeof sent === 'object' && sent !== null ? (sent as { task?: unknown }).task : undefined; + return typeof task === 'string' && task.trim() !== '' ? task : undefined; +}; + +/** The shipped tool, with what the model sent it kept on the way past. */ +const noting = (tool: Tool, sent: string[]): Tool => ({ + ...tool, + run: call => { + sent.push(call.arguments); + return tool.run(call); + }, +}); + +const oneSubagent = (model: string, kinds: readonly ApiKind[]) => + Effect.suspend(() => { + const sent: string[] = []; + const under = kilo({ apiKinds: kinds }); + const tool = noting(subagentTool({ system: subSystem, model, maxTokens: room }, under), sent); + const layers = Layer.merge(under, Layer.succeed(ToolRegistry, { tools: [tool] })); + const program = Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens, tools: ['subagent'] }); + return [...(yield* Stream.runCollect(session.ask(delegating)))]; + }); + return Effect.map(Effect.scoped(Effect.provide(program, layers)), events => ({ + ...watched(events), + valid: sent.some(one => taskOf(one) !== undefined), + together: 0, + })); + }); + +/* -------------------------------------------------------------------- time */ + +/** + * A question whose answer is the date, asked without saying so. + * + * "What is today's date" would tell the model which tool to reach for. What is + * being measured is whether it notices that its own answer would be stale, so + * the question is one it can answer wrongly without noticing. + */ +const dating = 'How many days are left in this month? Give me the number.'; + +const oneTime = (model: string, kinds: readonly ApiKind[]) => + Effect.suspend(() => { + const layers = Layer.merge( + kilo({ apiKinds: kinds }), + Layer.succeed(ToolRegistry, { tools: [timeTool()] }) + ); + const program = Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens, tools: ['time'] }); + return [...(yield* Stream.runCollect(session.ask(dating)))]; + }); + return Effect.map(Effect.scoped(Effect.provide(program, layers)), events => ({ + ...watched(events), + /* Nothing is read off the call, so a call that happened is a call that + worked. There is no payload here to get wrong. */ + valid: events.some(event => event.kind === 'toolCall'), + together: 0, + })); + }); + +/* -------------------------------------------------------------------- todo */ + +/** + * A task of several steps, without the word "list" or "steps" in it. + * + * Naming the tool's job in the question would measure obedience. What is being + * measured is whether a model reads a job of several parts as one to write + * down, which is the judgement the description has to produce. + */ +const planning = + 'Migrate my project from npm to pnpm: check the lockfile, update the CI ' + + 'workflow, and fix the install script. Start on it.'; + +const oneTodo = (model: string, kinds: readonly ApiKind[]) => + Effect.suspend(() => { + const drawn: (readonly Todo[])[] = []; + const tool = todoTool({ onChanged: todos => Effect.sync(() => void drawn.push(todos)) }); + const layers = Layer.merge( + kilo({ apiKinds: kinds }), + Layer.succeed(ToolRegistry, { tools: [tool] }) + ); + const program = Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens, tools: ['todo'] }); + return [...(yield* Stream.runCollect(session.ask(planning)))]; + }); + return Effect.map(Effect.scoped(Effect.provide(program, layers)), events => ({ + ...watched(events), + /* The harness is told only on a list the schema accepted, so being told at + all is the payload being right. */ + valid: drawn.length > 0, + /* How many steps went down in the first list. One is a model that did not + read the task as having parts. */ + together: drawn[0]?.length ?? 0, + })); + }); + +/* ------------------------------------------------------------------- table */ + +const scored = async (model: string) => ({ + model, + question: await tried(kinds => oneQuestion(model, kinds)), + subagent: await tried(kinds => oneSubagent(model, kinds)), + time: await tried(kinds => oneTime(model, kinds)), + todo: await tried(kinds => oneTodo(model, kinds)), +}); + +const rows = await Promise.all(models.map(scored)); + +const pad = (text: string, width: number) => text.padEnd(width); +const mark = (right: boolean) => (right ? 'yes' : 'NO'); + +console.log( + `\n${pad('model', 32)}${pad('question', 32)}${pad('subagent', 20)}${pad('time', 8)}todo\n` + + `${pad('', 32)}${pad('called valid batch waited', 32)}` + + `${pad('called valid waited', 20)}${pad('called', 8)}called valid steps` +); + +for (const { model, question, subagent, time, todo } of rows) { + console.log( + pad(model, 32) + + pad(mark(question.called), 7) + + pad(mark(question.valid), 6) + + pad(question.together > 1 ? 'yes' : String(question.together), 7) + + pad(mark(question.waited), 12) + + pad(mark(subagent.called), 7) + + pad(mark(subagent.valid), 6) + + pad(mark(subagent.waited), 7) + + pad(mark(time.called), 8) + + pad(mark(todo.called), 7) + + pad(mark(todo.valid), 6) + + String(todo.together) + ); + + /* The floor. Everything else is a number to tune a description against. */ + for (const [name, one] of [ + ['question', question], + ['subagent', subagent], + ['time', time], + ['todo', todo], + ] as const) { + if (!one.called) { + fail(`${model} never called ${name}, and said: ${JSON.stringify(one.said)}`); + } else if (!one.valid) { + fail(`${model} sent ${name} a payload it could not read`); + } + } +} + +const share = (of: (row: (typeof rows)[number]) => boolean) => + `${String(rows.filter(of).length)} of ${String(rows.length)}`; + +console.log( + `\nasked everything in one call: ${share(row => row.question.together > 1)}` + + `\nkept question's waiting default: ${share(row => row.question.waited)}` + + `\noverrode subagent's, as it had to: ${share(row => row.subagent.called && row.subagent.waited)}` + + `\nwrote down every step of three: ${share(row => row.todo.together >= 3)}` +); + +passed('every model called every tool and sent each one a payload it could read.'); diff --git a/packages/harness-sdk/e2e/tools.ts b/packages/harness-sdk/e2e/tools.ts new file mode 100644 index 0000000000..779121495a --- /dev/null +++ b/packages/harness-sdk/e2e/tools.ts @@ -0,0 +1,368 @@ +/** + * Proves tools against the provider, which is the only party that can say the + * package renders them right. + * + * A tool call is the one thing in this package that three shapes disagree + * about: `messages` writes blocks, `responses` writes items beside the message, + * `chat_completions` writes a field on the assistant message and a role of its + * own for the result. A fake `fetch` proves the package writes what it meant + * to. Only a real gateway proves the provider reads it — and a shape that + * refuses a round refuses the whole session, because a call whose result it + * will not read can never be answered. + * + * Three things run here: + * + * - **Every shape carries a round.** The model calls the tool, reads what it + * said, and answers with a word it could not have invented. + * - **The calls of one turn overlap.** Two cities, two calls, and the second + * starts before the first has finished. + * - **A call outlives the request.** The question tool takes longer than the + * deadline, the model is told so and carries on, and the session runs a round + * of its own when the answer lands. + * - **The model decides for itself.** A tool that named no deadline is walked + * away from because the model set `wait: false` on the call, and what it says + * still comes back. + * + * The last three read a **choice the model makes**, not a rule it must follow. + * Measured on 2026-09-06, `nvidia/nemotron-3.5-lightning` sent the two calls of + * one turn one after the other and guessed an answer the tool had not given + * yet. Nothing in this package can make a model call in parallel or wait, so a + * run that failed on it would go red on a different model every sweep. Those + * three count a miss and assert the floor underneath: what the package does + * once the model has chosen. The first, the shape carrying a round at all, is + * the package's own and stays absolute. + * + * The same line is drawn in `e2e/time.ts`, for the same reason. + */ +import { Duration, Effect, Layer, Stream } from 'effect'; +import type { ApiKind } from '../src/core/catalog.js'; +import { said } from '../src/core/model.js'; +import { openSession } from '../src/core/run.js'; +import { type Tool, ToolRegistry } from '../src/core/tool.js'; +import { type Asker, questionTool } from '../src/plugins/tools/question.js'; +import { kilo, models, room } from './setup.js'; +import { fail, passed, under, wrongIf } from './report.js'; + +const system = + 'You are a test harness with tools. Call a tool whenever one answers the ' + + 'question. When you have what a tool said, answer the user in one short ' + + "sentence that repeats the tool's words exactly. Never guess a value a tool " + + 'can give you.'; + +/** A word no model invents, so an answer carrying it came from the tool. */ +const codes: Readonly> = { Oslo: 'kestrel', Lisbon: 'marmoset' }; + +const ran: { readonly city: string; readonly at: number; readonly done: number }[] = []; + +/** What the tool was asked for, whatever the model wrapped it in. */ +const cityIn = (args: string): string => { + const found = Object.keys(codes).find(city => args.includes(city)); + return found ?? 'Oslo'; +}; + +const weather = (takes: Duration.DurationInput): Tool => ({ + definition: { + name: 'weather', + description: 'The weather in one city. Call it once per city.', + parameters: { + type: 'object', + properties: { city: { type: 'string', description: 'The city to report on.' } }, + required: ['city'], + additionalProperties: false, + }, + }, + run: call => + Effect.gen(function* () { + const city = cityIn(call.arguments); + const at = Date.now(); + yield* Effect.sleep(takes); + ran.push({ city, at, done: Date.now() }); + return `The weather in ${city} is ${codes[city] ?? 'kestrel'}.`; + }), +}); + +const withTools = (kind: ApiKind, tools: readonly Tool[]) => + Layer.merge(kilo({ apiKinds: [kind] }), Layer.succeed(ToolRegistry, { tools })); + +/** + * One phase, tried once more before its miss counts. + * + * A phase says `undefined` when the round never happened, which is a relay + * having a bad minute rather than a finding. Nothing is recorded on the first + * attempt: `fail` writes for the life of the run, so a phase that recorded its + * own miss would make the retry a no-op. Only a round that failed twice is + * recorded, and only here. + */ +const twice = async (what: string, once: () => Promise): Promise => { + const first = await once(); + if (first === true) { + return true; + } + const second = await once(); + if (first === undefined && second === undefined) { + fail(`${what}: the round failed twice`); + } + return second === true; +}; + +/** + * One round on one shape: the model calls, reads, and answers with the word. + * + * Says whether the model called on this shape, or `undefined` if the round + * never happened. Whether it calls is its own — `minimax/minimax-m3` skipped + * `messages` on 2026-09-06 and carried it on the next run. Calling on none of + * the three is the shape failing, and the caller asserts it. + */ +const runShape = async (model: string, kind: ApiKind): Promise => { + ran.length = 0; + const program = Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens: room, tools: ['weather'] }); + return yield* said(session.ask('What is the weather in Oslo?')); + }); + const answer = await Effect.runPromise( + Effect.either(Effect.scoped(Effect.provide(program, withTools(kind, [weather(0)])))) + ); + + if (answer._tag === 'Left') { + console.log(`${kind.padEnd(18)}FAILED ${JSON.stringify(answer.left)}`); + return undefined; + } + console.log(`${kind.padEnd(18)}${String(ran.length).padEnd(10)}${JSON.stringify(answer.right)}`); + if (ran.length === 0) { + return false; + } + /* From here the tool ran, so the rest is the package's to get right: it ran + once for one question, and what it wrote is what the model answered with. */ + wrongIf(ran.length !== 1, `${kind}: the tool ran ${String(ran.length)} times for one question`); + wrongIf( + !answer.right.includes('kestrel'), + `${kind}: the tool ran and the answer did not carry what it said` + ); + return true; +}; + +/** + * Two calls in one turn have to overlap, or a slow tool costs the wall clock. + * + * Says whether this model sent them together. Whether it does is its own; that + * the package runs them together when it does is the floor. + */ +const runTogether = async (model: string): Promise => { + ran.length = 0; + const program = Effect.gen(function* () { + const session = yield* openSession({ system, model, maxTokens: room, tools: ['weather'] }); + return yield* said(session.ask('What is the weather in Oslo and in Lisbon?')); + }); + const got = await Effect.runPromise( + Effect.either( + Effect.scoped(Effect.provide(program, withTools('messages', [weather('600 millis')]))) + ) + ); + if (got._tag === 'Left') { + console.log(`\ntwo calls in one turn FAILED ${JSON.stringify(got.left)}`); + return undefined; + } + + const [first, second] = [...ran].sort((one, other) => one.at - other.at); + if (first === undefined || second === undefined) { + console.log(`\ntwo cities produced ${String(ran.length)} calls, not two`); + return false; + } + const overlap = first.done - second.at; + console.log(`\ntwo calls in one turn overlapped by ${String(overlap)}ms`); + return overlap > 0; +}; + +/** + * The question tool, answered slower than the model waits. + * + * This is the shape of every real question: nobody answers in the moment they + * are asked. The model must be told the question is out, answer without it, and + * then be asked again on its own when the answer lands. + */ +const runBackgrounded = async (model: string): Promise => { + const asker: Asker = questions => + Effect.as( + Effect.sleep('3 seconds'), + questions.map(question => ({ id: question.id, text: 'ultramarine' })) + ); + const tool = questionTool(asker, { inlineFor: Duration.millis(500) }); + + const program = Effect.gen(function* () { + const session = yield* openSession({ + /* The claim here is the harness's deadline, so the model's veto over it is + taken away. A model that sets `wait` is honoured, which is what + `runWanted` is for. */ + system: `${system} Never set wait on a tool call.`, + model, + maxTokens: room, + tools: ['question'], + }); + /* Watch first: the round the session starts on its own happens whether or + not anybody is listening, and a late subscriber hears none of it. */ + const watching = yield* Effect.fork( + Effect.timeout( + Stream.runFold( + /* One round and no more. The stream itself never ends: it carries + every round the session ever starts on its own. */ + Stream.takeUntil(session.continued, one => 'failed' in one || one.event.kind === 'done'), + '', + (held: string, one) => + !('failed' in one) && one.event.kind === 'delta' ? held + one.event.text : held + ), + '60 seconds' + ) + ); + const first = yield* said( + session.ask( + 'Ask me for my favourite colour with the question tool, then tell me what I said.' + ) + ); + return { first, later: yield* watching.await }; + }); + + const got = await Effect.runPromise( + Effect.either(Effect.scoped(Effect.provide(program, withTools('messages', [tool])))) + ); + + if (got._tag === 'Left') { + console.log(`\nbackgrounded FAILED ${JSON.stringify(got.left)}`); + return undefined; + } + + const { first, later } = got.right; + const told = later._tag === 'Success' ? later.value : ''; + console.log(`\nasked, not waited: ${JSON.stringify(first)}`); + console.log(`told later: ${JSON.stringify(told)}`); + + /* The asker takes three seconds and the deadline is half of one, so the word + cannot have been handed over yet. A model that says it anyway guessed, and + a guess is the model's own: it never left a question outstanding, so there + is nothing here for the late round to carry. */ + if (first.includes('ultramarine')) { + return false; + } + wrongIf( + !told.includes('ultramarine'), + 'the model left the question outstanding and the session never told it what the person answered' + ); + return true; +}; + +/** + * The model choosing not to wait, on a tool that never asked to be backgrounded. + * + * This is the one claim a fake cannot make: that a real model reads the `wait` + * field it is offered and answers it. The tool takes three seconds and names no + * deadline of its own, so the session's thirty would hold the model there. Only + * the model's own `wait: false` moves it on — and the answer still arrives, in + * a round of its own, exactly as a deadline's would. + */ +const runWanted = async (model: string): Promise => { + ran.length = 0; + const program = Effect.gen(function* () { + const session = yield* openSession({ + system: `${system} A tool call you set wait to false on runs without you. Set wait to false whenever a call would take a while and you can say something useful before it answers.`, + model, + maxTokens: room, + tools: ['weather'], + }); + const watching = yield* Effect.fork( + Effect.timeout( + Stream.runFold( + Stream.takeUntil(session.continued, one => 'failed' in one || one.event.kind === 'done'), + '', + (held: string, one) => + !('failed' in one) && one.event.kind === 'delta' ? held + one.event.text : held + ), + '60 seconds' + ) + ); + const first = yield* said( + session.ask( + 'What is the weather in Oslo? Do not wait for the tool — tell me you have asked, and I will hear the rest when it answers.' + ) + ); + return { first, later: yield* watching.await }; + }); + + const got = await Effect.runPromise( + Effect.either( + Effect.scoped(Effect.provide(program, withTools('messages', [weather('3 seconds')]))) + ) + ); + + if (got._tag === 'Left') { + console.log(`\nwait: false FAILED ${JSON.stringify(got.left)}`); + return undefined; + } + + const { first, later } = got.right; + const told = later._tag === 'Success' ? later.value : ''; + console.log(`\ndid not wait: ${JSON.stringify(first)}`); + console.log(`told later: ${JSON.stringify(told)}`); + + /* It waited. `wait` is a field the model is offered, not one it is held to, + so this is the model deciding and not the package failing to honour it. */ + if (first.includes('kestrel')) { + return false; + } + /* Read from the tool and not from the answer. `xiaomi/mimo-v2.5` and + `minimax/minimax-m3` both said they had asked about Oslo on 2026-09-06 and + called nothing at all: the words of a model that never called and the words + of one that walked away from a call are the same words, and only this tells + them apart. Nothing was sent away, so no late round is owed. */ + if (ran.length === 0) { + console.log('the model said it had asked and never called the tool'); + return false; + } + wrongIf( + !told.includes('kestrel'), + 'the model set wait to false and the call it walked away from never came back' + ); + return true; +}; + +/** Which models made each choice. The floor is that some model made it. */ +const chose: Record = { together: [], backgrounded: [], wanted: [] }; + +for (const model of models) { + under(model); + console.log('model', model); + console.log('\nshape calls answered'); + + const shapes = ['messages', 'responses', 'chat_completions'] as const; + const called: ApiKind[] = []; + for (const kind of shapes) { + if (await twice(kind, () => runShape(model, kind))) { + called.push(kind); + } + } + wrongIf( + called.length === 0, + 'the model called the tool on none of the three shapes, which is the shape failing rather than one bad round' + ); + const made = { + together: await twice('two calls in one turn', () => runTogether(model)), + backgrounded: await twice('the backgrounded round', () => runBackgrounded(model)), + wanted: await twice('the round the model chose not to wait for', () => runWanted(model)), + }; + for (const [phase, held] of Object.entries(chose)) { + if (made[phase as keyof typeof made]) { + held.push(model); + } + } +} + +under(''); +for (const [phase, held] of Object.entries(chose)) { + console.log(`\n${phase.padEnd(14)}${String(held.length)} of ${String(models.length)} models`); + wrongIf( + held.length === 0, + `not one model ${phase === 'together' ? 'sent two calls of a turn together' : phase === 'backgrounded' ? 'left a question outstanding' : 'set wait to false'}, which is the package and not a model's choice` + ); +} + +passed( + 'every shape ran a tool, and where a model chose to overlap, to leave a question outstanding, or not to wait, the package carried it' +); diff --git a/packages/harness-sdk/e2e/tsconfig.json b/packages/harness-sdk/e2e/tsconfig.json new file mode 100644 index 0000000000..d9da299d4f --- /dev/null +++ b/packages/harness-sdk/e2e/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "es2023", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["esnext", "dom"], + "types": ["node"], + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "verbatimModuleSyntax": true, + "isolatedModules": true, + "skipLibCheck": true, + "noEmit": true, + + "plugins": [{ "transform": "typia/lib/transform" }] + }, + "include": ["**/*.ts"] +} diff --git a/packages/harness-sdk/migrations/0000_blushing_vance_astro.sql b/packages/harness-sdk/migrations/0000_blushing_vance_astro.sql new file mode 100644 index 0000000000..0e1fa37d9b --- /dev/null +++ b/packages/harness-sdk/migrations/0000_blushing_vance_astro.sql @@ -0,0 +1,17 @@ +CREATE TABLE `sessions` ( + `id` text PRIMARY KEY NOT NULL, + `system` text NOT NULL, + `model` text NOT NULL, + `effort` text, + `max_tokens` integer +); +--> statement-breakpoint +CREATE TABLE `turns` ( + `id` text PRIMARY KEY NOT NULL, + `session_id` text NOT NULL, + `role` text NOT NULL, + `content` text NOT NULL, + FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE INDEX `turns_session_id_id` ON `turns` (`session_id`,`id`); \ No newline at end of file diff --git a/packages/harness-sdk/migrations/0001_tranquil_reptil.sql b/packages/harness-sdk/migrations/0001_tranquil_reptil.sql new file mode 100644 index 0000000000..b26777b50e --- /dev/null +++ b/packages/harness-sdk/migrations/0001_tranquil_reptil.sql @@ -0,0 +1,13 @@ +CREATE TABLE `parts` ( + `id` text PRIMARY KEY NOT NULL, + `turn_id` text NOT NULL, + `session_id` text NOT NULL, + `kind` text NOT NULL, + `body` text NOT NULL, + `media` text, + FOREIGN KEY (`turn_id`) REFERENCES `turns`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE INDEX `parts_session_id_id` ON `parts` (`session_id`,`id`);--> statement-breakpoint +ALTER TABLE `turns` DROP COLUMN `content`; \ No newline at end of file diff --git a/packages/harness-sdk/migrations/0002_bent_susan_delgado.sql b/packages/harness-sdk/migrations/0002_bent_susan_delgado.sql new file mode 100644 index 0000000000..2d85dc9bb7 --- /dev/null +++ b/packages/harness-sdk/migrations/0002_bent_susan_delgado.sql @@ -0,0 +1 @@ +ALTER TABLE `parts` ADD `signature` text; \ No newline at end of file diff --git a/packages/harness-sdk/migrations/0003_flaky_gideon.sql b/packages/harness-sdk/migrations/0003_flaky_gideon.sql new file mode 100644 index 0000000000..1f7d931cdc --- /dev/null +++ b/packages/harness-sdk/migrations/0003_flaky_gideon.sql @@ -0,0 +1 @@ +ALTER TABLE `sessions` ADD `prompted` integer; \ No newline at end of file diff --git a/packages/harness-sdk/migrations/0004_keen_boomer.sql b/packages/harness-sdk/migrations/0004_keen_boomer.sql new file mode 100644 index 0000000000..7971031116 --- /dev/null +++ b/packages/harness-sdk/migrations/0004_keen_boomer.sql @@ -0,0 +1,4 @@ +ALTER TABLE `parts` ADD `call_id` text;--> statement-breakpoint +ALTER TABLE `parts` ADD `name` text;--> statement-breakpoint +ALTER TABLE `parts` ADD `failed` integer;--> statement-breakpoint +ALTER TABLE `sessions` ADD `tools` text; \ No newline at end of file diff --git a/packages/harness-sdk/migrations/meta/0000_snapshot.json b/packages/harness-sdk/migrations/meta/0000_snapshot.json new file mode 100644 index 0000000000..e3f1f1ceb6 --- /dev/null +++ b/packages/harness-sdk/migrations/meta/0000_snapshot.json @@ -0,0 +1,124 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b74fdfbc-63b3-4db2-875b-b7d012f2b0a8", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "system": { + "name": "system", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "effort": { + "name": "effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_tokens": { + "name": "max_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turns": { + "name": "turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "turns_session_id_id": { + "name": "turns_session_id_id", + "columns": [ + "session_id", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turns_session_id_sessions_id_fk": { + "name": "turns_session_id_sessions_id_fk", + "tableFrom": "turns", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/harness-sdk/migrations/meta/0001_snapshot.json b/packages/harness-sdk/migrations/meta/0001_snapshot.json new file mode 100644 index 0000000000..a08f151492 --- /dev/null +++ b/packages/harness-sdk/migrations/meta/0001_snapshot.json @@ -0,0 +1,205 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "68926c80-2152-4d2c-ad46-610fc0ba72d2", + "prevId": "b74fdfbc-63b3-4db2-875b-b7d012f2b0a8", + "tables": { + "parts": { + "name": "parts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "media": { + "name": "media", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "parts_session_id_id": { + "name": "parts_session_id_id", + "columns": [ + "session_id", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "parts_turn_id_turns_id_fk": { + "name": "parts_turn_id_turns_id_fk", + "tableFrom": "parts", + "tableTo": "turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "parts_session_id_sessions_id_fk": { + "name": "parts_session_id_sessions_id_fk", + "tableFrom": "parts", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "system": { + "name": "system", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "effort": { + "name": "effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_tokens": { + "name": "max_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turns": { + "name": "turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "turns_session_id_id": { + "name": "turns_session_id_id", + "columns": [ + "session_id", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turns_session_id_sessions_id_fk": { + "name": "turns_session_id_sessions_id_fk", + "tableFrom": "turns", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/harness-sdk/migrations/meta/0002_snapshot.json b/packages/harness-sdk/migrations/meta/0002_snapshot.json new file mode 100644 index 0000000000..330b92ecad --- /dev/null +++ b/packages/harness-sdk/migrations/meta/0002_snapshot.json @@ -0,0 +1,212 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "3b035da9-83c4-43e1-bee9-6e26e416d532", + "prevId": "68926c80-2152-4d2c-ad46-610fc0ba72d2", + "tables": { + "parts": { + "name": "parts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "media": { + "name": "media", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "parts_session_id_id": { + "name": "parts_session_id_id", + "columns": [ + "session_id", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "parts_turn_id_turns_id_fk": { + "name": "parts_turn_id_turns_id_fk", + "tableFrom": "parts", + "tableTo": "turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "parts_session_id_sessions_id_fk": { + "name": "parts_session_id_sessions_id_fk", + "tableFrom": "parts", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "system": { + "name": "system", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "effort": { + "name": "effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_tokens": { + "name": "max_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turns": { + "name": "turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "turns_session_id_id": { + "name": "turns_session_id_id", + "columns": [ + "session_id", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turns_session_id_sessions_id_fk": { + "name": "turns_session_id_sessions_id_fk", + "tableFrom": "turns", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/harness-sdk/migrations/meta/0003_snapshot.json b/packages/harness-sdk/migrations/meta/0003_snapshot.json new file mode 100644 index 0000000000..b6fb083da0 --- /dev/null +++ b/packages/harness-sdk/migrations/meta/0003_snapshot.json @@ -0,0 +1,219 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "10918ac5-62b6-4029-b3a9-4825f6278059", + "prevId": "3b035da9-83c4-43e1-bee9-6e26e416d532", + "tables": { + "parts": { + "name": "parts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "media": { + "name": "media", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "parts_session_id_id": { + "name": "parts_session_id_id", + "columns": [ + "session_id", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "parts_turn_id_turns_id_fk": { + "name": "parts_turn_id_turns_id_fk", + "tableFrom": "parts", + "tableTo": "turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "parts_session_id_sessions_id_fk": { + "name": "parts_session_id_sessions_id_fk", + "tableFrom": "parts", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "system": { + "name": "system", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "effort": { + "name": "effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_tokens": { + "name": "max_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prompted": { + "name": "prompted", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turns": { + "name": "turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "turns_session_id_id": { + "name": "turns_session_id_id", + "columns": [ + "session_id", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turns_session_id_sessions_id_fk": { + "name": "turns_session_id_sessions_id_fk", + "tableFrom": "turns", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/harness-sdk/migrations/meta/0004_snapshot.json b/packages/harness-sdk/migrations/meta/0004_snapshot.json new file mode 100644 index 0000000000..800c08fb37 --- /dev/null +++ b/packages/harness-sdk/migrations/meta/0004_snapshot.json @@ -0,0 +1,247 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "a24da24e-5958-44d8-bd2b-1dc98e157216", + "prevId": "10918ac5-62b6-4029-b3a9-4825f6278059", + "tables": { + "parts": { + "name": "parts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "media": { + "name": "media", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "call_id": { + "name": "call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "failed": { + "name": "failed", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "parts_session_id_id": { + "name": "parts_session_id_id", + "columns": [ + "session_id", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "parts_turn_id_turns_id_fk": { + "name": "parts_turn_id_turns_id_fk", + "tableFrom": "parts", + "tableTo": "turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "parts_session_id_sessions_id_fk": { + "name": "parts_session_id_sessions_id_fk", + "tableFrom": "parts", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "system": { + "name": "system", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "effort": { + "name": "effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_tokens": { + "name": "max_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prompted": { + "name": "prompted", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tools": { + "name": "tools", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turns": { + "name": "turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "turns_session_id_id": { + "name": "turns_session_id_id", + "columns": [ + "session_id", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turns_session_id_sessions_id_fk": { + "name": "turns_session_id_sessions_id_fk", + "tableFrom": "turns", + "tableTo": "sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/harness-sdk/migrations/meta/_journal.json b/packages/harness-sdk/migrations/meta/_journal.json new file mode 100644 index 0000000000..b3f8b2a196 --- /dev/null +++ b/packages/harness-sdk/migrations/meta/_journal.json @@ -0,0 +1,41 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1788471476648, + "tag": "0000_blushing_vance_astro", + "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1788473842176, + "tag": "0001_tranquil_reptil", + "breakpoints": true + }, + { + "idx": 2, + "version": "6", + "when": 1788482366363, + "tag": "0002_bent_susan_delgado", + "breakpoints": true + }, + { + "idx": 3, + "version": "6", + "when": 1788513858691, + "tag": "0003_flaky_gideon", + "breakpoints": true + }, + { + "idx": 4, + "version": "6", + "when": 1788521930480, + "tag": "0004_keen_boomer", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/packages/harness-sdk/package.json b/packages/harness-sdk/package.json new file mode 100644 index 0000000000..b873aadf68 --- /dev/null +++ b/packages/harness-sdk/package.json @@ -0,0 +1,78 @@ +{ + "name": "@kilocode/harness-sdk", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": "./dist/index.js", + "./core": "./dist/core/index.js", + "./plugins/fetch": "./dist/plugins/fetch/web.js", + "./plugins/gateway": "./dist/plugins/gateway/index.js", + "./plugins/prompt": "./dist/plugins/prompt/default.js", + "./plugins/tools": "./dist/plugins/tools/index.js", + "./plugins/store/node": "./dist/plugins/store/node.js", + "./plugins/store/expo": "./dist/plugins/store/expo.js", + "./testing": "./dist/core/conformance.js" + }, + "scripts": { + "build": "rm -rf dist && ttsc -p tsconfig.build.json", + "check": "pnpm check:ci && pnpm test:perf", + "check:boundaries": "! grep -rln \"plugins/\" src/core --include='*.ts' | grep -v -e '\\.test\\.ts$' -e '\\-fixture\\.ts$'", + "check:ci": "pnpm typecheck && pnpm typecheck:e2e && pnpm lint && pnpm check:boundaries && pnpm check:migrations && pnpm test && pnpm build && pnpm check:platform && pnpm check:package", + "check:migrations": "ttsx --project scripts/tsconfig.json scripts/check-migrations.ts", + "check:package": "ttsx --project scripts/tsconfig.json scripts/check-package.ts", + "check:platform": "ttsx --project scripts/tsconfig.json scripts/check-platform.ts", + "lint": "pnpm -w exec oxlint --config packages/harness-sdk/.oxlintrc.json packages/harness-sdk/src", + "migrations": "drizzle-kit generate && ttsx --project scripts/tsconfig.json scripts/inline-migrations.ts && pnpm -w exec oxfmt packages/harness-sdk/src/plugins/store/migrations.ts", + "test": "vitest run --passWithNoTests", + "test:e2e": "ttsx e2e/live.ts", + "test:e2e:all": "ttsx e2e/all.ts", + "test:e2e:cancel": "ttsx e2e/cancel.ts", + "test:e2e:compact": "ttsx e2e/compact.ts", + "test:e2e:conversation": "ttsx e2e/conversation.ts", + "test:e2e:clone": "ttsx e2e/clone.ts", + "test:e2e:image": "ttsx e2e/image.ts", + "test:e2e:models": "ttsx e2e/models.ts", + "test:e2e:queue": "ttsx e2e/queue.ts", + "test:e2e:probe": "ttsx e2e/probe.ts", + "test:e2e:resume": "ttsx e2e/resume.ts", + "test:e2e:replay": "ttsx e2e/replay.ts", + "test:e2e:reasoning": "ttsx e2e/reasoning.ts", + "test:e2e:subagent": "ttsx e2e/subagent.ts", + "test:e2e:session": "ttsx e2e/session.ts", + "test:e2e:shapes": "ttsx e2e/shapes.ts", + "test:e2e:stop": "ttsx e2e/stop.ts", + "test:e2e:together": "ttsx e2e/together.ts", + "test:e2e:tool-matrix": "ttsx e2e/tool-matrix.ts", + "test:e2e:time": "ttsx e2e/time.ts", + "test:e2e:todo": "ttsx e2e/todo.ts", + "test:e2e:tools": "ttsx e2e/tools.ts", + "test:perf": "vitest run --config vitest.perf.config.ts", + "test:watch": "vitest", + "typecheck": "ttsc --noEmit", + "typecheck:e2e": "ttsc --noEmit -p e2e/tsconfig.json" + }, + "dependencies": { + "drizzle-orm": "catalog:", + "effect": "3.22.1", + "eventsource-parser": "3.0.8", + "typia": "14.0.4" + }, + "devDependencies": { + "@anthropic-ai/sdk": "0.104.1", + "@ttsc/unplugin": "0.28.3", + "@types/node": "catalog:", + "drizzle-kit": "catalog:", + "openai": "6.49.0", + "ttsc": "0.28.3", + "typescript": "7.0.2", + "vitest": "catalog:" + }, + "files": [ + "dist", + "README.md", + "PLUGINS.md" + ] +} diff --git a/packages/harness-sdk/scripts/check-migrations.ts b/packages/harness-sdk/scripts/check-migrations.ts new file mode 100644 index 0000000000..3dd853707a --- /dev/null +++ b/packages/harness-sdk/scripts/check-migrations.ts @@ -0,0 +1,49 @@ +import { execFileSync } from 'node:child_process'; +import { join } from 'node:path'; + +/** + * Fails when the checked-in migrations do not match the schema. + * + * The SQL is generated from `schema.ts` and then inlined into the bundle, so + * three things have to agree: the schema, the SQL under `migrations/`, and + * `src/plugins/store/migrations.ts`. Nothing at run time notices when they do + * not — the store simply applies migrations that no longer describe the + * columns the queries select, and the first symptom is a broken read on a + * device. + * + * So this regenerates them and asks git whether anything moved. It is the + * price of inlining, and it is why inlining is allowed. + */ + +const root = join(import.meta.dirname, '..'); +const watched = ['migrations', 'src/plugins/store/migrations.ts']; + +const git = (...args: readonly string[]): string => + execFileSync('git', args, { cwd: root, encoding: 'utf8' }); + +/* A dirty tree before the run would be reported as drift afterwards, which + would be a lie about what caused it. */ +const dirtyBefore = git('status', '--porcelain', '--', ...watched).trim(); +if (dirtyBefore !== '') { + process.stdout.write( + `the migrations are already modified, so drift cannot be told from your own edits:\n${dirtyBefore}\n` + ); + process.exit(1); +} + +execFileSync('pnpm', ['migrations'], { cwd: root, stdio: 'inherit' }); + +const drift = git('status', '--porcelain', '--', ...watched).trim(); +if (drift === '') { + process.stdout.write('the migrations match the schema\n'); + process.exit(0); +} + +/* What it generated is left where it is: on a real drift that is the answer, + and the message says how to throw it away when it is not. */ +process.stdout.write( + `the migrations do not match the schema. Run \`pnpm migrations\` and commit what it writes,\n` + + `or \`git checkout -- ${watched.join(' ')}\` and clean up \`migrations/\` to undo this run:\n${drift}\n\n` + + git('diff', '--', ...watched) +); +process.exit(1); diff --git a/packages/harness-sdk/scripts/check-package.ts b/packages/harness-sdk/scripts/check-package.ts new file mode 100644 index 0000000000..fdbee8298b --- /dev/null +++ b/packages/harness-sdk/scripts/check-package.ts @@ -0,0 +1,186 @@ +import { Effect, Layer, Stream } from 'effect'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import type { ModelEvent } from '../src/core/model.js'; +import type { SessionHandle } from '../src/core/handle.js'; +import type { SessionOptions } from '../src/core/wiring.js'; +import type { KiloSetup } from '../src/plugins/kilo.js'; + +/** + * Fails when the built package does not import, or does not run. + * + * Every test in this repository runs against `src/`. A consumer runs against + * `dist/`, reached through the `exports` map, and the two are not the same + * code: typia's validators are compiled into the build, the imports are + * rewritten, and a subpath that resolves to nothing fails at a consumer's + * first import with nothing here to have caught it. + * + * So this asks each entry point for one name it promises, and then asks one + * session a question through the built gateway against a `fetch` that answers + * from memory. That last part is the point: it runs a compiled validator over + * a stream event, which no other check does. + * + * It reads `dist/`, so `pnpm build` runs first. + */ + +const root = join(import.meta.dirname, '..'); + +interface Entry { + readonly subpath: string; + readonly promises: readonly string[]; +} + +/** Every `exports` subpath, and a name a caller reaches it for. */ +const entries: readonly Entry[] = [ + { subpath: '.', promises: ['layerKilo', 'openSession', 'ModelClient', 'said'] }, + { subpath: './core', promises: ['ModelClient', 'SessionStore', 'wiringFor', 'makeId'] }, + { subpath: './plugins/fetch', promises: ['webFetch'] }, + { subpath: './plugins/gateway', promises: ['layerKiloGateway'] }, + { subpath: './plugins/prompt', promises: ['assemble', 'layerAssembler'] }, + { + subpath: './plugins/tools', + promises: ['questionTool', 'subagentTool', 'timeTool', 'todoTool'], + }, + { subpath: './plugins/store/node', promises: ['layerNodeStore'] }, + { subpath: './plugins/store/expo', promises: ['layerExpoStore'] }, + { subpath: './testing', promises: ['checkStore', 'checkAssembler'] }, +]; + +/** What the main entry must not carry, because nobody runs it in production. */ +const withheld: Readonly> = { + '.': ['checkStore', 'checkAssembler', 'webFetch'], +}; + +const map = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')) as { + exports: Readonly>; +}; + +const broken: string[] = []; + +/* The table above is written by hand, and a subpath added to `package.json` + and not to it is a subpath nothing here loads. Two lines beat remembering. */ +const listed = new Set(entries.map(entry => entry.subpath)); +const unchecked = Object.keys(map.exports).filter(subpath => !listed.has(subpath)); +if (unchecked.length > 0) { + broken.push(`${unchecked.join(', ')} is in the exports map and not in this check`); +} + +/* And an entry point nobody is told about is an entry point nobody imports. + The README's own table is what is read, not the whole page: a subpath named + once in a code example is still a subpath missing from the list a consumer + reads to find out what there is. */ +const readme = readFileSync(join(root, 'README.md'), 'utf8'); +const table = readme.slice(readme.indexOf('## Entry points')); +const undocumented = Object.keys(map.exports).filter( + subpath => !table.includes(`\`@kilocode/harness-sdk${subpath.slice(1)}\` |`) +); +if (undocumented.length > 0) { + broken.push(`${undocumented.join(', ')} is an entry point the README does not name`); +} + +const fileOf = (subpath: string): string | undefined => { + const target = map.exports[subpath]; + return target === undefined ? undefined : join(root, target); +}; + +const loaded: Record> = {}; + +for (const entry of entries) { + const file = fileOf(entry.subpath); + if (file === undefined) { + broken.push(`${entry.subpath} is not in the exports map`); + continue; + } + try { + const module = (await import(pathToFileURL(file).href)) as Record; + loaded[entry.subpath] = module; + const missing = entry.promises.filter(name => module[name] === undefined); + if (missing.length > 0) { + broken.push(`${entry.subpath} exports no ${missing.join(', no ')}`); + } + } catch (cause) { + broken.push(`${entry.subpath} does not import: ${String(cause)}`); + } +} + +/* An entry point is what a consumer bundles. A name that leaks back into the + main one takes its whole file with it, and nothing in a type says so. */ +for (const [subpath, names] of Object.entries(withheld)) { + const module = loaded[subpath]; + const found = names.filter(name => module?.[name] !== undefined); + if (found.length > 0) { + broken.push(`${subpath} exports ${found.join(', ')}, which belongs to another entry point`); + } +} + +/** One streamed answer, so a compiled validator has something to read. */ +const frames = [ + { type: 'message_start', message: { usage: { input_tokens: 5 } } }, + { type: 'content_block_delta', delta: { text: 'built' } }, + { type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 1 } }, +] + .map(frame => `data: ${JSON.stringify(frame)}\n\n`) + .join(''); + +const answering = () => + Promise.resolve({ + ok: true, + status: 200, + text: () => Promise.resolve(''), + stream: async function* stream() { + yield frames; + }, + }); + +const rootModule = loaded['.']; +if (rootModule === undefined) { + process.stdout.write(`the built package does not import:\n${broken.join('\n')}\n`); + process.exit(1); +} + +/* The build is supposed to match the types its own source declares, so those + are what the values coming out of `dist/` are read as. The imports are types + only, and nothing of `src/` is loaded. */ +const layerKilo = rootModule['layerKilo'] as (setup: KiloSetup) => Layer.Layer; +const openSession = rootModule['openSession'] as ( + options: SessionOptions +) => Effect.Effect; + +try { + const said = await Effect.runPromise( + Effect.scoped( + Effect.provide( + Effect.flatMap(openSession({ system: 'sys', model: 'm', maxTokens: 8 }), session => + Stream.runFold(session.ask('hi'), '', (held: string, event: ModelEvent) => + event.kind === 'delta' ? held + event.text : held + ) + ), + layerKilo({ + baseUrl: 'https://gateway.test', + org: { kind: 'personal' }, + fetch: answering, + token: 'tok', + fallback: { apiKinds: ['messages'] }, + }) + ) + ) + ); + if (said !== 'built') { + broken.push(`a session built from dist/ answered ${JSON.stringify(said)}, not "built"`); + } +} catch (cause) { + broken.push(`a session built from dist/ could not answer: ${String(cause)}`); +} + +if (broken.length === 0) { + process.stdout.write( + `the built package imports and answers, from all ${String(entries.length)} entry points\n` + ); + process.exit(0); +} + +process.stdout.write( + `the built package is not what it promises. Check the exports map and the build:\n${broken.join('\n')}\n` +); +process.exit(1); diff --git a/packages/harness-sdk/scripts/check-platform.ts b/packages/harness-sdk/scripts/check-platform.ts new file mode 100644 index 0000000000..0d2d1f1681 --- /dev/null +++ b/packages/harness-sdk/scripts/check-platform.ts @@ -0,0 +1,115 @@ +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; + +/** + * Fails when the built package names something it must not: a platform in the + * code, or a dev dependency in the published types. + * + * Principle 12 says anything a runtime does differently is a plugin point, and + * `tsconfig.json` sets `"types": []` so a first-party `process` or `Buffer` is + * a compile error. That is all the compiler can do: it cannot see a + * dependency's own imports, and `skipLibCheck` removes the rest of the + * leverage. So the rule is checked by reading the build, which AGENTS.md has + * asked a person to do by hand until now. + * + * It reads `dist/`, so `pnpm build` runs first. + */ + +const root = join(import.meta.dirname, '..'); +const dist = join(root, 'dist'); + +const filesUnder = (folder: string, ending: string): readonly string[] => + readdirSync(folder).flatMap(name => { + const path = join(folder, name); + if (statSync(path).isDirectory()) { + return filesUnder(path, ending); + } + return path.endsWith(ending) ? [path] : []; + }); + +/** + * Each rule is a pattern that must not appear, and the files it applies to. + * The patterns match code, not prose: `node:crypto` is named in a comment in + * `core/id.ts` explaining why the package does not import it, and that comment + * is the opposite of a violation. + */ +const rules: readonly { + readonly what: string; + readonly pattern: RegExp; + readonly where: (path: string) => boolean; + readonly why: string; +}[] = [ + { + what: 'an import of a Node builtin', + pattern: /(?:from|import|require)\s*\(?\s*['"]node:/u, + where: () => true, + why: 'the package must import on a runtime that has no Node builtins', + }, + { + what: '`globalThis`', + pattern: /\bglobalThis\b/u, + where: path => path.startsWith('core/'), + why: 'reading the global is a plugin’s job, and every plugin lives outside core/', + }, + { + what: '`process` or `Buffer`', + pattern: /\b(?:process\.[a-z]|Buffer\.)/u, + where: path => path.startsWith('core/'), + why: 'neither exists in a browser or in a React Native release build', + }, +]; + +const broken: string[] = []; + +for (const file of filesUnder(dist, '.js')) { + const path = relative(dist, file); + const lines = readFileSync(file, 'utf8').split('\n'); + for (const rule of rules) { + if (!rule.where(path)) { + continue; + } + lines.forEach((line, index) => { + if (rule.pattern.test(line)) { + broken.push(`dist/${path}:${String(index + 1)} names ${rule.what}: ${line.trim()}`); + } + }); + } +} + +/** + * A dependency used only for its types must not reach the published types. + * + * `openai` and `@anthropic-ai/sdk` are the contract the three wires are written + * against, and nothing of either survives the build: every import of them is a + * type. That is what lets them be dev dependencies, which is two large packages + * a consumer does not install. Exporting a type built out of one puts it back + * in a `.d.ts`, and the consumer's own typecheck then fails on a package they + * were never told to add. The compiler cannot see this: here they are + * installed. + */ +const buildOnly = ['openai', '@anthropic-ai/sdk']; + +for (const file of filesUnder(dist, '.d.ts')) { + const path = relative(dist, file); + const lines = readFileSync(file, 'utf8').split('\n'); + lines.forEach((line, index) => { + const named = buildOnly.find(name => line.includes(`'${name}'`)); + if (named !== undefined) { + broken.push( + `dist/${path}:${String(index + 1)} names ${named}, which a consumer does not install` + ); + } + }); +} + +if (broken.length === 0) { + process.stdout.write('the build names no platform and no dev dependency\n'); + process.exit(0); +} + +process.stdout.write( + `the build names something it must not. Make a platform a plugin point, or move it under plugins/. Stop exporting a type built out of a dev dependency:\n${broken.join('\n')}\n\n` + + rules.map(rule => ` ${rule.what}: ${rule.why}`).join('\n') + + '\n' +); +process.exit(1); diff --git a/packages/harness-sdk/scripts/inline-migrations.ts b/packages/harness-sdk/scripts/inline-migrations.ts new file mode 100644 index 0000000000..060c837c1b --- /dev/null +++ b/packages/harness-sdk/scripts/inline-migrations.ts @@ -0,0 +1,52 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +/** + * Turns the SQL drizzle-kit wrote into a TypeScript module. + * + * The store applies migrations from an array held in the bundle, never from + * files on disk: React Native has no filesystem to read them from. This script + * is the bridge, and it runs at author time only. + */ + +interface Journal { + readonly entries: readonly { readonly tag: string }[]; +} + +const root = join(import.meta.dirname, '..'); +const folder = join(root, 'migrations'); +const target = join(root, 'src', 'plugins', 'store', 'migrations.ts'); +const breakpoint = '--> statement-breakpoint'; + +const journal = JSON.parse(readFileSync(join(folder, 'meta', '_journal.json'), 'utf8')) as Journal; + +const statementsOf = (tag: string): readonly string[] => + readFileSync(join(folder, `${tag}.sql`), 'utf8') + .split(breakpoint) + .map(statement => statement.trim()) + .filter(statement => statement.length > 0); + +const quote = (statement: string): string => ` ${JSON.stringify(statement)},`; + +const render = (tag: string): string => + [` /* ${tag} */`, ' [', ...statementsOf(tag).map(quote), ' ],'].join('\n'); + +const body = [ + '/* Generated by `pnpm migrations`. Edit the schema and run it again. */', + '', + '/**', + ' * Every migration, oldest first, each one a list of statements to run in', + ' * order. The version a database has applied is its index plus one, held in', + " * SQLite's own `user_version`, so the store needs no table of its own to know", + ' * where it stands.', + ' */', + 'const migrations: readonly (readonly string[])[] = [', + ...journal.entries.map(entry => render(entry.tag)), + '];', + '', + 'export { migrations };', + '', +].join('\n'); + +writeFileSync(target, body); +process.stdout.write(`${String(journal.entries.length)} migrations inlined into ${target}\n`); diff --git a/packages/harness-sdk/scripts/tsconfig.json b/packages/harness-sdk/scripts/tsconfig.json new file mode 100644 index 0000000000..46f594ad1b --- /dev/null +++ b/packages/harness-sdk/scripts/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "es2023", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["esnext"], + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "verbatimModuleSyntax": true, + "isolatedModules": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["**/*.ts"] +} diff --git a/packages/harness-sdk/src/core/ask.ts b/packages/harness-sdk/src/core/ask.ts new file mode 100644 index 0000000000..f95d4d0f65 --- /dev/null +++ b/packages/harness-sdk/src/core/ask.ts @@ -0,0 +1,120 @@ +import { Data, Effect, Ref, Stream } from 'effect'; +import { compactIfFull } from './compact.js'; +import { exchangeFor, remember, rollback } from './exchange.js'; +import { type Answer, roundsFrom } from './loop.js'; +import type { ModelError, ModelEvent } from './model.js'; +import type { StoreError } from './storage.js'; +import type { PartDraft } from './turn.js'; +import type { Wiring } from './wiring.js'; + +/** + * Something else already holds the session: a question was asked, or a + * compaction started, while an answer was still streaming. One session does + * one thing at a time, so the second is refused rather than queued. Wait for + * the stream to end, then try again. + */ +class SessionBusyError extends Data.TaggedError('harness/SessionBusyError')<{ + readonly sessionId: string; +}> {} + +/** + * What one question may change. Only `maxTokens` may: it never reaches the + * rendered prefix, so it costs no cache. The model, the system prompt, the + * effort, and the tools are frozen for the life of the session. + */ +interface AskOptions { + readonly maxTokens?: number; +} + +/** + * Adds the question, then builds the stream of everything that answers it. + * + * A turn is added to the session as it is made and to the store only when the + * whole exchange is done, because a half written exchange would poison the + * prefix of every later request. If no answer arrives, `rollback` takes the + * question back out again. What the rounds do is `loop.ts`. + */ +const answerOf = ( + wiring: Wiring, + input: string | readonly PartDraft[], + options: AskOptions | undefined +): Effect.Effect => + Effect.gen(function* () { + /* Before anything else. A session that has filled the window would be + refused, and compacting after the question was added would summarise the + question along with the answers it has not had yet. */ + yield* compactIfFull(wiring); + const exchange = yield* exchangeFor(wiring, input); + yield* remember(wiring, exchange.question); + const rounds = roundsFrom({ wiring, exchange, options }, wiring.tools.length > 0); + return Stream.ensuring(rounds, rollback(wiring, exchange)); + }); + +/** + * Asks the model with the session already held. + * + * It is what `askWith` does once it has the lock, and what the driver in + * `background.ts` uses under `whileFree`: that driver has to hold the session + * before it takes anything out of the line, or a message could be taken out, + * find the session busy, and be neither queued nor asked while a caller is + * looking at it. + */ +const askHeld = + (wiring: Wiring) => + ( + input: string | readonly PartDraft[], + options?: AskOptions + ): Stream.Stream => + Stream.unwrap(answerOf(wiring, input, options)); + +/** + * Asks the model and streams the reply. + * + * One session does one thing at a time: two answers at once would both build + * on the same prefix, and the second would miss the cache. A second question + * asked while the first still streams fails with `SessionBusyError`. It is + * refused rather than queued because a queued question cannot be released + * under `Stream.merge` — the merged stream holds every child resource until + * all children finish, so waiting would deadlock, and uninterruptibly. + */ +const askWith = + (wiring: Wiring) => + ( + input: string | readonly PartDraft[], + options?: AskOptions + ): Stream.Stream => + Stream.unwrap( + Effect.flatMap( + Ref.getAndSet(wiring.busy, true), + (held): Effect.Effect => + held + ? Effect.fail(new SessionBusyError({ sessionId: wiring.id })) + : answerOf(wiring, input, options).pipe( + Effect.map(answer => Stream.ensuring(answer, Ref.set(wiring.busy, false))), + Effect.onError(() => Ref.set(wiring.busy, false)) + ) + ) + ); + +/** + * Runs the work only when nothing else holds the session. + * + * Compaction rewrites the whole conversation, and a question in flight holds + * the session as it stood before it was asked, to put back if no answer comes. + * Both at once lose the summary from memory while the store keeps it. So + * compaction takes the same lock a question takes, and is refused the same way. + */ +const whileFree = ( + wiring: Wiring, + work: Effect.Effect +): Effect.Effect => + Effect.acquireUseRelease( + Effect.flatMap(Ref.getAndSet(wiring.busy, true), held => + held ? Effect.fail(new SessionBusyError({ sessionId: wiring.id })) : Effect.void + ), + () => work, + () => Ref.set(wiring.busy, false) + ); + +export type { AskOptions }; +export { askHeld, askWith, SessionBusyError, whileFree }; diff --git a/packages/harness-sdk/src/core/background.test.ts b/packages/harness-sdk/src/core/background.test.ts new file mode 100644 index 0000000000..5f36eff253 --- /dev/null +++ b/packages/harness-sdk/src/core/background.test.ts @@ -0,0 +1,191 @@ +import { Deferred, Duration, Effect, Fiber, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import type { ModelEvent } from './model.js'; +import { recordingStore, runWith } from './session-fixture.js'; +import { type Tool, ToolFailure, ToolRegistry } from './tool.js'; + +/** + * What happens to a call the model stopped waiting for. + * + * Every tool can be backgrounded, and the harness decides when, not the tool. + * The model is told the call is still running and carries on; the work keeps + * going; and when it finally answers, the session starts a round of its own to + * say so. Nothing about that is optional or opt-in — a person answering a + * question, a build finishing, a deploy landing: all of them outlive a request, + * and a harness that could only wait would be useless for every one of them. + */ + +const tool = (name: string, run: Tool['run'], inlineFor?: Duration.DurationInput): Tool => ({ + definition: { name, description: name, parameters: { type: 'object', properties: {} } }, + ...(inlineFor === undefined ? {} : { inlineFor }), + run, +}); + +const registry = (...tools: readonly Tool[]) => Layer.succeed(ToolRegistry, { tools }); + +const options = { + system: 'sys', + model: 'claude-opus-5', + maxTokens: 1024, + tools: ['slow'], + /* Nothing waits: every call to this session's tool is backgrounded at once. */ + inlineFor: Duration.zero, +}; + +const call = { id: 'tc_1', name: 'slow', arguments: '{}' }; + +const resultsIn = (events: readonly ModelEvent[]) => + events.filter(event => event.kind === 'toolResult').map(event => event.result); + +/** The text of every request, so a test can see what the model was told. */ +const askedWith = (parts: readonly { readonly kind: string }[]): readonly string[] => + parts.filter(part => part.kind === 'text').map(part => String(Reflect.get(part, 'text'))); + +it('tells the model a call is still running rather than holding the request open', async () => { + const { value, calls } = await runWith({ + options, + tools: registry( + /* Never answers while the question is in flight. A harness that waited on + this would sit on an open request until the provider gave up. */ + tool('slow', () => Effect.never) + ), + replies: [{ deltas: [], calls: [call], stop: 'tools' }, { deltas: ['I will wait'] }], + use: session => + Effect.map(Stream.runCollect(session.ask('start the build')), chunk => [...chunk]), + }); + + const [result] = resultsIn(value); + expect(result).toMatchObject({ callId: 'tc_1', failed: false }); + expect(result?.body).toContain('still running'); + /* The exchange finished. The model answered on top of a call it never got. */ + expect(calls).toHaveLength(2); +}); + +it('starts a round of its own when the call finally answers', async () => { + const store = recordingStore(); + const seen = await runWith({ + options, + store: store.layer, + replies: [ + { deltas: [], calls: [call], stop: 'tools' }, + { deltas: ['I will wait'] }, + { deltas: ['the build passed'] }, + ], + tools: registry(tool('slow', () => Effect.succeed('exit 0'))), + use: session => + Effect.gen(function* () { + /* Watch before asking, so the round the session starts on its own has + somewhere to go the moment it happens. */ + const watching = yield* Effect.fork(Stream.runCollect(Stream.take(session.continued, 2))); + yield* Stream.runDrain(session.ask('start the build')); + return [...(yield* Fiber.join(watching))]; + }), + }); + + /* Nobody asked a question. The result landing is what started the round. */ + const deltas = seen.value.flatMap(one => + 'failed' in one || one.event.kind !== 'delta' ? [] : [one.event.text] + ); + expect(deltas).toEqual(['the build passed']); + /* And the request carried the answer as something the conversation said, + never as a second result for a call that was already answered. */ + const said = askedWith(seen.calls[2]?.prompt.messages.at(-1)?.parts ?? []); + expect(said.join('')).toContain('exit 0'); + expect(said.join('')).toContain('has finished'); +}); + +it('writes the round it started on its own, like any other exchange', async () => { + const store = recordingStore(); + await runWith({ + options, + store: store.layer, + replies: [ + { deltas: [], calls: [call], stop: 'tools' }, + { deltas: ['I will wait'] }, + { deltas: ['the build passed'] }, + ], + tools: registry(tool('slow', () => Effect.succeed('exit 0'))), + use: session => + Effect.gen(function* () { + const watching = yield* Effect.fork(Stream.runDrain(Stream.take(session.continued, 2))); + yield* Stream.runDrain(session.ask('start the build')); + yield* watching.await; + }), + }); + + /* Two exchanges: the one that was asked for, and the one the session ran on + its own. The second is in the store like anything else. */ + expect(store.seen).toEqual([ + 'user:start the build', + 'assistant:', + 'user:', + 'assistant:I will wait', + expect.stringContaining('user:The slow call you made earlier'), + 'assistant:the build passed', + 'flush', + ]); +}); + +it('tells the model a backgrounded call failed, in the round it starts', async () => { + const { calls } = await runWith({ + options, + replies: [ + { deltas: [], calls: [call], stop: 'tools' }, + { deltas: ['I will wait'] }, + { deltas: ['it did not work'] }, + ], + tools: registry(tool('slow', () => Effect.fail(new ToolFailure({ cause: 'exit 1' })))), + use: session => + Effect.gen(function* () { + const watching = yield* Effect.fork(Stream.runDrain(Stream.take(session.continued, 2))); + yield* Stream.runDrain(session.ask('start the build')); + yield* watching.await; + }), + }); + + const said = askedWith(calls[2]?.prompt.messages.at(-1)?.parts ?? []).join(''); + expect(said).toContain('has failed'); + expect(said).toContain('exit 1'); +}); + +it('waits for the question in flight rather than asking over it', async () => { + const gate = Effect.runSync(Deferred.make()); + const { calls } = await runWith({ + options, + replies: [ + { deltas: [], calls: [call], stop: 'tools' }, + { deltas: ['I will wait'] }, + { deltas: ['and now the build'] }, + ], + /* Answers immediately, so the result lands while the second round of the + first question is still streaming. */ + tools: registry(tool('slow', () => Effect.as(Deferred.await(gate), 'exit 0'))), + use: session => + Effect.gen(function* () { + const watching = yield* Effect.fork(Stream.runDrain(Stream.take(session.continued, 2))); + yield* Deferred.succeed(gate, true); + yield* Stream.runDrain(session.ask('start the build')); + yield* watching.await; + }), + }); + + /* Three calls and not four: the round the session started waited for the + question, rather than running beside it and missing the cache. */ + expect(calls).toHaveLength(3); + expect(askedWith(calls[2]?.prompt.messages.at(-1)?.parts ?? []).join('')).toContain('exit 0'); +}); + +it('continues nothing on its own when nothing is waiting', async () => { + const { calls } = await runWith({ + replies: [{ deltas: ['hello'] }], + use: session => + Effect.zipRight( + Stream.runDrain(session.ask('hi')), + /* Nothing joined the line, so this ends on the timeout rather than on + an event, and the session is none the worse for it. */ + Effect.ignore(Effect.timeout(Stream.runDrain(session.continued), Duration.millis(30))) + ), + }); + + expect(calls).toHaveLength(1); +}); diff --git a/packages/harness-sdk/src/core/background.ts b/packages/harness-sdk/src/core/background.ts new file mode 100644 index 0000000000..074a64b2ae --- /dev/null +++ b/packages/harness-sdk/src/core/background.ts @@ -0,0 +1,110 @@ +import { Duration, Effect, PubSub, Queue, Schedule, Stream } from 'effect'; +import { askHeld, type SessionBusyError, whileFree } from './ask.js'; +import type { ModelError } from './model.js'; +import { type Continued, takeRun, wake, type Waiting } from './queue.js'; +import type { StoreError } from './storage.js'; +import type { Wiring } from './wiring.js'; + +/** + * What the session says when nobody is streaming an answer out of it. + * + * Two things reach the model this way. A message a caller queued while the + * session was busy, and the result of a tool the model stopped waiting for. The + * second cannot go back as a tool result: the call it belongs to was already + * answered, and every shape refuses a second result for one call. So it goes + * back as something the conversation says, in a turn of its own. + * + * Either way the session asks the model without anybody having asked a + * question, and a caller watches through `session.continued`. That is the + * difference between this and waiting for the next question: a build that + * finishes, a person who answers ten minutes later, a message typed while the + * last answer was still arriving — each is work to do at that moment, not at + * whatever moment somebody next types. + */ + +/** + * How long to wait before trying again when a question is already streaming. + * + * One session does one thing at a time, so a round that is ready mid-question + * waits for it. Nothing is lost by waiting: what it will say is already in + * hand, and the round is built fresh on each attempt. + */ +const whenBusy = Schedule.spaced(Duration.millis(50)).pipe(Schedule.upTo(Duration.minutes(5))); + +const partsIn = (run: readonly Waiting[]) => run.flatMap(one => one.parts); + +const idsIn = (run: readonly Waiting[]) => run.map(one => one.id); + +/** + * One round: what waited goes in as a turn, the model answers, and everything + * it says reaches whoever is watching. + * + * The session is held before anything is taken out of the line, and for the + * whole round. That order is the point. Taking first and then finding the + * session busy would leave a message neither waiting nor asked, so a caller + * looking at `queued` would not see it and `cancel` would say it was too late + * while nothing had been sent. + */ +const attempt = (wiring: Wiring): Effect.Effect => + whileFree( + wiring, + Effect.flatMap(takeRun(wiring.pending), (run): Effect.Effect => { + const answering = idsIn(run); + const say = (one: Continued) => PubSub.publish(wiring.continued, one); + /* An empty run is what a cancelled message leaves behind. It starts no + round rather than asking the model nothing at all. */ + return run.length === 0 + ? Effect.void + : Stream.runForEach(askHeld(wiring)(partsIn(run), run[0]?.options), event => + say({ answering, event }) + ).pipe( + /* The round is what failed, and the caller is told so on the same + stream that carries every other round. */ + Effect.catchAll((failed: ModelError | StoreError) => + Effect.asVoid(say({ answering, failed })) + ) + ); + }) + ); + +/** + * Waits on the line for as long as the session lives. + * + * Forked into the session's own scope, so it stops when the session closes and + * runs whether or not anybody is watching. A queued message that nobody listens + * for is still asked; only the events go unseen, and the transcript holds them. + * + * A round is retried while the session is busy and given up on when the model + * or the store refuses it. The failure goes out on `continued` as one more + * thing that happened, marked with the message it was owed to. It is not a + * failure of the stream: the driver goes straight back to the line, and a + * caller who lost their subscription to the first refused round would never + * hear about any round after it. + */ +const drivePending = (wiring: Wiring): Effect.Effect => + Effect.forever( + Effect.zipRight( + Queue.take(wiring.pending.arrived), + attempt(wiring).pipe( + Effect.retry({ while: () => true, schedule: whenBusy }), + /* Five minutes of a session that never went free. Nothing was taken out + of the line, so the entries are still in it: they are shown by + `queued` and they can still be cancelled. The token that pointed at + them is spent, though, so the bell is rung again — otherwise the line + waits on the next thing to join it, and a caller's last message is + never asked. */ + Effect.catchAll(failed => + Effect.zipRight( + PubSub.publish(wiring.continued, { answering: [], failed }), + wake(wiring.pending) + ) + ) + ) + ) + ); + +/** What a caller reads to see the rounds it did not ask for. */ +const continuedOf = (wiring: Wiring): Stream.Stream => + Stream.fromPubSub(wiring.continued); + +export { continuedOf, drivePending }; diff --git a/packages/harness-sdk/src/core/catalog.ts b/packages/harness-sdk/src/core/catalog.ts new file mode 100644 index 0000000000..a0a8d82e4c --- /dev/null +++ b/packages/harness-sdk/src/core/catalog.ts @@ -0,0 +1,50 @@ +import { Context, Data, type Effect } from 'effect'; + +/** The three shapes the gateway speaks. A model does not always speak all three. */ +type ApiKind = 'messages' | 'responses' | 'chat_completions'; + +/** What the package needs to know about a model before it calls one. */ +interface ModelFacts { + /** Which shapes this model speaks. An empty list means the package cannot call it. */ + readonly apiKinds: readonly ApiKind[]; + /** The most tokens this model will produce, when the catalog knows the number. */ + readonly maxOutputTokens?: number; + /** + * The most tokens this model reads in one request, when the catalog knows the + * number. A session compacts itself when it fills a share of this. Without it + * a session never compacts: a guessed window would either cut a conversation + * that fit, or fail to save one that did not. + */ + readonly contextWindow?: number; +} + +class CatalogError extends Data.TaggedError('harness/CatalogError')<{ + readonly model: string; + readonly cause: unknown; +}> {} + +/** + * Answers what a model can do. + * + * This is a plugin because nobody publishes these facts in one place. The + * gateway resolves the shapes from the serving provider and exposes them + * nowhere, a hard-coded table goes stale, and a live lookup costs a request. + * The caller decides which trade it wants. + * + * It returns an Effect so a plugin may fetch. A plugin that fetches must cache, + * because this sits on the request path and one question asks two or three + * times: the gateway asks which shape to send, the session asks for the window + * before it decides whether to compact, and it asks for the output ceiling too + * unless the caller named one. + */ +interface ModelCatalogService { + readonly facts: (model: string) => Effect.Effect; +} + +class ModelCatalog extends Context.Tag('harness/ModelCatalog')< + ModelCatalog, + ModelCatalogService +>() {} + +export type { ApiKind, ModelCatalogService, ModelFacts }; +export { CatalogError, ModelCatalog }; diff --git a/packages/harness-sdk/src/core/ceiling.test.ts b/packages/harness-sdk/src/core/ceiling.test.ts new file mode 100644 index 0000000000..11821f5796 --- /dev/null +++ b/packages/harness-sdk/src/core/ceiling.test.ts @@ -0,0 +1,84 @@ +import { Effect, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { layerSeededEntropy } from '../plugins/entropy/seeded.js'; +import { fakeModel } from '../plugins/model/fake.js'; +import { layerAssembler } from '../plugins/prompt/default.js'; +import { openSession } from './run.js'; +import { catalogSaying, emptyCatalog, options, run } from './session-fixture.js'; + +it('raises the token ceiling for one question only', async () => { + const { calls } = await run([{ deltas: ['x'] }], session => + Effect.zipRight( + Stream.runDrain(session.ask('a', { maxTokens: 4096 })), + Stream.runDrain(session.ask('b')) + ) + ); + expect(calls.map(call => call.maxTokens)).toEqual([4096, 1024]); +}); + +it('takes the ceiling from the model catalog when the caller names none', async () => { + const model = fakeModel([{ deltas: ['x'] }]); + await Effect.runPromise( + Effect.provide( + Effect.scoped( + Effect.flatMap(openSession({ system: 'sys', model: 'm' }), session => + Stream.runDrain(session.ask('a')) + ) + ), + Layer.mergeAll(layerAssembler, catalogSaying(2048), layerSeededEntropy(1), model.layer) + ) + ); + expect(model.calls[0]?.maxTokens).toBe(2048); +}); + +it('lets the session and then the question beat the catalog', async () => { + const model = fakeModel([{ deltas: ['x'] }]); + await Effect.runPromise( + Effect.provide( + Effect.scoped( + Effect.flatMap(openSession({ ...options, maxTokens: 512 }), session => + Effect.zipRight( + Stream.runDrain(session.ask('a')), + Stream.runDrain(session.ask('b', { maxTokens: 99 })) + ) + ) + ), + Layer.mergeAll(layerAssembler, catalogSaying(2048), layerSeededEntropy(1), model.layer) + ) + ); + expect(model.calls.map(call => call.maxTokens)).toEqual([512, 99]); +}); + +it('falls back to 4096 when the catalog cannot name a limit', async () => { + const model = fakeModel([{ deltas: ['x'] }]); + await Effect.runPromise( + Effect.provide( + Effect.scoped( + Effect.flatMap(openSession({ system: 'sys', model: 'm' }), session => + Stream.runDrain(session.ask('a')) + ) + ), + Layer.mergeAll(layerAssembler, emptyCatalog, layerSeededEntropy(1), model.layer) + ) + ); + + /* A catalog that cannot answer must not stop the question. The ceiling is + the package's floor, not the catalog's opinion. */ + expect(model.calls[0]?.maxTokens).toBe(4096); +}); + +it('asks anyway when the catalog fails and the caller named a ceiling', async () => { + const model = fakeModel([{ deltas: ['x'] }]); + await Effect.runPromise( + Effect.provide( + Effect.scoped( + Effect.flatMap(openSession({ ...options, maxTokens: 77 }), session => + Stream.runDrain(session.ask('a')) + ) + ), + Layer.mergeAll(layerAssembler, emptyCatalog, layerSeededEntropy(1), model.layer) + ) + ); + + expect(model.calls[0]?.maxTokens).toBe(77); +}); diff --git a/packages/harness-sdk/src/core/compact.test.ts b/packages/harness-sdk/src/core/compact.test.ts new file mode 100644 index 0000000000..79cc265a0b --- /dev/null +++ b/packages/harness-sdk/src/core/compact.test.ts @@ -0,0 +1,275 @@ +import { Effect, Either, Fiber, Option, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { ModelError } from './model.js'; +import { textIn } from './prompt.js'; +import { catalogWindowed, options, recordingStore, runWith, texts } from './session-fixture.js'; + +/** + * A session grows until the model refuses the request. Compaction replaces the + * conversation with a summary of itself and replays nothing before it. + * + * The window is tiny in these tests and the usage is scripted, so the trigger + * fires on the call the test intends and not on a token count that drifts. + */ + +const window = 1000; + +/** A call that fills the window, so the next question compacts first. */ +const full = { deltas: ['answered'], usage: { inputTokens: 900, cacheReadTokens: 0 } }; +/** A call that does not. */ +const roomy = { deltas: ['answered'], usage: { inputTokens: 10, cacheReadTokens: 0 } }; + +const askTwice = runWith({ + replies: [full, { deltas: ['the notes'] }, { deltas: ['after'] }], + catalog: catalogWindowed(window), + use: session => + Effect.zipRight( + Effect.zipRight(Stream.runDrain(session.ask('one')), Stream.runDrain(session.ask('two'))), + session.history + ), +}); + +it('summarises the conversation once it fills the window', async () => { + const { value, calls } = await askTwice; + + /* The summariser is a call of its own, between the two questions, and it + carries the transcript plus the instruction. */ + expect(calls).toHaveLength(3); + expect(calls[1]?.prompt.messages.map(textIn).at(-1)).toContain('Summarise the conversation'); + + expect(texts(value)).toEqual([ + 'user:one', + 'assistant:answered', + 'user:', + 'user:two', + 'assistant:after', + ]); +}); + +it('asks the next question with the summary and nothing before it', async () => { + const { calls } = await askTwice; + + /* The whole point: the turns before the summary are gone from the prompt. + Keeping the recent ones verbatim is the shape that fails, because a + thinking block is signed against the history that stood when it was made. */ + const asked = calls[2]?.prompt.messages.map(textIn); + expect(asked).toEqual(['Summary of the conversation so far:\n\nthe notes', 'two']); +}); + +it('asks the summariser with the session key and the session effort', async () => { + /* The gateway reads `cacheKey` as the session, so a summary sent without it + routes on its own and pays full price for a prefix the session already has + cached — and this is the one call that resends everything. `effort` is part + of that key, so leaving it off would miss the entry even with the key. */ + const { calls } = await runWith({ + replies: [full, { deltas: ['the notes'] }, { deltas: ['after'] }], + catalog: catalogWindowed(window), + options: { ...options, effort: 'high' }, + use: session => + Effect.zipRight(Stream.runDrain(session.ask('one')), Stream.runDrain(session.ask('two'))), + }); + + expect(calls[1]).toMatchObject({ cacheKey: calls[0]?.cacheKey, effort: 'high' }); +}); + +it('leaves a session that still fits alone', async () => { + const { calls } = await runWith({ + replies: [roomy, { deltas: ['after'] }], + catalog: catalogWindowed(window), + use: session => + Effect.zipRight(Stream.runDrain(session.ask('one')), Stream.runDrain(session.ask('two'))), + }); + + /* Two questions, two calls. A summariser that ran here would cost a request + and throw the cache away for nothing. */ + expect(calls).toHaveLength(2); + expect(calls[1]?.prompt.messages.map(textIn)).toEqual(['one', 'answered', 'two']); +}); + +it('never compacts when the catalog names no window', async () => { + /* Guessing a window would either cut a conversation that fit, or fail to + save one that did not. Saying nothing is the honest answer. */ + const { calls } = await runWith({ + replies: [full, { deltas: ['after'] }], + use: session => + Effect.zipRight(Stream.runDrain(session.ask('one')), Stream.runDrain(session.ask('two'))), + }); + + expect(calls).toHaveLength(2); +}); + +it('counts what the cache read towards the window', async () => { + /* A cached prefix still fills the window. Counting only the uncached tokens + would let a long session run until the provider refused it. */ + const { calls } = await runWith({ + replies: [ + { deltas: ['answered'], usage: { inputTokens: 3, cacheReadTokens: 900 } }, + { deltas: ['the notes'] }, + { deltas: ['after'] }, + ], + catalog: catalogWindowed(window), + use: session => + Effect.zipRight(Stream.runDrain(session.ask('one')), Stream.runDrain(session.ask('two'))), + }); + + expect(calls).toHaveLength(3); +}); + +it('obeys a caller who sets the share', async () => { + const { calls } = await runWith({ + replies: [roomy, { deltas: ['the notes'] }, { deltas: ['after'] }], + catalog: catalogWindowed(window), + options: { ...options, compactAt: 0.001 }, + use: session => + Effect.zipRight(Stream.runDrain(session.ask('one')), Stream.runDrain(session.ask('two'))), + }); + + expect(calls).toHaveLength(3); +}); + +it('compacts when the caller says so, whatever the window says', async () => { + const { value, calls } = await runWith({ + replies: [roomy, { deltas: ['the notes'] }, { deltas: ['after'] }], + catalog: catalogWindowed(window), + use: session => + Effect.zipRight( + Effect.zipRight( + Effect.zipRight(Stream.runDrain(session.ask('one')), session.compact), + Stream.runDrain(session.ask('two')) + ), + session.history + ), + }); + + expect(calls).toHaveLength(3); + expect(calls[2]?.prompt.messages.map(textIn)).toEqual([ + 'Summary of the conversation so far:\n\nthe notes', + 'two', + ]); + /* The earlier turns are still the record of what happened. They are simply + not what the model is asked with. */ + expect(value.length).toBe(5); +}); + +it('writes the summary to the store, so a continued session starts there too', async () => { + const store = recordingStore(); + await runWith({ + replies: [full, { deltas: ['the notes'] }, { deltas: ['after'] }], + catalog: catalogWindowed(window), + store: store.layer, + use: session => + Effect.zipRight(Stream.runDrain(session.ask('one')), Stream.runDrain(session.ask('two'))), + }); + + expect(store.seen).toEqual([ + 'user:one', + 'assistant:answered', + 'user:', + 'user:two', + 'assistant:after', + 'flush', + ]); +}); + +it('stops charging the old prompt size against the new one', async () => { + /* After a compaction the request is small again, so a second compaction + must not fire on the size the call before it reported. */ + const { calls } = await runWith({ + replies: [full, { deltas: ['the notes'] }, roomy, { deltas: ['after'] }], + catalog: catalogWindowed(window), + use: session => + Effect.zipRight( + Effect.zipRight(Stream.runDrain(session.ask('one')), Stream.runDrain(session.ask('two'))), + Stream.runDrain(session.ask('three')) + ), + }); + + /* Three questions and one summariser. A second summariser would mean the + session compacted on a number that no longer described it. */ + expect(calls).toHaveLength(4); +}); + +it('leaves the session alone when the summary call fails, and tries again', async () => { + /* A summariser that cannot answer must not take the conversation with it. + The question fails, the turns stand as they were, and the next question + tries to compact again — the session is still too full not to. */ + const { value, calls } = await runWith({ + replies: [ + full, + { deltas: [], fail: new ModelError({ reason: 'transport', cause: 'the socket died' }) }, + { deltas: ['the notes'] }, + { deltas: ['after'] }, + ], + catalog: catalogWindowed(window), + use: session => + Effect.zipRight( + Effect.zipRight( + Stream.runDrain(session.ask('one')), + Effect.either(Stream.runDrain(session.ask('two'))) + ), + Effect.zipRight(Stream.runDrain(session.ask('three')), session.history) + ), + }); + + /* Four calls: the question, the summary that failed, the summary that did + not, and the question that followed it. */ + expect(calls).toHaveLength(4); + expect(texts(value)).toEqual([ + 'user:one', + 'assistant:answered', + 'user:', + 'user:three', + 'assistant:after', + ]); +}); + +it('refuses to compact while a question is still streaming', async () => { + /* Compaction rewrites the whole conversation, and a question already in + flight holds the session as it stood before it was asked, to put back if + no answer comes. The two together lose the summary from memory while the + store keeps it, so the session and its record disagree. One session does + one thing at a time, which is the rule `ask` already follows. */ + const { value, calls } = await runWith({ + replies: [{ deltas: ['said'], stall: true }, { deltas: ['after'] }], + use: session => + Effect.gen(function* () { + const reading = yield* Effect.fork(Stream.runDrain(session.ask('why'))); + yield* Effect.sleep('10 millis'); + const refused = yield* Effect.either(session.compact); + yield* Fiber.interrupt(reading); + /* The refusal must not hold the lock it did not take, so the session + still answers. */ + yield* Stream.runDrain(session.ask('and')); + return { refused, history: yield* session.history }; + }), + }); + + expect(Either.getLeft(value.refused).pipe(Option.map(error => error._tag))).toStrictEqual( + Option.some('harness/SessionBusyError') + ); + /* Two calls: the question that stalled, and the one after it. No summary was + ever asked for. */ + expect(calls).toHaveLength(2); + expect(texts(value.history)).toEqual(['user:and', 'assistant:after']); +}); + +it('counts what the summary call cost', async () => { + /* A summary is a call to the model like any other: it is billed, and a + caller reading `usage` to know what a session spent must see it. Leaving + it out under-reports every session that ever compacted. */ + const { value } = await runWith({ + replies: [ + { deltas: ['answered'], usage: { inputTokens: 900, outputTokens: 5 } }, + { deltas: ['the notes'], usage: { inputTokens: 40, outputTokens: 20 } }, + { deltas: ['after'], usage: { inputTokens: 7, outputTokens: 3 } }, + ], + catalog: catalogWindowed(1000), + use: session => + Effect.zipRight( + Effect.zipRight(Stream.runDrain(session.ask('one')), Stream.runDrain(session.ask('two'))), + session.usage + ), + }); + + expect(value).toMatchObject({ inputTokens: 947, outputTokens: 28 }); +}); diff --git a/packages/harness-sdk/src/core/compact.ts b/packages/harness-sdk/src/core/compact.ts new file mode 100644 index 0000000000..6f12776011 --- /dev/null +++ b/packages/harness-sdk/src/core/compact.ts @@ -0,0 +1,152 @@ +import { Effect, Option, Ref, Stream } from 'effect'; +import type { ModelError } from './model.js'; +import { appendTurn, sinceSummary } from './session.js'; +import { onStore, type StoreError } from './storage.js'; +import { add } from './usage.js'; +import { makeTurn } from './turn.js'; +import type { Wiring } from './wiring.js'; + +/** + * Compaction: the conversation is replaced by a summary of itself. + * + * A session grows until the model refuses the request. The answer is the simple + * one: summarise everything into a single message, start the next request with + * that summary, and replay nothing else. + * + * The shape matters. Summarising the old turns and keeping the recent ones + * verbatim looks better and is refused: a thinking block is signed against the + * whole history that stood when it was produced, so a retained turn replayed + * after a summary fails on its signature. Nothing carried over here is tied to + * the old transcript. + * + * Compaction throws the model cache away, because every byte of the prefix + * changes. That is the price of the session continuing at all. + */ + +/** + * What the summariser is told to keep. + * + * Everything before the summary is gone from the prompt, so the summary is all + * the model will have of that work. A summariser left to its own judgement + * writes a readable paragraph and drops the identifiers. + */ +const instruction = + 'Summarise the conversation above so it can continue without the earlier messages. ' + + 'Keep every fact, decision, name, number, identifier, file path, and open question ' + + 'that a later turn could need. Write compact notes, not prose. ' + + 'Add nothing that was not said, and do not answer anything.'; + +/** How the summary is introduced, so the model reads it as the record, not as a question. */ +const heading = 'Summary of the conversation so far:'; + +/** The tokens one call put in front of the model, cached or not. */ +const promptedOf = (usage: { + readonly inputTokens: number; + readonly cacheReadTokens: number; +}): number => usage.inputTokens + usage.cacheReadTokens; + +/** + * The share of the window a session may fill before it compacts. It leaves room + * for the answer and for the question that triggers the check. + */ +const defaultCompactAt = 0.8; + +/** + * Whether the last call filled enough of the window to compact now. + * + * The number comes from the provider's own count of the last request, so no + * tokeniser is needed and no estimate can drift. A catalog that does not name a + * window never compacts: guessing one would either truncate a conversation that + * fit, or fail to save one that did not. + */ +const windowOf = (wiring: Wiring): Effect.Effect> => + wiring.catalog.facts(wiring.model).pipe( + Effect.map(facts => Option.fromNullable(facts.contextWindow)), + Effect.orElseSucceed(() => Option.none()) + ); + +const isFull = (wiring: Wiring): Effect.Effect => + Effect.zipWith(Ref.get(wiring.prompted), windowOf(wiring), (prompted, window) => + Option.match(window, { + onNone: () => false, + onSome: size => prompted >= size * (wiring.compactAt ?? defaultCompactAt), + }) + ); + +/** + * Asks the model to summarise what the session holds now. + * + * The counts go into the session's total. A summary is a call like any other + * and is billed like one, so a caller reading `usage` to know what a session + * spent must see it; leaving it out under-reports every session that ever + * compacted. + */ +const summaryOf = (wiring: Wiring): Effect.Effect => + Effect.flatMap(Ref.get(wiring.state), session => { + const prompt = wiring.assembler.assemble({ + system: wiring.system, + turns: sinceSummary(session.turns), + }); + return wiring.client + .stream({ + prompt: { + system: prompt.system, + messages: [ + ...prompt.messages, + { role: 'user', parts: [{ kind: 'text', text: instruction }], cache: false }, + ], + }, + model: wiring.model, + maxTokens: wiring.summaryTokens ?? defaultSummaryTokens, + ...(wiring.effort === undefined ? {} : { effort: wiring.effort }), + /* The same key every other call of this session carries. The gateway + reads it as the session, so without it the summary routes on its own + and pays full price for a prefix the session already has cached. */ + cacheKey: wiring.id, + }) + .pipe( + Stream.tap(event => + event.kind === 'done' + ? Ref.update(wiring.totals, held => add(held, event.usage)) + : Effect.void + ), + Stream.runFold('', (held, event) => (event.kind === 'delta' ? held + event.text : held)) + ); + }); + +/** + * The ceiling on a summary. It is a wall, not a target: a summariser that runs + * past it produces a summary cut off mid-note, which is worse than a short one. + */ +const defaultSummaryTokens = 2048; + +/** + * Replaces the conversation with a summary of itself. + * + * The summary is a turn like any other, so it is written to the store and read + * back by a session that is continued later. The turns before it stay where + * they are: they are the record of what happened, and only the prompt starts + * after them. + */ +const compactSession = (wiring: Wiring): Effect.Effect => + Effect.gen(function* () { + const said = yield* summaryOf(wiring); + const turn = yield* makeTurn(wiring.entropy, { + sessionId: wiring.id, + role: 'user', + parts: [{ kind: 'summary', body: `${heading}\n\n${said}` }], + }); + yield* Ref.update(wiring.state, session => appendTurn(session, turn)); + /* The next request starts from the summary, so what the last one cost says + nothing about what the next one will, in memory and in the store alike. */ + yield* onStore(wiring.store, plugin => + plugin.append({ sessionId: wiring.id, turns: [turn], prompted: 0 }) + ); + yield* Ref.set(wiring.prompted, 0); + }); + +/** Compacts when the last call filled the window, and does nothing otherwise. */ +const compactIfFull = (wiring: Wiring): Effect.Effect => + Effect.flatMap(isFull(wiring), full => (full ? compactSession(wiring) : Effect.void)); + +export { compactIfFull, compactSession, defaultCompactAt, isFull, promptedOf }; diff --git a/packages/harness-sdk/src/core/conformance.ts b/packages/harness-sdk/src/core/conformance.ts new file mode 100644 index 0000000000..378288a327 --- /dev/null +++ b/packages/harness-sdk/src/core/conformance.ts @@ -0,0 +1,294 @@ +import { Clock, Effect, Option } from 'effect'; +import type { Prompt, PromptAssemblerService } from './prompt.js'; +import type { SessionStoreService, StoredExchange, StoredSession } from './storage.js'; +import type { Turn } from './turn.js'; + +/** + * What a plugin has to get right, run against the plugin. + * + * A plugin point is two functions and a type, so writing one is easy and + * getting one wrong is easier: a store that reorders turns, drops a signature, + * or loses a column typechecks, answers every call, and breaks the model cache + * one reload later. An assembler that rewrites an earlier message typechecks + * too, and costs the whole prefix on every question from then on. + * + * Neither of those shows up as an error. They show up as a bill. So the package + * ships the checks rather than describing them: run one against your plugin in + * whatever test runner you already have, and assert the answer is empty. Each + * answers a list of what it found, in the words the author needs, and neither + * fails: a store that refuses a write is a finding. PLUGINS.md has the rest. + */ + +/** What a plugin got wrong. Empty means it conforms. */ +type Broken = readonly string[]; + +const wrongIf = (broken: boolean, wrong: string): Broken => (broken ? [wrong] : []); + +/** + * Equal but for the order of an object's keys, and for a field that is absent + * rather than undefined. + * + * A store rebuilds what it gives back, so one that is right in every way that + * matters may still name its fields in another order or leave an optional one + * off. Comparing the JSON would call both a defect. + */ +const named = (held: object) => Object.entries(held).filter(([, value]) => value !== undefined); + +const same = (one: unknown, other: unknown): boolean => { + if (one === other) { + return true; + } + if (!(one instanceof Object) || !(other instanceof Object)) { + return false; + } + if (Array.isArray(one) || Array.isArray(other)) { + return ( + Array.isArray(one) && + Array.isArray(other) && + one.length === other.length && + one.every((item, at) => same(item, other[at])) + ); + } + const left = named(one); + return ( + left.length === named(other).length && + left.every(([key, value]) => same(value, Reflect.get(other, key))) + ); +}; + +/** + * Every part kind in one turn, so a store that drops a column is caught by the + * check rather than by a session that will not replay a year from now. + * + * The identifiers here sort in the order the turns and parts were made, because + * a real one does: `makeId` builds a ULID, and a store is allowed to read them + * back in that order rather than keeping a column for it. + */ +const answerFor = (sessionId: string): Turn => ({ + id: 'trn_2', + sessionId, + role: 'assistant', + parts: [ + { id: 'prt_2_1', kind: 'reasoning', body: 'thought about it', signature: 'sig_abc' }, + { id: 'prt_2_2', kind: 'redacted', body: 'ENCRYPTED' }, + { id: 'prt_2_3', kind: 'toolCall', body: '{"city":"Oslo"}', callId: 'tc_1', name: 'weather' }, + { id: 'prt_2_4', kind: 'toolResult', body: 'it rains', callId: 'tc_1', failed: false }, + { id: 'prt_2_5', kind: 'text', body: 'it rains in Oslo' }, + ], +}); + +const questionFor = (sessionId: string): Turn => ({ + id: 'trn_1', + sessionId, + role: 'user', + parts: [ + { id: 'prt_1_1', kind: 'text', body: 'what is the weather' }, + { id: 'prt_1_2', kind: 'image', body: 'aGk=', media: 'image/png' }, + ], +}); + +const laterFor = (sessionId: string): Turn => ({ + id: 'trn_3', + sessionId, + role: 'user', + parts: [{ id: 'prt_3_1', kind: 'text', body: 'and tomorrow' }], +}); + +const sessionFor = (id: string): StoredSession => ({ + id, + system: 'You are terse.', + model: 'anthropic/claude-haiku-4.5', + effort: 'medium', + maxTokens: 512, + tools: ['weather', 'question'], +}); + +/** Runs one store call and reports a refusal rather than raising it. */ +const tried = ( + work: Effect.Effect, + what: string +): Effect.Effect<{ readonly got: Option.Option; readonly wrong: Broken }> => + Effect.match(work, { + onFailure: (cause: unknown) => ({ + got: Option.none(), + wrong: [`${what} refused the call: ${String(cause)}`], + }), + onSuccess: (got: A) => ({ got: Option.some(got), wrong: [] }), + }); + +/** A session that was never written must read as nothing, not as an empty one. */ +const checkUnknown = (store: SessionStoreService, id: string): Effect.Effect => + Effect.map( + Effect.all({ + read: tried(store.read(`${id}_never`), 'read'), + load: tried(store.load(`${id}_never`), 'load'), + }), + ({ read, load }) => [ + ...read.wrong, + ...load.wrong, + ...wrongIf( + Option.getOrUndefined(read.got)?._tag === 'Some', + 'read answered Some for a session that was never created. It must answer None.' + ), + ...wrongIf( + (Option.getOrUndefined(load.got) ?? []).length > 0, + 'load answered turns for a session that was never created. It must answer none.' + ), + ] + ); + +/** What was written comes back as it was written, field for field. */ +const checkSession = (store: SessionStoreService, id: string): Effect.Effect => + Effect.gen(function* () { + const written = sessionFor(id); + const created = yield* tried(store.create(written), 'create'); + const read = yield* tried(store.read(id), 'read'); + const got = Option.flatten(read.got); + return [ + ...created.wrong, + ...read.wrong, + ...wrongIf(Option.isNone(got), 'read answered None for a session create was given.'), + ...wrongIf( + Option.isSome(got) && !same({ ...got.value, prompted: undefined }, written), + 'read gave back a session that is not the one create was given. Every field ' + + 'is reopened from the store, so one that is dropped reopens the session ' + + `differently. Written ${JSON.stringify(written)}, read back ` + + `${JSON.stringify(Option.getOrUndefined(got))}.` + ), + ]; + }); + +/** One exchange written, and whatever the store said about writing it. */ +const appended = (store: SessionStoreService, exchange: StoredExchange) => + tried(store.append(exchange), 'append'); + +/** Turns come back in the order they were written, byte for byte. */ +const checkTurns = (store: SessionStoreService, id: string): Effect.Effect => + Effect.gen(function* () { + const first: readonly Turn[] = [questionFor(id), answerFor(id)]; + const second: readonly Turn[] = [laterFor(id)]; + const one = yield* appended(store, { sessionId: id, turns: first, prompted: 11 }); + const two = yield* appended(store, { sessionId: id, turns: second, prompted: 22 }); + const flushed = yield* tried(store.flush(), 'flush'); + const loaded = yield* tried(store.load(id), 'load'); + const got = Option.getOrElse(loaded.got, (): readonly Turn[] => []); + return [ + ...one.wrong, + ...two.wrong, + ...flushed.wrong, + ...loaded.wrong, + ...wrongIf( + !same(got, [...first, ...second]), + 'load gave back turns that are not the ones append was given, in the order it ' + + 'was given them. A turn that comes back changed, reordered, or short of a ' + + 'part rebuilds the prompt prefix differently and misses the model cache on ' + + `every request from then on. Written ${JSON.stringify([...first, ...second])}, ` + + `read back ${JSON.stringify(got)}.` + ), + ]; + }); + +/** The count the last append carried is what a reopened session starts from. */ +const checkPrompted = (store: SessionStoreService, id: string): Effect.Effect => + Effect.map(tried(store.read(id), 'read'), read => { + const got = Option.getOrUndefined(Option.flatten(read.got))?.prompted; + return [ + ...read.wrong, + ...wrongIf( + got !== 22, + 'read gave back a prompted count of ' + + `${String(got)} after two appends of 11 and 22. It must be the last one: ` + + 'it is what decides whether a reopened session compacts before its next ' + + 'question, and a stale one compacts too early or not at all.' + ), + ]; + }); + +/** One session's turns must not reach another's. */ +const checkApart = (store: SessionStoreService, id: string): Effect.Effect => + Effect.gen(function* () { + const other = `${id}_other`; + yield* tried(store.create(sessionFor(other)), 'create'); + yield* appended(store, { sessionId: other, turns: [laterFor(other)], prompted: 1 }); + const loaded = yield* tried(store.load(id), 'load'); + const got = Option.getOrElse(loaded.got, (): readonly Turn[] => []); + return [ + ...loaded.wrong, + ...wrongIf( + got.some(turn => turn.sessionId !== id), + 'load gave back a turn belonging to another session. Every read is by ' + + 'session, and a store that answers across them puts one conversation into ' + + "another's prompt." + ), + ]; + }); + +/** + * Checks a `SessionStore` plugin against everything a session needs from one. + * + * It writes two sessions under identifiers of its own, so it is safe to run + * against a real store; nothing else is touched, and nothing is deleted, which + * a store has no method for. Run it against a fresh store for the clearest + * answer. + */ +const checkStore = (store: SessionStoreService): Effect.Effect => + Effect.gen(function* () { + const id = `ses_check_${String(yield* Clock.currentTimeMillis)}`; + const unknown = yield* checkUnknown(store, id); + const session = yield* checkSession(store, id); + const turns = yield* checkTurns(store, id); + const prompted = yield* checkPrompted(store, id); + const apart = yield* checkApart(store, id); + return [...unknown, ...session, ...turns, ...prompted, ...apart]; + }); + +const builtBy = (assembler: PromptAssemblerService, turns: readonly Turn[]) => + assembler.assemble({ system: 'You are terse.', turns }); + +/** + * What each message says, without its breakpoint. + * + * The breakpoint is a marker and not content: an assembler marks the last + * message so the next request reads everything before it, so the mark moves + * with every turn while nothing that was sent changes. Holding it against an + * assembler would fail the one this package ships. + */ +const said = (prompt: Prompt) => prompt.messages.map(({ cache: _cache, ...rest }) => rest); + +/** + * Checks a `PromptAssembler` plugin against the two invariants that decide + * whether the model cache is won or lost. + * + * Both are silent when broken. The same input giving different bytes, or an + * appended turn changing what came before it, moves the prefix: every request + * from then on writes the cache instead of reading it, and the only symptom is + * the bill. + */ +const checkAssembler = (assembler: PromptAssemblerService): Broken => { + const id = 'ses_check'; + const asked: readonly Turn[] = [questionFor(id), answerFor(id)]; + const before = builtBy(assembler, asked); + const after = builtBy(assembler, [...asked, laterFor(id)]); + return [ + ...wrongIf( + JSON.stringify(before) !== JSON.stringify(builtBy(assembler, asked)), + 'assemble gave different bytes for the same input. Something in it varies: a ' + + 'clock, a random value, or a key order. Every question would miss the cache.' + ), + ...wrongIf( + JSON.stringify(after.system) !== JSON.stringify(before.system), + 'assemble changed the system prompt when a turn was appended. It is the front ' + + 'of the cached prefix, so a change there costs the whole conversation on ' + + 'every question from then on.' + ), + ...wrongIf( + JSON.stringify(said(after).slice(0, before.messages.length)) !== JSON.stringify(said(before)), + 'assemble rewrote what came before an appended turn. Everything up to the new ' + + 'turn must be byte for byte what it was, or the prefix moves and the whole ' + + 'conversation is written to the cache again on every question.' + ), + ]; +}; + +export type { Broken }; +export { checkAssembler, checkStore }; diff --git a/packages/harness-sdk/src/core/entropy.ts b/packages/harness-sdk/src/core/entropy.ts new file mode 100644 index 0000000000..f1e818aef6 --- /dev/null +++ b/packages/harness-sdk/src/core/entropy.ts @@ -0,0 +1,36 @@ +import { Context, Data } from 'effect'; + +/** + * The platform could not supply randomness. Every runtime that has a source + * exposes it differently, and a mobile runtime may have none until a polyfill + * is installed, so this is reported at wiring time rather than at the first + * identifier. + */ +class EntropyError extends Data.TaggedError('harness/EntropyError')<{ + readonly cause: unknown; +}> {} + +/** + * Where random bytes come from. + * + * This is the one thing the package cannot do for itself on every runtime: + * Node, a browser, a worker and a mobile app each hold their randomness + * somewhere different. It is a plugin point so that the core needs no platform + * at all, and so a caller on a runtime the package has never seen can supply + * its own rather than wait for the package to learn about it. + * + * The call is synchronous because it sits on the identifier path, which runs + * twice per question. + */ +interface EntropySourceService { + /** Returns `count` random bytes. Each byte must be uniform over 0..255. */ + readonly bytes: (count: number) => Uint8Array; +} + +class EntropySource extends Context.Tag('harness/EntropySource')< + EntropySource, + EntropySourceService +>() {} + +export type { EntropySourceService }; +export { EntropyError, EntropySource }; diff --git a/packages/harness-sdk/src/core/exchange.ts b/packages/harness-sdk/src/core/exchange.ts new file mode 100644 index 0000000000..28433b30a5 --- /dev/null +++ b/packages/harness-sdk/src/core/exchange.ts @@ -0,0 +1,260 @@ +import { Effect, Ref } from 'effect'; +import { promptedOf } from './compact.js'; +import type { ModelEvent, ModelUsage, StopReason } from './model.js'; +import { appendTurn, type Session } from './session.js'; +import { onStore, type StoreError } from './storage.js'; +import type { ToolCall } from './tool.js'; +import { makeTurn, partsOf, type PartDraft, type Turn } from './turn.js'; +import { add } from './usage.js'; +import type { Wiring } from './wiring.js'; + +/** + * One question and everything it produces before the model stops asking. + * + * A question and its answer are written together or not at all. Half of an + * exchange is worse than none: a transcript that ends on an unanswered + * question sends it again with every later request, the caller pays for it + * each time, and the model may answer it late on top of whatever was asked + * next. So the turn goes into the session as the question is asked, into the + * store only when the answer arrives, and back out again when it does not. + * + * Tools make one question into several rounds — the model asks for a tool, the + * tool answers, the model is asked again — and that rule holds across all of + * them. Every shape refuses a call whose result is missing, so a store holding + * half a round holds a session nobody can continue. The turns are collected as + * they are made and written once, at the end, by `commit`. + * + * What drives the rounds is `loop.ts`. + */ + +/** Adds a turn to the session in memory. The store hears at the end of the exchange. */ +const remember = (wiring: Wiring, turn: Turn): Effect.Effect => + Ref.update(wiring.state, session => appendTurn(session, turn)); + +/** + * What is collected while one round streams, to become the assistant's turn. + * + * One record rather than a ref per field. Copying the other on every token + * costs 0.054 us against 0.402 for the update alone, measured 2026-09-04 over + * 200000 rounds, which is a third of a percent of the 18.1 us a token costs + * through the whole session. `endRound` reads it once. + */ +interface Spoken { + readonly text: string; + /** + * The thinking, in the order it arrived, encrypted blocks among the rest. + * + * It is a list and not a pair of fields because the provider refuses a turn + * whose thinking blocks do not come back in the order it produced them. A + * model that has part of its reasoning redacted returns thinking, then an + * encrypted block, then more thinking; holding the words in one field and + * the encrypted blocks in another loses which came first. + */ + readonly thought: readonly PartDraft[]; + /** + * The tools the model asked for, in the order it asked. A turn may hold + * several, and each is answered before the next request goes out. + */ + readonly calls: readonly PartDraft[]; +} + +const nothingSaid: Spoken = { text: '', thought: [], calls: [] }; + +interface Exchange { + readonly question: Turn; + /** What the round now streaming has said. Emptied at the start of each one. */ + readonly spoken: Ref.Ref; + /** + * Every turn this question has made: the answers, the calls, and the results. + * They are written to the store together, once, when the loop ends. + */ + readonly written: Ref.Ref; + /** Why the model stopped the round that just ended. `tools` means ask again. */ + readonly stop: Ref.Ref; + /** How many times the model has been asked. See `maxRounds`. */ + readonly rounds: Ref.Ref; + /** True once the store has it. See `rollback`. */ + readonly answered: Ref.Ref; + /** The session as it stood before the question, to go back to. */ + readonly before: Session; +} + +/** + * The thinking comes first, in the order the model produced it, then the words, + * then the tools it asked for. Every shape wants that order and refuses another. + * A reasoning block is kept even with no words: a provider that returns the + * thinking as a summary defaults to no summary at all, so the block is empty and + * still has to go back exactly as it came. + * + * The text part is always there when nothing else is: an answer of no words is + * still an answer, and a turn with no parts would shorten the prompt that + * follows. It is left out of a turn that asked for a tool and said nothing, + * because a provider refuses an empty text block beside a call. + */ +const partsSaid = (spoken: Spoken): readonly PartDraft[] => { + const said: readonly PartDraft[] = + spoken.text.length === 0 && spoken.calls.length > 0 + ? [] + : [{ kind: 'text', body: spoken.text }]; + return [...spoken.thought, ...said, ...spoken.calls]; +}; + +/** Keeps a turn this question made, in memory and on the list to be written. */ +const collect = (wiring: Wiring, exchange: Exchange, turn: Turn): Effect.Effect => + Effect.zipRight( + remember(wiring, turn), + Ref.update(exchange.written, held => [...held, turn]) + ); + +/** + * Closes one round: what the model said becomes a turn, and what the round cost + * goes into the session's total and into the count that decides compaction. + * + * The turn is not written to the store here. The model may be about to ask for + * a tool, and a call stored without its result is a session that cannot be + * continued, so the writing waits for `commit`. + */ +const endRound = ( + wiring: Wiring, + exchange: Exchange, + ended: { readonly usage: ModelUsage; readonly stop: StopReason } +): Effect.Effect => + Ref.get(exchange.spoken).pipe( + Effect.flatMap(spoken => + makeTurn(wiring.entropy, { + sessionId: wiring.id, + role: 'assistant', + parts: partsSaid(spoken), + }) + ), + Effect.tap(answer => collect(wiring, exchange, answer)), + Effect.tap(() => Ref.set(exchange.stop, ended.stop)), + Effect.tap(() => Ref.update(exchange.rounds, held => held + 1)), + /* What this call put in front of the model, which is what decides whether + the next question compacts first. It is the provider's own count, so no + tokeniser is needed and no estimate can drift. */ + Effect.tap(() => Ref.set(wiring.prompted, promptedOf(ended.usage))), + Effect.tap(() => Ref.update(wiring.totals, held => add(held, ended.usage))) + ); + +/** + * Writes the question and everything it produced, as one unit. + * + * The question is written here rather than when it was asked, so the store + * never holds a question with no answer, and never a call with no result. + */ +const commit = (wiring: Wiring, exchange: Exchange): Effect.Effect => + Effect.flatMap( + Effect.all({ turns: Ref.get(exchange.written), prompted: Ref.get(wiring.prompted) }), + ({ turns, prompted }) => + onStore(wiring.store, plugin => + plugin.append({ sessionId: wiring.id, turns: [exchange.question, ...turns], prompted }) + ) + ).pipe(Effect.zipRight(Ref.set(exchange.answered, true))); + +/** + * Collects one thinking event into the block it belongs to. + * + * The words and the signature arrive on separate events, so both land on the + * block still open. A block stays open until something else arrives: an + * encrypted block closes it, a signature closes it, and the next thinking event + * opens a new one. A model produces two signed blocks in a row between tool + * calls, and merging them would hand the provider one block under the other's + * seal. + */ +const thinking = ( + spoken: Ref.Ref, + event: Extract +): Effect.Effect => + Ref.update(spoken, held => { + /* Nothing said and nothing sealed. A provider ends a thinking block with + one of these, and opening a block on it leaves an unsigned block after + the signed one: the wire drops what it cannot sign, so the thinking would + go back to the provider with a hole in it. */ + if (event.text === '' && event.signature === undefined) { + return held; + } + const last = held.thought.at(-1); + const open = last?.kind === 'reasoning' && last.signature === undefined ? last : undefined; + const sealed = event.signature; + const grown: PartDraft = { + kind: 'reasoning', + body: (open?.body ?? '') + event.text, + ...(sealed === undefined ? {} : { signature: sealed }), + }; + const before = open === undefined ? held.thought : held.thought.slice(0, -1); + return { ...held, thought: [...before, grown] }; + }); + +/** + * Takes the question back out when no answer came. + * + * A transcript that ends on an unanswered question sends it again with every + * later request: the caller pays for it each time, and the model may answer it + * late, on top of whatever was asked next. Nothing else may have touched the + * session in between, because one session does one thing at a time — that is + * what `whileFree` in `ask.ts` holds, for a compaction as much as a question. + */ +const rollback = (wiring: Wiring, exchange: Exchange): Effect.Effect => + Effect.flatMap(Ref.get(exchange.answered), done => + done ? Effect.void : Ref.set(wiring.state, exchange.before) + ); + +/** Everything one question needs before it goes out, made in one place. */ +const exchangeFor = ( + wiring: Wiring, + input: string | readonly PartDraft[] +): Effect.Effect => + Effect.all({ + before: Ref.get(wiring.state), + question: makeTurn(wiring.entropy, { + sessionId: wiring.id, + role: 'user', + parts: partsOf(input), + }), + spoken: Ref.make(nothingSaid), + written: Ref.make([]), + stop: Ref.make('unknown'), + rounds: Ref.make(0), + answered: Ref.make(false), + }); + +/** Empties what the last round said, so the next one starts its own turn. */ +const nextRound = (exchange: Exchange): Effect.Effect => + Ref.set(exchange.spoken, nothingSaid); + +/** One piece of the answer's text. The only thing on the per-token path. */ +const said = (spoken: Ref.Ref, text: string): Effect.Effect => + Ref.update(spoken, held => ({ ...held, text: held.text + text })); + +/** One tool the model asked for, kept in the order it asked. */ +const called = (spoken: Ref.Ref, call: ToolCall): Effect.Effect => { + const part: PartDraft = { + kind: 'toolCall', + body: call.arguments, + callId: call.id, + name: call.name, + }; + return Ref.update(spoken, held => ({ ...held, calls: [...held.calls, part] })); +}; + +/** One block of thinking the provider encrypted, kept where it arrived. */ +const hidden = (spoken: Ref.Ref, data: string): Effect.Effect => { + const block: PartDraft = { kind: 'redacted', body: data }; + return Ref.update(spoken, held => ({ ...held, thought: [...held.thought, block] })); +}; + +export type { Exchange }; +export { + called, + collect, + commit, + endRound, + exchangeFor, + hidden, + nextRound, + remember, + rollback, + said, + thinking, +}; diff --git a/packages/harness-sdk/src/core/fetch.ts b/packages/harness-sdk/src/core/fetch.ts new file mode 100644 index 0000000000..8c31920d88 --- /dev/null +++ b/packages/harness-sdk/src/core/fetch.ts @@ -0,0 +1,41 @@ +/** + * The smallest part of `fetch` the package uses. The package declares it rather + * than pulling in the DOM library, so the core keeps running on Node, in a + * browser, and in a mobile app. The caller adapts its own `fetch`. + */ +interface HttpResponse { + readonly ok: boolean; + readonly status: number; + readonly text: () => Promise; + /** Decoded body chunks. A caller that streams must supply this. */ + readonly stream?: () => AsyncIterable; +} + +/** + * The signal a runtime's `AbortController` produces. + * + * The package declares it rather than pulling in the DOM library, and never + * reads it: it makes one per call and hands it over, so that a caller who stops + * listening stops the request as well. An adapter that calls the platform + * `fetch` names the type it has, as `(request.signal ?? null) as AbortSignal | + * null`. That cast is why no adapter ships here — only code that already has + * the runtime's own signal type can join the two. The README writes it out. + */ +interface AbortLike { + readonly aborted: boolean; +} + +interface HttpRequest { + readonly method: 'POST'; + readonly headers: Readonly>; + readonly body: string; + /** + * Aborted when the caller stops reading. Absent when the runtime has no + * `AbortController`, in which case a call cannot be stopped early. + */ + readonly signal?: AbortLike; +} + +type FetchLike = (url: string, request: HttpRequest) => Promise; + +export type { AbortLike, FetchLike, HttpRequest, HttpResponse }; diff --git a/packages/harness-sdk/src/core/handle.ts b/packages/harness-sdk/src/core/handle.ts new file mode 100644 index 0000000000..4895589812 --- /dev/null +++ b/packages/harness-sdk/src/core/handle.ts @@ -0,0 +1,121 @@ +import { Effect, Ref, type Stream } from 'effect'; +import { type AskOptions, askWith, type SessionBusyError, whileFree } from './ask.js'; +import { continuedOf, drivePending } from './background.js'; +import { compactSession } from './compact.js'; +import type { ModelError, ModelEvent, ModelUsage } from './model.js'; +import { cancelQueued, type Continued, enqueueMessage, type Waiting } from './queue.js'; +import type { StoreError } from './storage.js'; +import { backgroundNow, type RunningCall, runningIn } from './waiting.js'; +import type { PartDraft, Turn } from './turn.js'; +import type { Wiring } from './wiring.js'; + +/** A live session. It owns the turns, so a caller cannot lose one. */ +interface SessionHandle { + readonly id: string; + /** Asks the model. The stream ends with `done`, which carries this call's counts. */ + readonly ask: ( + input: string | readonly PartDraft[], + options?: AskOptions + ) => Stream.Stream; + /** + * Hands the session a message to ask when it is free, and answers with the + * identifier that cancels it. + * + * This is `ask` for a caller who cannot wait for the answer where they stand: + * a person typing while the last answer is still arriving. It never refuses, + * because it never competes for the session — it joins a line, and the line + * is answered in the order it formed. The answer arrives on `continued`, + * marked with the identifier this returns. + */ + readonly queue: ( + input: string | readonly PartDraft[], + options?: AskOptions + ) => Effect.Effect; + /** + * Takes a queued message back out again. True when it was still waiting. + * + * False means it was already asked, or was never here. Neither is an error: a + * caller racing their own cancel button against the session is ordinary, and + * a message the provider has already seen cannot be taken back. + */ + readonly cancel: (id: string) => Effect.Effect; + /** What is waiting to be said, in the order it will be. Empty when nothing is. */ + readonly queued: Effect.Effect; + /** + * The calls the model is waiting on right now, oldest first. Show it, or read + * it to decide what has waited long enough. + */ + readonly running: Effect.Effect; + /** + * Stops the model waiting for one call, now, and answers whether it was still + * waiting. + * + * The work is untouched: it keeps running, and what it says arrives in a + * round of its own, exactly as it would have on the deadline. This is the + * deadline brought forward by somebody who knows better than a fixed number — + * a person watching a call take too long, or an agent deciding it has waited + * enough. The session does not need to know which of them it was. + * + * False means the call has already been answered, has already gone to the + * background, or was never here. None of those is an error. + */ + readonly background: (callId: string) => Effect.Effect; + /** Every turn so far, oldest first. Appending a turn never changes this one. */ + readonly history: Effect.Effect; + /** The counts of every call so far. Pass to `hitRatio` for the cache share. */ + readonly usage: Effect.Effect; + /** + * Replaces the conversation with a summary of itself, now, whatever the + * window says. A session does this on its own when it fills the window; this + * is for a caller that knows sooner, such as one changing subject. + */ + readonly compact: Effect.Effect; + /** + * The rounds the session ran without being asked where the caller stands: a + * queued message, or a backgrounded tool that finally answered. Every event + * carries the identifiers of the queued entries its round answers, so one + * message's answer is told from another's. Reading it a second time starts a + * second subscription rather than continuing the first. + * + * `done` ends one call to the model and not the round: a round that calls a + * tool makes several, and every one of those but the last stops on `tools`. + * A queued message is answered in full on the first `done` that stops on + * anything else. + * + * A round the model or the store refused arrives here as `failed`, marked + * with the same identifiers, rather than as a failure of the stream. One + * message's bad news is not the end of the feed: the session keeps running + * rounds for the rest of the line, and a caller whose subscription died on + * the first refused round would hear about none of them. + * + * The rounds happen whether or not anybody reads this. A caller that does not + * watch loses the events, never the work, and the transcript holds all of it. + */ + readonly continued: Stream.Stream; +} + +/** + * The handle, and the thing that drives what the session does on its own. + * + * The driver is forked into the session's scope, so it lives as long as the + * session and stops with it. Every session starts one: any session can be + * queued to, whether or not it has a tool. + */ +const handleOf = (wiring: Wiring): Effect.Effect => + Effect.as(Effect.forkIn(drivePending(wiring), wiring.scope), { + id: wiring.id, + ask: askWith(wiring), + queue: (input: string | readonly PartDraft[], options?: AskOptions) => + enqueueMessage(wiring.pending, input, options), + cancel: (id: string) => cancelQueued(wiring.pending, id), + queued: Ref.get(wiring.pending.waiting), + running: runningIn(wiring), + background: (callId: string) => backgroundNow(wiring, callId), + history: Effect.map(Ref.get(wiring.state), session => session.turns), + usage: Ref.get(wiring.totals), + compact: whileFree(wiring, compactSession(wiring)), + continued: continuedOf(wiring), + }); + +export type { SessionHandle }; +export { handleOf }; diff --git a/packages/harness-sdk/src/core/id.test.ts b/packages/harness-sdk/src/core/id.test.ts new file mode 100644 index 0000000000..231098caf1 --- /dev/null +++ b/packages/harness-sdk/src/core/id.test.ts @@ -0,0 +1,39 @@ +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { seededEntropy } from '../plugins/entropy/seeded.js'; +import { makeId } from './id.js'; + +const entropy = seededEntropy(7); + +const many = (count: number): readonly string[] => + Effect.runSync(Effect.all(Array.from({ length: count }, () => makeId(entropy, 'trn')))); + +it('sorts by the order it made them, which is what the prompt prefix relies on', () => { + const ids = many(5000); + expect([...ids].toSorted()).toEqual(ids); +}); + +it('never repeats an identifier', () => { + const ids = many(5000); + expect(new Set(ids).size).toBe(ids.length); +}); + +it('encodes the time in the leading ten characters', () => { + const [id] = many(1); + const time = id?.slice('trn_'.length, 'trn_'.length + 10) ?? ''; + const alphabet = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; + const decoded = { value: 0 }; + for (let index = 0; index < time.length; index += 1) { + decoded.value = decoded.value * 32 + alphabet.indexOf(time.charAt(index)); + } + expect(Math.abs(decoded.value - Date.now())).toBeLessThan(1000); +}); + +it('carries into the next millisecond when the random part is exhausted', () => { + /* An entropy source pinned to the top of the range: every draw is 31, so the + next identifier in the same millisecond has nowhere to carry to. */ + const exhausted = { bytes: (count: number) => new Uint8Array(count).fill(255) }; + const ids = Effect.runSync(Effect.all(Array.from({ length: 3 }, () => makeId(exhausted, 'trn')))); + expect([...ids].toSorted()).toEqual(ids); + expect(new Set(ids).size).toBe(3); +}); diff --git a/packages/harness-sdk/src/core/id.ts b/packages/harness-sdk/src/core/id.ts new file mode 100644 index 0000000000..9b98d52958 --- /dev/null +++ b/packages/harness-sdk/src/core/id.ts @@ -0,0 +1,92 @@ +import { Effect } from 'effect'; +import type { EntropySourceService } from './entropy.js'; + +/** + * Makes an identifier of the form `{prefix}_{ulid}`. + * + * The ULID is built here rather than taken from the `ulid` package. That + * package resolves to a build that imports `node:crypto` under the `node` + * export condition, and to one that detects a global `crypto` at module scope + * otherwise — so importing it either drags a runtime into the core or throws + * on a runtime that has no global source yet. Both break a package that must + * run anywhere. The encoding below is about forty lines of arithmetic with no + * platform in it; the randomness comes from the `EntropySource` plugin and the + * time from Effect's `Clock`. + * + * The ordering is deliberately not pluggable. An identifier must sort by the + * order it was made in, because a store rebuilds the prompt prefix in that + * order and a prefix in the wrong order misses the model cache. A plugin + * returning a random identifier would typecheck, pass every test, and break + * that one reload later. + */ + +/** Crockford base 32, least ambiguous first. This is the ULID alphabet. */ +const alphabet = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; +const base = 32; +const timeLength = 10; +const randomLength = 16; + +const digit = (value: number): string => alphabet[value] ?? '0'; + +const encodeTime = (milliseconds: number): string => { + let left = milliseconds; + let encoded = ''; + for (let index = 0; index < timeLength; index += 1) { + encoded = digit(left % base) + encoded; + left = Math.floor(left / base); + } + return encoded; +}; + +/** 256 is a whole number of 32s, so a byte modulo the base stays uniform. */ +const draw = (entropy: EntropySourceService): number[] => { + const bytes = entropy.bytes(randomLength); + const digits: number[] = []; + for (let index = 0; index < randomLength; index += 1) { + digits.push((bytes[index] ?? 0) % base); + } + return digits; +}; + +/** Adds one to the random part, carrying left. False when all 80 bits are set. */ +const bump = (digits: number[]): boolean => { + for (let index = digits.length - 1; index >= 0; index -= 1) { + const value = digits[index] ?? 0; + if (value < base - 1) { + digits[index] = value + 1; + return true; + } + digits[index] = 0; + } + return false; +}; + +/** + * One module means one monotonic sequence. Two sequences can hand out the same + * millisecond twice, which is a flake that only shows up under load. + */ +const sequence = { time: -1, digits: [] as number[] }; + +const refill = (entropy: EntropySourceService): void => { + sequence.digits = draw(entropy); +}; + +const nextUlid = (entropy: EntropySourceService, now: number): string => { + if (now > sequence.time) { + sequence.time = now; + refill(entropy); + } else if (!bump(sequence.digits)) { + /* Eighty bits used inside one millisecond. Take the next millisecond + rather than hand out an identifier that sorts before the last one. */ + sequence.time += 1; + refill(entropy); + } + return encodeTime(sequence.time) + sequence.digits.map(digit).join(''); +}; + +const makeId = (entropy: EntropySourceService, prefix: string): Effect.Effect => + Effect.clockWith(clock => + Effect.map(clock.currentTimeMillis, now => `${prefix}_${nextUlid(entropy, now)}`) + ); + +export { makeId }; diff --git a/packages/harness-sdk/src/core/image.test.ts b/packages/harness-sdk/src/core/image.test.ts new file mode 100644 index 0000000000..af54a05555 --- /dev/null +++ b/packages/harness-sdk/src/core/image.test.ts @@ -0,0 +1,118 @@ +import { DatabaseSync } from 'node:sqlite'; +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { layerNodeStore } from '../plugins/store/node.js'; +import { assemble } from '../plugins/prompt/default.js'; +import { seededEntropy } from '../plugins/entropy/seeded.js'; +import { SessionStore, type SessionStoreService } from './storage.js'; +import { makeTurn, textOf, type PartDraft } from './turn.js'; + +/** A one-pixel PNG, small enough to read and real enough to be an image. */ +const pixel = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + +const entropy = seededEntropy(7); +const session = { id: 'ses_1', system: 'sys', model: 'claude-opus-5' }; + +const stored = (run: (store: SessionStoreService) => Effect.Effect): Promise => + Effect.runPromise( + Effect.provide(Effect.flatMap(SessionStore, run), layerNodeStore(new DatabaseSync(':memory:'))) + ); + +const question: readonly PartDraft[] = [ + { kind: 'text', body: 'what is in this picture' }, + { kind: 'image', body: pixel, media: 'image/png' }, +]; + +it('carries an image and its media type through the store, in place', async () => { + const loaded = await stored(store => + Effect.gen(function* () { + yield* store.create(session); + const turn = yield* makeTurn(entropy, { + sessionId: session.id, + role: 'user', + parts: question, + }); + yield* store.append({ sessionId: session.id, turns: [turn], prompted: 0 }); + return yield* store.load(session.id); + }) + ); + + /* The order inside a turn is the order the model reads. An image that came + back before its question would change what was asked. */ + expect(loaded[0]?.parts).toMatchObject([ + { kind: 'text', body: 'what is in this picture' }, + { kind: 'image', body: pixel, media: 'image/png' }, + ]); +}); + +it('gives every part its own identifier', async () => { + const turn = Effect.runSync( + makeTurn(entropy, { sessionId: session.id, role: 'user', parts: question }) + ); + + const ids = turn.parts.map(part => part.id); + expect(new Set(ids).size).toBe(2); + expect(ids.every(id => id.startsWith('prt_'))).toBe(true); +}); + +it('reads the text of a turn that also holds an image', () => { + const turn = Effect.runSync( + makeTurn(entropy, { sessionId: session.id, role: 'user', parts: question }) + ); + + expect(textOf(turn)).toBe('what is in this picture'); +}); + +it('refuses to load an image row that names no media type', async () => { + const database = new DatabaseSync(':memory:'); + const failed = await Effect.runPromise( + Effect.provide( + Effect.flatMap(SessionStore, store => + Effect.gen(function* () { + yield* store.create(session); + yield* store.append({ + sessionId: session.id, + turns: [{ id: 'trn_1', sessionId: session.id, role: 'user', parts: [] }], + prompted: 0, + }); + database + .prepare( + 'INSERT INTO parts (id, turn_id, session_id, kind, body) VALUES (?, ?, ?, ?, ?)' + ) + .run('prt_1', 'trn_1', session.id, 'image', pixel); + return yield* Effect.flip(store.load(session.id)); + }) + ), + layerNodeStore(database) + ) + ); + + expect(failed).toMatchObject({ operation: 'load' }); + expect(String(failed.cause)).toContain('names no media type'); +}); + +it('puts every part of the turn in the prompt, in the order it arrived', () => { + const turn = Effect.runSync( + makeTurn(entropy, { + sessionId: session.id, + role: 'assistant', + parts: [ + { kind: 'reasoning', body: 'thinking about it', signature: 'sig' }, + { kind: 'text', body: 'a picture' }, + { kind: 'image', body: pixel, media: 'image/png' }, + ], + }) + ); + + const prompt = assemble({ system: 'sys', turns: [turn] }); + + /* The reasoning goes back with the rest. The provider drops what the model + cannot read and does not bill for it, and a block removed by hand can fail + the request on its ordering or on its signature. */ + expect(prompt.messages[0]?.parts).toEqual([ + { kind: 'reasoning', text: 'thinking about it', signature: 'sig' }, + { kind: 'text', text: 'a picture' }, + { kind: 'image', media: 'image/png', data: pixel }, + ]); +}); diff --git a/packages/harness-sdk/src/core/index.ts b/packages/harness-sdk/src/core/index.ts new file mode 100644 index 0000000000..058cfe8b97 --- /dev/null +++ b/packages/harness-sdk/src/core/index.ts @@ -0,0 +1,22 @@ +export type { AskOptions } from './ask.js'; +export { SessionBusyError } from './ask.js'; +export * from './catalog.js'; +export * from './compact.js'; +export * from './conformance.js'; +export * from './entropy.js'; +export * from './fetch.js'; +export * from './handle.js'; +export * from './id.js'; +export * from './model.js'; +export * from './prompt.js'; +export * from './queue.js'; +export * from './retry.js'; +export * from './resume.js'; +export * from './run.js'; +export * from './session.js'; +export * from './storage.js'; +export * from './token.js'; +export * from './tool.js'; +export * from './turn.js'; +export * from './usage.js'; +export * from './wiring.js'; diff --git a/packages/harness-sdk/src/core/lock.test.ts b/packages/harness-sdk/src/core/lock.test.ts new file mode 100644 index 0000000000..9957b516b4 --- /dev/null +++ b/packages/harness-sdk/src/core/lock.test.ts @@ -0,0 +1,111 @@ +import { Effect, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { runWith } from './session-fixture.js'; +import { type Tool, type ToolDefinition, ToolRegistry } from './tool.js'; + +/** + * A session serialises nothing, and a tool that must not be re-entered says so + * itself. + * + * The session used to own this, as a `concurrent: false` flag and a permit the + * runner took before calling `run`. It was the wrong owner twice over. A permit + * per session locked nothing between two sessions, which was a plain bug; and + * the fix for that — one permit per tool object, kept by the core — had the core + * inventing an identity for a thing it does not own, to protect a thing it + * cannot see. What needs protecting is the terminal, the file, the person: all + * the caller's, all supplied by the caller along with the tool that touches + * them. So the caller holds the permit, in the tool, next to the thing. + * + * What that buys is the invariant these tests are for: **a session is + * independent of every other in every way.** There is nothing left in the core + * that two sessions share. + */ + +const options = { + system: 'sys', + model: 'claude-opus-5', + maxTokens: 1024, + tools: ['hold'], +}; + +const call = { id: 'tc_1', name: 'hold', arguments: '{}' }; + +const definition: ToolDefinition = { + name: 'hold', + description: 'hold', + parameters: { type: 'object', properties: {}, additionalProperties: false }, +}; + +/** Says when it starts and stops, and takes long enough that an overlap shows. */ +const marking = (seen: string[]): Effect.Effect => + Effect.sync(() => void seen.push('in')) + .pipe(Effect.flatMap(() => Effect.sleep('50 millis'))) + .pipe(Effect.tap(() => Effect.sync(() => void seen.push('out')))) + .pipe(Effect.as('held')); + +/** A tool that lets anything overlap, which is every tool by default. */ +const open = (seen: string[]): Tool => ({ definition, run: () => marking(seen) }); + +/** A tool that holds one thing, and holds a permit beside it. */ +const guarded = (seen: string[]): Tool => { + const permit = Effect.unsafeMakeSemaphore(1); + return { definition, run: () => permit.withPermits(1)(marking(seen)) }; +}; + +/** One session, asking for the tool once. */ +const asking = (tool: Tool) => + runWith({ + options, + tools: Layer.succeed(ToolRegistry, { tools: [tool] }), + replies: [{ deltas: [], calls: [call], stop: 'tools' }, { deltas: ['done'] }], + use: session => Stream.runCollect(session.ask('hold it')), + }); + +it('keeps two sessions out of a tool that holds its own permit', async () => { + const seen: string[] = []; + const tool = guarded(seen); + + await Promise.all([asking(tool), asking(tool)]); + + /* One in and out before the next in. This is a parent and its subagent over + one terminal, and the tool is the only party that knew there was one. */ + expect(seen).toStrictEqual(['in', 'out', 'in', 'out']); +}); + +it('leaves two sessions alone when the tool holds nothing', async () => { + const seen: string[] = []; + const tool = open(seen); + + await Promise.all([asking(tool), asking(tool)]); + + expect(seen).toStrictEqual(['in', 'in', 'out', 'out']); +}); + +it('serialises two calls in one turn, from inside the tool', async () => { + const seen: string[] = []; + const tool = guarded(seen); + + await runWith({ + options, + tools: Layer.succeed(ToolRegistry, { tools: [tool] }), + replies: [ + { deltas: [], calls: [call, { ...call, id: 'tc_2' }], stop: 'tools' }, + { deltas: ['done'] }, + ], + use: session => Stream.runCollect(session.ask('hold it twice')), + }); + + /* The runner starts both at once, because the model asks for several when + they are independent. The tool is what makes these two not independent. */ + expect(seen).toStrictEqual(['in', 'out', 'in', 'out']); +}); + +it('keeps two tools over two things out of each other’s way', async () => { + const seen: string[] = []; + + /* Two permits, because two tools built separately hold two different things. + Serialising one against the other would be a wait bought for nothing. */ + await Promise.all([asking(guarded(seen)), asking(guarded(seen))]); + + expect(seen).toStrictEqual(['in', 'in', 'out', 'out']); +}); diff --git a/packages/harness-sdk/src/core/loop.test.ts b/packages/harness-sdk/src/core/loop.test.ts new file mode 100644 index 0000000000..5622cbbf77 --- /dev/null +++ b/packages/harness-sdk/src/core/loop.test.ts @@ -0,0 +1,258 @@ +import { Duration, Effect, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import type { ModelEvent, ModelRequest } from './model.js'; +import { recordingStore, runWith, texts } from './session-fixture.js'; +import { type Tool, type ToolCall, ToolFailure, ToolRegistry } from './tool.js'; + +/** + * What a question does when the model asks for a tool. + * + * The rule the whole loop exists to keep is that a call and its result are one + * unit. The model asks, the tool answers, the model is asked again, and only + * when it stops asking does any of it reach the store. Every shape refuses a + * call whose result is missing, so a store that held half a round would hold a + * session nobody could continue. + */ + +const call = (id: string, name: string, args = '{}'): ToolCall => ({ id, name, arguments: args }); + +const tool = (name: string, run: Tool['run']): Tool => ({ + definition: { name, description: name, parameters: { type: 'object', properties: {} } }, + run, +}); + +const registry = (...tools: readonly Tool[]) => Layer.succeed(ToolRegistry, { tools }); + +const saying = (name: string, body: string) => tool(name, () => Effect.succeed(body)); + +const options = { + system: 'sys', + model: 'claude-opus-5', + maxTokens: 1024, + tools: ['weather'], +}; + +const resultsIn = (events: readonly ModelEvent[]) => + events.filter(event => event.kind === 'toolResult').map(event => event.result); + +/** The parts of one request, as a reader of the wire would see them. */ +const shapeOf = (request: ModelRequest | undefined): readonly string[] => + (request?.prompt.messages ?? []).flatMap(message => + message.parts.map(part => `${message.role}:${part.kind}`) + ); + +it('runs the tool, answers the model with it, and asks again', async () => { + const { value, calls } = await runWith({ + options, + tools: registry(saying('weather', 'it rains')), + replies: [ + { deltas: [], calls: [call('tc_1', 'weather')], stop: 'tools' }, + { deltas: ['it rains outside'] }, + ], + use: session => + Effect.map(Stream.runCollect(session.ask('what is it like out')), chunk => [...chunk]), + }); + + expect(calls).toHaveLength(2); + /* The second request carries the call and the result, in that order and in + that shape. Anything else and the provider refuses the request outright. */ + expect(shapeOf(calls[1])).toEqual(['user:text', 'assistant:toolCall', 'user:toolResult']); + expect(resultsIn(value)).toEqual([{ callId: 'tc_1', body: 'it rains', failed: false }]); +}); + +it('writes the question, the call, the result and the answer as one exchange', async () => { + const store = recordingStore(); + await runWith({ + options, + store: store.layer, + tools: registry(saying('weather', 'it rains')), + replies: [ + { deltas: [], calls: [call('tc_1', 'weather')], stop: 'tools' }, + { deltas: ['it rains outside'] }, + ], + use: session => Stream.runDrain(session.ask('what is it like out')), + }); + + /* One append, holding every turn. A store that saw the call before the result + existed would hold a session that could never be asked anything again. */ + expect(store.seen).toEqual([ + 'user:what is it like out', + 'assistant:', + 'user:', + 'assistant:it rains outside', + 'flush', + ]); +}); + +it('hands the model a failed result rather than failing the question', async () => { + const broken = tool('weather', () => Effect.fail(new ToolFailure({ cause: 'no network' }))); + const { value } = await runWith({ + options, + tools: registry(broken), + replies: [ + { deltas: [], calls: [call('tc_1', 'weather')], stop: 'tools' }, + { deltas: ['I could not find out'] }, + ], + use: session => + Effect.map(Stream.runCollect(session.ask('what is it like out')), chunk => [...chunk]), + }); + + const [result] = resultsIn(value); + expect(result?.failed).toBe(true); + /* The model is told what went wrong, because it is the only party that can + decide whether to try again or say so. */ + expect(result?.body).toContain('no network'); +}); + +it('refuses a tool the session does not offer, and says so to the model', async () => { + const { value } = await runWith({ + options, + tools: registry(saying('weather', 'it rains')), + replies: [ + { deltas: [], calls: [call('tc_1', 'stocks')], stop: 'tools' }, + { deltas: ['I cannot do that'] }, + ], + use: session => + Effect.map(Stream.runCollect(session.ask('what is it like out')), chunk => [...chunk]), + }); + + const [result] = resultsIn(value); + expect(result).toMatchObject({ callId: 'tc_1', failed: true }); + expect(result?.body).toContain('no tool named stocks'); +}); + +it('runs the calls of one turn at once, and leaves a tool to serialise itself', async () => { + const order: string[] = []; + const marking = (name: string) => + Effect.sync(() => order.push(`${name} in`)).pipe( + Effect.zipRight(Effect.sleep(Duration.millis(20))), + Effect.zipRight(Effect.sync(() => order.push(`${name} out`))), + Effect.as('done') + ); + const slow = (name: string) => tool(name, () => marking(name)); + /* The session serialises nothing, so a tool that must not overlap says so in + its own body. This is the four lines a caller writes. */ + const serialised = (name: string) => { + const permit = Effect.unsafeMakeSemaphore(1); + return tool(name, () => permit.withPermits(1)(marking(name))); + }; + + await runWith({ + options: { ...options, tools: ['a', 'b', 'serial'] }, + tools: registry(slow('a'), slow('b'), serialised('serial')), + replies: [ + { + deltas: [], + calls: [ + call('tc_1', 'a'), + call('tc_2', 'b'), + call('tc_3', 'serial'), + call('tc_4', 'serial'), + ], + stop: 'tools', + }, + { deltas: ['done'] }, + ], + use: session => Stream.runDrain(session.ask('go')), + }); + + /* `a` and `b` overlap: both are in before either is out. */ + expect(order.slice(0, 2).toSorted()).toEqual(['a in', 'b in']); + /* The two calls to `serial` do not: its own permit holds the second back. */ + const serial = order.filter(step => step.startsWith('serial')); + expect(serial).toEqual(['serial in', 'serial out', 'serial in', 'serial out']); +}); + +it('stops offering tools at the round ceiling and asks for an answer in words', async () => { + const { calls } = await runWith({ + options: { ...options, maxRounds: 2 }, + tools: registry(saying('weather', 'it rains')), + /* A model that never stops asking. Without the ceiling this never returns. */ + replies: [{ deltas: [], calls: [call('tc_1', 'weather')], stop: 'tools' }], + use: session => Stream.runDrain(session.ask('what is it like out')), + }); + + /* Two rounds with the tools, then one without. The last one cannot ask for + anything, so the exchange ends on something the model said. */ + expect(calls).toHaveLength(3); + expect(calls.slice(0, 2).map(request => request.tools?.length)).toEqual([1, 1]); + expect(calls[2]?.tools).toBeUndefined(); +}); + +it('keeps every turn of the exchange in the history, in the order they happened', async () => { + const { value } = await runWith({ + options, + tools: registry(saying('weather', 'it rains')), + replies: [ + { deltas: ['let me look'], calls: [call('tc_1', 'weather')], stop: 'tools' }, + { deltas: ['it rains outside'] }, + ], + use: session => + Effect.zipRight(Stream.runDrain(session.ask('what is it like out')), session.history), + }); + + expect(texts(value)).toEqual([ + 'user:what is it like out', + 'assistant:let me look', + 'user:', + 'assistant:it rains outside', + ]); +}); + +it('offers no tools at all to a session that named none', async () => { + const { calls } = await runWith({ + replies: [{ deltas: ['hello'] }], + use: session => Stream.runDrain(session.ask('hi')), + }); + + expect(calls[0]?.tools).toBeUndefined(); +}); + +it('counts what every round cost, not only the last', async () => { + const { value } = await runWith({ + options, + tools: registry(saying('weather', 'it rains')), + replies: [ + { + deltas: [], + calls: [call('tc_1', 'weather')], + stop: 'tools', + usage: { inputTokens: 10, outputTokens: 3 }, + }, + { deltas: ['it rains'], usage: { inputTokens: 20, outputTokens: 5 } }, + ], + use: session => Effect.zipRight(Stream.runDrain(session.ask('out?')), session.usage), + }); + + /* A caller reading `usage` to know what a question cost must see every round + of it. One question is now several billed calls. */ + expect(value).toMatchObject({ inputTokens: 30, outputTokens: 8 }); +}); + +it('leaves nothing behind when the caller walks away mid-loop', async () => { + const store = recordingStore(); + const ran: string[] = []; + await runWith({ + options, + store: store.layer, + tools: registry( + tool('weather', () => + Effect.sync(() => { + ran.push('weather'); + return 'it rains'; + }) + ) + ), + replies: [ + { deltas: [], calls: [call('tc_1', 'weather')], stop: 'tools' }, + { deltas: ['a'], stall: true }, + ], + use: session => + Effect.ignore(Effect.timeout(Stream.runDrain(session.ask('out?')), Duration.millis(50))), + }); + + /* The tool ran, and nothing was written: the exchange never finished, so the + store holds no call and no half answer. */ + expect(ran).toEqual(['weather']); + expect(store.seen).toEqual(['flush']); +}); diff --git a/packages/harness-sdk/src/core/loop.ts b/packages/harness-sdk/src/core/loop.ts new file mode 100644 index 0000000000..146be15ae2 --- /dev/null +++ b/packages/harness-sdk/src/core/loop.ts @@ -0,0 +1,173 @@ +import { Effect, Ref, Stream } from 'effect'; +import type { AskOptions } from './ask.js'; +import { isFull } from './compact.js'; +import { + called, + collect, + commit, + endRound, + type Exchange, + hidden, + nextRound, + said, + thinking, +} from './exchange.js'; +import type { ModelError, ModelEvent, ModelRequest } from './model.js'; +import { sinceSummary } from './session.js'; +import type { StoreError } from './storage.js'; +import { definitionsOf } from './tool.js'; +import { callsIn, resultsTurn, runCalls } from './tools.js'; +import type { Wiring } from './wiring.js'; + +/** + * One question, and every round it takes to answer it. + * + * Without tools a question is one request and one reply. With them the model + * answers by asking for something, the tools answer, and the model is asked + * again, until it stops asking. All of it is one exchange: one entry in the + * store, one question in the transcript, one stream to the caller. + * + * The loop ends three ways. The model stops asking, which is the usual one. The + * round ceiling is reached. Or the last request filled enough of the window that + * the next one would be refused. The last two end the same way, with one more + * request that offers no tools at all, so the model has to answer in words: an + * exchange that stopped on a tool result would leave the transcript ending on + * something the model never replied to, which no shape will take back. + */ + +/** + * The last resort, for a model the catalog does not name a limit for. It is a + * floor, not an opinion: a caller that cares names a number. + */ +const defaultMaxTokens = 4096; + +/** + * How many times one question may go back to the model. It is a wall against a + * model that calls the same tool forever, and every round past it is real money. + */ +const defaultMaxRounds = 24; + +/** + * One question beats the session, and the session beats the catalog. The + * catalog is only asked when nobody named a number, so the usual path costs + * no lookup. A catalog that cannot answer is not an error here — the package + * falls back rather than refusing to ask the question. + */ +const ceilingOf = (wiring: Wiring, options: AskOptions | undefined): Effect.Effect => { + const named = options?.maxTokens ?? wiring.maxTokens; + return named === undefined + ? wiring.catalog.facts(wiring.model).pipe( + Effect.map(facts => facts.maxOutputTokens ?? defaultMaxTokens), + Effect.orElseSucceed(() => defaultMaxTokens) + ) + : Effect.succeed(named); +}; + +/** The stream one question answers with. */ +type Answer = Stream.Stream; + +/** What every round of one question shares. */ +interface Round { + readonly wiring: Wiring; + readonly exchange: Exchange; + readonly options: AskOptions | undefined; +} + +/** + * The request for one round. `offer` is false on the round that has to end the + * exchange: the model cannot ask for what it is not given. + */ +const requestFor = (round: Round, offer: boolean): Effect.Effect => + Effect.gen(function* () { + const { wiring } = round; + const { turns } = yield* Ref.get(wiring.state); + /* Everything from the last summary onward. Before the first compaction + that is every turn, and the call costs one scan of a list already held. */ + const asked = sinceSummary(turns); + const maxTokens = yield* ceilingOf(wiring, round.options); + const tools = offer ? definitionsOf(wiring.tools, wiring.inlineFor) : []; + return { + prompt: wiring.assembler.assemble({ system: wiring.system, turns: asked }), + model: wiring.model, + maxTokens, + ...(wiring.effort === undefined ? {} : { effort: wiring.effort }), + cacheKey: wiring.id, + ...(tools.length === 0 ? {} : { tools }), + }; + }); + +/** Everything one round hears, kept where it belongs. */ +const heard = (round: Round, event: ModelEvent): Effect.Effect => { + switch (event.kind) { + case 'delta': { + return said(round.exchange.spoken, event.text); + } + case 'reasoning': { + return thinking(round.exchange.spoken, event); + } + case 'redacted': { + return hidden(round.exchange.spoken, event.data); + } + case 'toolCall': { + return called(round.exchange.spoken, event.call); + } + case 'done': { + return Effect.asVoid(endRound(round.wiring, round.exchange, event)); + } + /* Made here, from a tool, and never by the model. It is on the stream for + the caller to show, and there is nothing to collect. */ + case 'toolResult': { + return Effect.void; + } + } +}; + +/** Whether the model may be asked once more with its tools in hand. */ +const mayContinue = (wiring: Wiring, rounds: number): Effect.Effect => + Effect.map(isFull(wiring), full => !full && rounds < (wiring.maxRounds ?? defaultMaxRounds)); + +const roundsFrom = (round: Round, offer: boolean): Answer => + Stream.unwrap( + Effect.map(Effect.zipRight(nextRound(round.exchange), requestFor(round, offer)), request => + round.wiring.client.stream(request).pipe( + Stream.tap(event => heard(round, event)), + Stream.concat(Stream.unwrap(afterRound(round, offer))) + ) + ) + ); + +/** + * Runs what the round asked for, and decides whether there is another. + * + * The results reach the caller as events of their own, so a harness can show + * what its tools did without reading the transcript back. + */ +const answering = (round: Round, calls: ReturnType): Effect.Effect => + Effect.gen(function* () { + const results = yield* runCalls(round.wiring, calls); + yield* collect(round.wiring, round.exchange, yield* resultsTurn(round.wiring, results)); + const rounds = yield* Ref.get(round.exchange.rounds); + const again = yield* mayContinue(round.wiring, rounds); + const events = results.map((result): ModelEvent => ({ kind: 'toolResult', result })); + return Stream.concat(Stream.fromIterable(events), roundsFrom(round, again)); + }); + +/** + * What happens when a round's stream ends: either the exchange is written, or + * the tools run and the model is asked again. + */ +const afterRound = (round: Round, offered: boolean): Effect.Effect => + Effect.gen(function* () { + const stop = yield* Ref.get(round.exchange.stop); + const written = yield* Ref.get(round.exchange.written); + const last = written.at(-1); + const calls = last === undefined ? [] : callsIn(last); + if (!offered || stop !== 'tools' || calls.length === 0) { + yield* commit(round.wiring, round.exchange); + return Stream.empty; + } + return yield* answering(round, calls); + }); + +export type { Answer, Round }; +export { defaultMaxRounds, defaultMaxTokens, roundsFrom }; diff --git a/packages/harness-sdk/src/core/model.ts b/packages/harness-sdk/src/core/model.ts new file mode 100644 index 0000000000..cd7b46fe2c --- /dev/null +++ b/packages/harness-sdk/src/core/model.ts @@ -0,0 +1,148 @@ +import { Context, Data, type Effect, Stream } from 'effect'; +import type { Prompt } from './prompt.js'; +import type { ToolCall, ToolDefinition, ToolResult } from './tool.js'; + +/** + * How hard the model should think. Both model SDKs spell the levels this way. + * + * This is not `maxTokens`. `maxTokens` is a wall the server enforces and the + * model cannot see; effort is a dial the model itself follows. A reasoning + * model spends its thinking out of `maxTokens`, so a low wall and a high effort + * together produce no answer at all. + */ +type Effort = 'low' | 'medium' | 'high' | 'xhigh' | 'max'; + +/** + * What the model was asked. `model` and `effort` are part of the cache key. + * `maxTokens` is not: it never reaches the rendered prefix, so it may vary from + * one call to the next at no cost. + */ +interface ModelRequest { + readonly prompt: Prompt; + readonly model: string; + readonly maxTokens: number; + readonly effort?: Effort; + /** Groups the requests of one session onto one cache entry. Use the session id. */ + readonly cacheKey?: string; + /** + * The tools the model may ask for. Every shape puts these in front of the + * messages, so they are part of the cached prefix and must not change within + * a session. A request that offers none omits the field rather than sending + * an empty list, which some shapes refuse. + */ + readonly tools?: readonly ToolDefinition[]; +} + +/** + * Token counts for one reply. `cacheReadTokens / (cacheReadTokens + inputTokens)` + * is the cache hit ratio, which must stay above 95 percent. + */ +interface ModelUsage { + readonly inputTokens: number; + readonly outputTokens: number; + readonly cacheReadTokens: number; + readonly cacheWriteTokens: number; +} + +/** + * Why the model stopped talking. + * + * `end` is a finished answer. `maxTokens` is a wall: the answer stops + * mid-sentence, and a caller that treats it as finished stores half a thought + * and builds every later request on it. `refusal` is the model declining. + * `tools` is the model waiting on a tool it asked for, which is the one reason + * that is not the end of anything: the loop answers the calls and asks again. + * `unknown` is a shape that reported nothing, which is not an error. + */ +type StopReason = 'end' | 'maxTokens' | 'refusal' | 'tools' | 'unknown'; + +/** + * A model call failed. `status` is the HTTP status when the transport has one. + * + * `stream` is the one that arrives after the answer started. Every shape may + * report a failure part way through a stream that it would have reported as a + * status had the call not been streamed, so the answer already in the caller's + * hands is a fragment. It is a reason of its own because the caller has + * something to throw away, which is true of no other one. + */ +class ModelError extends Data.TaggedError('harness/ModelError')<{ + readonly reason: 'transport' | 'status' | 'body' | 'unsupported' | 'stream'; + readonly status?: number; + readonly cause: unknown; +}> {} + +/** + * One piece of a streamed reply. The last event of a stream is always `done`. + * + * `reasoning` is the model thinking aloud. It is a separate kind because it is + * not the answer: a caller shows it apart from the answer, or not at all. + * + * The signature arrives on its own `reasoning` event, after the thinking and + * with no text, because that is how the provider streams it. A shape that + * issues no signature sends none, and that thinking cannot be replayed. + */ +type ModelEvent = + | { readonly kind: 'delta'; readonly text: string } + | { readonly kind: 'reasoning'; readonly text: string; readonly signature?: string } + /** Thinking the provider encrypted. There is nothing here to show a reader. */ + | { readonly kind: 'redacted'; readonly data: string } + /** + * A tool the model asked for, whole. It arrives as its block closes and not + * at the end of the stream, so the tool starts while the model is still + * writing the next call. + * + * The arguments stream in pieces on every shape. The transport collects them, + * because collecting them is reading the wire; nothing above it ever sees + * half a call. + */ + | { readonly kind: 'toolCall'; readonly call: ToolCall } + /** + * What a tool gave back. The session makes this one, not the transport: a + * harness shows what its tools did, and reading the transcript back after + * every round to find out is worse than being told. + */ + | { readonly kind: 'toolResult'; readonly result: ToolResult } + | { readonly kind: 'done'; readonly usage: ModelUsage; readonly stop: StopReason }; + +const zeroUsage: ModelUsage = { + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, +}; + +/** + * Sends an assembled prompt and streams the reply. Transport only: a plugin + * must not build or change the prompt, because a changed prefix drops the cache. + * + * Streaming is the only way in. Every caller inside the harness wants the reply + * as it arrives, and a second non-streaming path would be a second parsing of + * every shape, exercised by one caller and free to disagree with this one. + */ +interface ModelClientService { + readonly stream: (request: ModelRequest) => Stream.Stream; +} + +class ModelClient extends Context.Tag('harness/ModelClient')() {} + +/** + * What the model said, once it has said all of it. + * + * A stream of events is what a harness wants — it shows the words as they + * arrive, and it shows the tools it ran. Something that only wants the answer + * has to fold the stream, keeping the deltas and dropping everything else, and + * that fold was written out by hand in twelve of this package's own live runs + * before it was written here. + * + * It keeps the words and nothing else. Thinking is not the answer, a tool call + * is not the answer, and a round that called a tool contributes whatever the + * model said after it — which is why this is a fold over the whole stream and + * not a read of the last event. + */ +const said = (answer: Stream.Stream): Effect.Effect => + Stream.runFold(answer, '', (held: string, event) => + event.kind === 'delta' ? held + event.text : held + ); + +export type { Effort, ModelClientService, ModelEvent, ModelRequest, ModelUsage, StopReason }; +export { ModelClient, ModelError, said, zeroUsage }; diff --git a/packages/harness-sdk/src/core/on-demand.test.ts b/packages/harness-sdk/src/core/on-demand.test.ts new file mode 100644 index 0000000000..4ceaf79808 --- /dev/null +++ b/packages/harness-sdk/src/core/on-demand.test.ts @@ -0,0 +1,132 @@ +import { Deferred, Duration, Effect, Fiber, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import type { Continued } from './queue.js'; +import { runWith } from './session-fixture.js'; +import { type Tool, ToolRegistry } from './tool.js'; + +/** + * Sending a call to the background while the model is still waiting for it. + * + * The deadline is a guess made before the call started. Somebody watching it + * knows better: a person who has waited long enough, or an agent that decides + * the answer is not worth the open request. Both say the same thing to the + * session, and the session does not need to know which of them it was. + * + * Nothing is cancelled. The work carries on, and what it says arrives in a + * round of its own — the same path the deadline takes, brought forward. + */ + +const call = { id: 'tc_1', name: 'slow', arguments: '{}' }; + +/* Long enough that nothing here reaches it. Every backgrounding in this file is + somebody deciding, never the clock. */ +const options = { + system: 'sys', + model: 'claude-opus-5', + maxTokens: 1024, + tools: ['slow'], + inlineFor: Duration.minutes(5), +}; + +const tool = (run: Tool['run']): Layer.Layer => + Layer.succeed(ToolRegistry, { + tools: [ + { + definition: { + name: 'slow', + description: 'slow', + parameters: { type: 'object', properties: {} }, + }, + run, + }, + ], + }); + +const resultsIn = (events: readonly { readonly kind: string }[]) => + events.filter(event => event.kind === 'toolResult'); + +const said = (seen: readonly Continued[]): string => + seen + .map(one => { + if ('failed' in one) { + return ''; + } + return one.event.kind === 'delta' ? one.event.text : ''; + }) + .join(''); + +it('moves the model on when the caller sends a running call away', async () => { + const started = Effect.runSync(Deferred.make()); + const answer = Effect.runSync(Deferred.make()); + const { value } = await runWith({ + options, + tools: tool(() => Effect.zipRight(Deferred.succeed(started, true), Deferred.await(answer))), + replies: [ + { deltas: [], calls: [call], stop: 'tools' }, + { deltas: ['I will carry on'] }, + { deltas: ['it says nine'] }, + ], + use: session => + Effect.gen(function* () { + const watching = yield* Effect.fork(Stream.runCollect(Stream.take(session.continued, 2))); + const asking = yield* Effect.fork(Stream.runCollect(session.ask('start the build'))); + /* Wait for the call to actually start, then send it away. Nothing about + this is the clock: the deadline here is five minutes. */ + yield* Deferred.await(started); + const waiting = yield* session.running; + const sent = yield* session.background(call.id); + const events = [...(yield* Fiber.join(asking))]; + /* The answer lands only after the model has been moved on, which is the + whole shape of a call that outlives its request. */ + yield* Deferred.succeed(answer, 'nine'); + return { waiting, sent, events, seen: [...(yield* Fiber.join(watching))] }; + }), + }); + + expect(value.sent).toBe(true); + expect(value.waiting).toMatchObject([{ id: 'tc_1', name: 'slow' }]); + const [result] = resultsIn(value.events); + expect(result).toMatchObject({ result: { callId: 'tc_1', failed: false } }); + expect(said(value.seen)).toBe('it says nine'); +}); + +it('says no when there was nothing left to send away', async () => { + const { value } = await runWith({ + options, + tools: tool(() => Effect.succeed('exit 0')), + replies: [{ deltas: [], calls: [call], stop: 'tools' }, { deltas: ['the build passed'] }], + use: session => + Effect.gen(function* () { + /* Nothing is running yet. */ + const before = yield* session.background(call.id); + yield* Stream.runDrain(session.ask('start the build')); + /* And by now the call has answered, so it cannot be sent anywhere. */ + return { before, after: yield* session.background(call.id), left: yield* session.running }; + }), + }); + + expect(value).toEqual({ before: false, after: false, left: [] }); +}); + +it('sends a call away once, however many times it is asked to', async () => { + const started = Effect.runSync(Deferred.make()); + const { value, calls } = await runWith({ + options, + tools: tool(() => Effect.zipRight(Deferred.succeed(started, true), Effect.never)), + replies: [{ deltas: [], calls: [call], stop: 'tools' }, { deltas: ['I will carry on'] }], + use: session => + Effect.gen(function* () { + const asking = yield* Effect.fork(Stream.runDrain(session.ask('start the build'))); + yield* Deferred.await(started); + const first = yield* session.background(call.id); + const again = yield* session.background(call.id); + yield* Fiber.join(asking); + return { first, again }; + }), + }); + + /* The second press is a person racing their own hand. It answers false and + changes nothing: two rounds, not three. */ + expect(value).toEqual({ first: true, again: false }); + expect(calls).toHaveLength(2); +}); diff --git a/packages/harness-sdk/src/core/prefix.test.ts b/packages/harness-sdk/src/core/prefix.test.ts new file mode 100644 index 0000000000..a15b74619a --- /dev/null +++ b/packages/harness-sdk/src/core/prefix.test.ts @@ -0,0 +1,87 @@ +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { seededEntropy } from '../plugins/entropy/seeded.js'; +import { assemble } from '../plugins/prompt/default.js'; +import type { Prompt } from './prompt.js'; +import { makeSession } from './session.js'; +import { makeTurn } from './turn.js'; + +const entropy = seededEntropy(3); +const system = 'You are a harness.'; + +const turnsOf = (sessionId: string, count: number) => + Array.from({ length: count }, (_, index) => + Effect.runSync( + makeTurn(entropy, { + sessionId, + role: index % 2 === 0 ? 'user' : 'assistant', + parts: [{ kind: 'text', body: `message number ${String(index)}` }], + }) + ) + ); + +/** + * The cache invariant, stated as one property: as a session grows, everything + * the model has already seen must render to the same bytes it rendered last + * time. A prompt that reorders, rewrites or re-marks an earlier turn drops the + * whole prefix, and the only symptom in production is a bill. + * + * This is the one performance requirement the package fully controls, so it is + * asserted as behavior rather than measured as a duration. + */ +const grow = (count: number): readonly Prompt[] => { + const session = Effect.runSync(makeSession(entropy)); + const every = turnsOf(session.id, count); + return Array.from({ length: count }, (_, index) => + assemble({ system, turns: every.slice(0, index + 1) }) + ); +}; + +it('never changes a byte of what the model has already seen', () => { + const prompts = grow(50); + + for (let index = 1; index < prompts.length; index += 1) { + const earlier = prompts[index - 1]; + const later = prompts[index]; + if (earlier === undefined || later === undefined) { + throw new Error('the growth series is incomplete'); + } + + /* Every message the earlier prompt did not end on must survive untouched, + cache marks included. The last one is allowed to move, because the + breakpoint moves with it. */ + const settled = earlier.messages.length - 1; + expect(JSON.stringify(later.messages.slice(0, settled))).toBe( + JSON.stringify(earlier.messages.slice(0, settled)) + ); + expect(JSON.stringify(later.system)).toBe(JSON.stringify(earlier.system)); + } +}); + +it('marks exactly one breakpoint in the system and one on the last message', () => { + const session = Effect.runSync(makeSession(entropy)); + const prompt = assemble({ system, turns: turnsOf(session.id, 8) }); + + expect(prompt.system.filter(block => block.cache)).toHaveLength(1); + expect(prompt.messages.map(message => message.cache)).toEqual([ + false, + false, + false, + false, + false, + false, + false, + true, + ]); +}); + +it('moves the breakpoint forward by exactly one message per turn', () => { + const session = Effect.runSync(makeSession(entropy)); + const marks = [4, 5, 6].map(count => + assemble({ system, turns: turnsOf(session.id, count) }).messages.findIndex( + message => message.cache + ) + ); + + expect(marks).toEqual([3, 4, 5]); +}); diff --git a/packages/harness-sdk/src/core/prompt.test.ts b/packages/harness-sdk/src/core/prompt.test.ts new file mode 100644 index 0000000000..63e03c9bdf --- /dev/null +++ b/packages/harness-sdk/src/core/prompt.test.ts @@ -0,0 +1,61 @@ +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { assemble } from '../plugins/prompt/default.js'; +import { seededEntropy } from '../plugins/entropy/seeded.js'; +import { makeTurn } from './turn.js'; +import { textIn } from './prompt.js'; + +const entropy = seededEntropy(1); + +const system = 'You are a harness.'; + +const run = (effect: Effect.Effect): A => Effect.runSync(effect); + +const turns = (...contents: readonly string[]) => + run( + Effect.all( + contents.map(content => + makeTurn(entropy, { + sessionId: 'ses_1', + role: 'user', + parts: [{ kind: 'text', body: content }], + }) + ) + ) + ); + +it('breaks the cache after the system prompt and on the last turn', () => { + const prompt = assemble({ system, turns: turns('a', 'b') }); + expect(prompt.system).toEqual([{ text: system, cache: true }]); + expect(prompt.messages.map(message => message.cache)).toEqual([false, true]); +}); + +it('gives the same bytes for the same input', () => { + const input = { system, turns: turns('a', 'b') }; + expect(JSON.stringify(assemble(input))).toBe(JSON.stringify(assemble(input))); +}); + +it('leaves every earlier message unchanged when a turn is appended', () => { + const before = turns('a', 'b'); + const [added] = run( + Effect.all([ + makeTurn(entropy, { + sessionId: 'ses_1', + role: 'assistant', + parts: [{ kind: 'text', body: 'c' }], + }), + ]) + ); + const after = [...before, added]; + + const grown = assemble({ system, turns: after }); + expect(grown.messages.slice(0, before.length).map(textIn)).toEqual( + assemble({ system, turns: before }).messages.map(textIn) + ); + expect(grown.system).toEqual(assemble({ system, turns: before }).system); +}); + +it('still breaks the cache after the system prompt when the session has no turns', () => { + const prompt = assemble({ system, turns: [] }); + expect(prompt.system).toEqual([{ text: system, cache: true }]); +}); diff --git a/packages/harness-sdk/src/core/prompt.ts b/packages/harness-sdk/src/core/prompt.ts new file mode 100644 index 0000000000..f46486bb5f --- /dev/null +++ b/packages/harness-sdk/src/core/prompt.ts @@ -0,0 +1,89 @@ +import { Context } from 'effect'; +import type { Turn, TurnRole } from './turn.js'; + +/** + * A block of the system prompt. `cache` marks a cache breakpoint: the model + * caches every byte up to and including this block. + */ +interface PromptBlock { + readonly text: string; + readonly cache: boolean; +} + +/** + * One piece of a message, as the transport plugin will render it. + * + * Reasoning is here, and it goes back to the provider unchanged. Stripping it + * saves nothing — the API drops what the model cannot read, unbilled — and + * removing a block can fail the request on ordering or on the signature. A + * reasoning part with no signature cannot be replayed at all, so the shape + * leaves that one out. + */ +type PromptPart = + | { readonly kind: 'text'; readonly text: string } + | { readonly kind: 'reasoning'; readonly text: string; readonly signature?: string } + | { readonly kind: 'redacted'; readonly data: string } + | { readonly kind: 'image'; readonly media: string; readonly data: string } + /** A tool the model asked for, on its way back into the transcript. */ + | { + readonly kind: 'toolCall'; + readonly callId: string; + readonly name: string; + readonly arguments: string; + } + /** What the tool gave back. It answers the call of the same identifier. */ + | { + readonly kind: 'toolResult'; + readonly callId: string; + readonly body: string; + readonly failed: boolean; + }; + +/** `cache` marks the breakpoint, which belongs to the message, not to a part. */ +interface PromptMessage { + readonly role: TurnRole; + readonly parts: readonly PromptPart[]; + readonly cache: boolean; +} + +/** What the transport plugin sends. The order on the wire is system, then messages. */ +interface Prompt { + readonly system: readonly PromptBlock[]; + readonly messages: readonly PromptMessage[]; +} + +interface PromptInput { + readonly system: string; + readonly turns: readonly Turn[]; +} + +/** + * Turns a session into a prompt. This is where the model cache is won or lost, + * so an assembler must hold two invariants: + * + * 1. The same input gives the same bytes. No clock, no random value, no key + * order that varies. + * 2. Appending a turn changes nothing it said before that turn. `cache` is the + * exception and not content: the breakpoint marks the last message, so it + * moves with every turn while everything sent before it stays as it was. + * + * `checkAssembler` runs both against an assembler. + */ +interface PromptAssemblerService { + readonly assemble: (input: PromptInput) => Prompt; +} + +/** The text of a message, which is all of it for a message that carries no image. */ +const textIn = (message: PromptMessage): string => + message.parts + .filter(part => part.kind === 'text') + .map(part => part.text) + .join(''); + +class PromptAssembler extends Context.Tag('harness/PromptAssembler')< + PromptAssembler, + PromptAssemblerService +>() {} + +export type { Prompt, PromptAssemblerService, PromptBlock, PromptInput, PromptMessage, PromptPart }; +export { PromptAssembler, textIn }; diff --git a/packages/harness-sdk/src/core/queue.test.ts b/packages/harness-sdk/src/core/queue.test.ts new file mode 100644 index 0000000000..9275d1d35e --- /dev/null +++ b/packages/harness-sdk/src/core/queue.test.ts @@ -0,0 +1,285 @@ +import { Deferred, Duration, Effect, Fiber, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { ModelError } from './model.js'; +import type { Continued } from './queue.js'; +import type { SessionHandle } from './handle.js'; +import { recordingStore, runWith } from './session-fixture.js'; +import { type Tool, ToolRegistry } from './tool.js'; + +/** + * A message a caller sends while the session is busy, and taking one back. + * + * One session does one thing at a time, so a person typing while the last + * answer is still arriving cannot be answered where they stand. They join a + * line instead, the line is answered in the order it formed, and a message + * still in the line can be taken out of it. That last part is what makes the + * first part usable: a queue nobody can cancel is a queue that sends the thing + * you changed your mind about. + */ + +const options = { system: 'sys', model: 'claude-opus-5', maxTokens: 1024 }; + +/** The event of one thing that happened, or nothing when the round failed. */ +const eventIn = (one: Continued) => ('failed' in one ? undefined : one.event); + +const said = (seen: readonly Continued[]): string => + seen + .map(one => { + const event = eventIn(one); + return event?.kind === 'delta' ? event.text : ''; + }) + .join(''); + +/** + * Everything each round of `continued` said, and what it was answering. An + * identifier is answered by exactly one round, so grouping by it is grouping by + * round. + */ +const rounds = (seen: readonly Continued[]) => { + const held = new Map(); + for (const one of seen) { + const key = one.answering.join(','); + held.set(key, { answering: one.answering, said: (held.get(key)?.said ?? '') + said([one]) }); + } + return [...held.values()]; +}; + +/** Watches `continued` until it has seen `count` rounds end. */ +const watching = (session: SessionHandle, count: number) => { + let ended = 0; + return Effect.fork( + Stream.runCollect( + Stream.takeUntil(session.continued, one => { + ended += eventIn(one)?.kind === 'done' ? 1 : 0; + return ended === count; + }) + ) + ); +}; + +it('asks a queued message and marks the answer with the identifier it gave back', async () => { + const { value, calls } = await runWith({ + options, + replies: [{ deltas: ['queued answer'] }], + use: session => + Effect.gen(function* () { + const seen = yield* watching(session, 1); + const id = yield* session.queue('what is up'); + return { id, seen: [...(yield* Fiber.join(seen))] }; + }), + }); + + expect(calls).toHaveLength(1); + expect(rounds(value.seen)).toEqual([{ answering: [value.id], said: 'queued answer' }]); +}); + +it('answers the line in the order it formed, one turn per message', async () => { + const store = recordingStore(); + const { value } = await runWith({ + options, + store: store.layer, + replies: [{ deltas: ['first'] }, { deltas: ['second'] }], + use: session => + Effect.gen(function* () { + const seen = yield* watching(session, 2); + const one = yield* session.queue('one'); + const two = yield* session.queue('two'); + return { one, two, seen: [...(yield* Fiber.join(seen))] }; + }), + }); + + expect(rounds(value.seen)).toEqual([ + { answering: [value.one], said: 'first' }, + { answering: [value.two], said: 'second' }, + ]); + /* Two messages a caller wrote are two turns. Running them together would put + words the caller wrote apart into one thing the conversation said. */ + expect(store.seen).toEqual([ + 'user:one', + 'assistant:first', + 'user:two', + 'assistant:second', + 'flush', + ]); +}); + +it('never asks a message that was cancelled while it waited', async () => { + const store = recordingStore(); + const { value, calls } = await runWith({ + options, + store: store.layer, + replies: [{ deltas: ['only the second'] }], + use: session => + Effect.gen(function* () { + const seen = yield* watching(session, 1); + const dropped = yield* session.queue('never mind'); + const kept = yield* session.queue('this one'); + /* Both are still in the line: nothing can have been asked yet, because + the driver holds the session before it takes anything out. */ + const took = yield* session.cancel(dropped); + return { took, kept, seen: [...(yield* Fiber.join(seen))] }; + }), + }); + + expect(value.took).toBe(true); + expect(calls).toHaveLength(1); + expect(rounds(value.seen)).toEqual([{ answering: [value.kept], said: 'only the second' }]); + expect(store.seen).toEqual(['user:this one', 'assistant:only the second', 'flush']); +}); + +it('says a message was not taken back when it had already been asked', async () => { + const { value } = await runWith({ + options, + replies: [{ deltas: ['done'] }], + use: session => + Effect.gen(function* () { + const seen = yield* watching(session, 1); + const id = yield* session.queue('go'); + yield* Fiber.join(seen); + return { late: yield* session.cancel(id), missing: yield* session.cancel('que_nothing') }; + }), + }); + + /* Both false, and neither is a failure: a caller racing their own cancel + button against the session is the ordinary case, and a message the provider + has seen cannot be taken back. */ + expect(value).toEqual({ late: false, missing: false }); +}); + +const gate = Effect.runSync(Deferred.make()); +/* Says when the tool is provably running, so the test queues while the session + is busy rather than while a forked fiber is still scheduled. */ +const started = Effect.runSync(Deferred.make()); + +/** A tool that holds the session open until the test lets it go. */ +const held: Tool = { + definition: { name: 'held', description: 'held', parameters: { type: 'object', properties: {} } }, + run: () => + Effect.zipRight(Deferred.succeed(started, true), Effect.as(Deferred.await(gate), 'let go')), +}; + +it('shows what is waiting, in the order it will be asked', async () => { + const { value } = await runWith({ + options: { ...options, tools: ['held'], inlineFor: Duration.minutes(1) }, + tools: Layer.succeed(ToolRegistry, { tools: [held] }), + replies: [ + { deltas: [], calls: [{ id: 'tc_1', name: 'held', arguments: '{}' }], stop: 'tools' }, + { deltas: ['after the tool'] }, + { deltas: ['first queued'] }, + { deltas: ['second queued'] }, + ], + use: session => + Effect.gen(function* () { + const seen = yield* watching(session, 2); + /* The session is busy from here: the tool is waiting on the gate and the + model is waiting on the tool. */ + const asking = yield* Effect.fork(Stream.runDrain(session.ask('use the tool'))); + yield* Deferred.await(started); + const one = yield* session.queue('one'); + const two = yield* session.queue('two'); + const waiting = yield* session.queued; + yield* Deferred.succeed(gate, true); + yield* Fiber.join(asking); + return { one, two, waiting, seen: [...(yield* Fiber.join(seen))] }; + }), + }); + + /* Both are visible while the session is busy, which is the whole point: a + caller can show what will be sent, and take one of them back. */ + expect(value.waiting.map(one => ({ id: one.id, kind: one.kind }))).toEqual([ + { id: value.one, kind: 'message' }, + { id: value.two, kind: 'message' }, + ]); + expect(rounds(value.seen)).toEqual([ + { answering: [value.one], said: 'first queued' }, + { answering: [value.two], said: 'second queued' }, + ]); +}); + +it('answers the tool results waiting at the front of the line together', async () => { + const slow: Tool = { + definition: { + name: 'slow', + description: 'slow', + parameters: { type: 'object', properties: {} }, + }, + run: call => Effect.succeed(`${call.name} ${call.id} is done`), + }; + + const { value, calls } = await runWith({ + options: { ...options, tools: ['slow'], inlineFor: Duration.zero }, + tools: Layer.succeed(ToolRegistry, { tools: [slow] }), + replies: [ + { + deltas: [], + calls: [ + { id: 'tc_1', name: 'slow', arguments: '{}' }, + { id: 'tc_2', name: 'slow', arguments: '{}' }, + ], + stop: 'tools', + }, + { deltas: ['I will wait'] }, + { deltas: ['both are done'] }, + ], + use: session => + Effect.gen(function* () { + const seen = yield* watching(session, 1); + yield* Stream.runDrain(session.ask('run both')); + return [...(yield* Fiber.join(seen))]; + }), + }); + + /* One round, not two. The model asked for both calls in one turn and is + waiting on both, so telling it about them one request at a time would cost + a call and tell it less each time. */ + const [round] = rounds(value); + expect(round?.said).toBe('both are done'); + expect(round?.answering).toHaveLength(2); + expect(calls).toHaveLength(3); + const last = calls[2]?.prompt.messages.at(-1)?.parts ?? []; + const words = last.map(part => (part.kind === 'text' ? part.text : '')).join(''); + expect(words).toContain('tc_1'); + expect(words).toContain('tc_2'); +}); + +it('takes the ceiling a queued message named, and not the session default', async () => { + const { calls } = await runWith({ + options, + replies: [{ deltas: ['ok'] }], + use: session => + Effect.gen(function* () { + const seen = yield* watching(session, 1); + yield* session.queue('be brief', { maxTokens: 7 }); + yield* Fiber.join(seen); + }), + }); + + expect(calls[0]?.maxTokens).toBe(7); +}); + +it('tells the caller a round failed, and goes on running the rest of the line', async () => { + const { value, calls } = await runWith({ + options, + replies: [ + /* The first queued message is refused outright. The second must not be. */ + { deltas: [], fail: new ModelError({ reason: 'transport', cause: 'no route' }) }, + { deltas: ['the second one'] }, + ], + use: session => + Effect.gen(function* () { + const seen = yield* watching(session, 1); + const refused = yield* session.queue('one'); + const answered = yield* session.queue('two'); + return { refused, answered, seen: [...(yield* Fiber.join(seen))] }; + }), + }); + + /* Both were asked. The first failed at the model, which is one round's bad + news and not the end of the feed. */ + expect(calls).toHaveLength(2); + const failures = value.seen.filter(one => 'failed' in one); + expect(failures).toMatchObject([{ answering: [value.refused] }]); + expect(rounds(value.seen.filter(one => !('failed' in one)))).toEqual([ + { answering: [value.answered], said: 'the second one' }, + ]); +}); diff --git a/packages/harness-sdk/src/core/queue.ts b/packages/harness-sdk/src/core/queue.ts new file mode 100644 index 0000000000..f1ff082497 --- /dev/null +++ b/packages/harness-sdk/src/core/queue.ts @@ -0,0 +1,156 @@ +import { Effect, Queue, Ref } from 'effect'; +import type { AskOptions } from './ask.js'; +import type { ContinuedError } from './wiring.js'; +import type { EntropySourceService } from './entropy.js'; +import { makeId } from './id.js'; +import type { ModelEvent } from './model.js'; +import { type PartDraft, partsOf } from './turn.js'; + +/** + * What the session has been given to say and has not said yet. + * + * One session does one thing at a time, so anything that arrives while an + * answer is streaming has to wait somewhere. Two things arrive that way: a + * message a caller sent while the session was busy, and the result of a tool + * the model stopped waiting for. Both are the same shape — words to put in + * front of the model, in a turn of their own — so both wait in one line, and + * the line is answered in the order it formed. + * + * A caller may take a message back out again while it is still waiting. That is + * the whole of cancelling: a message that has not been sent costs nothing to + * drop, and one that has been sent cannot be taken back from the provider. + */ + +const idPrefix = 'que'; + +/** What one entry in the line is. `kind` says who put it there. */ +interface Waiting { + readonly id: string; + /** + * `message` is a caller's own, and only a caller's own is worth cancelling. + * `toolResult` is an answer the model is waiting for: cancelling one leaves + * the model believing a call is still running that nobody will ever report. + */ + readonly kind: 'message' | 'toolResult'; + /** What the model will read, as the turn it will read it in. */ + readonly parts: readonly PartDraft[]; + /** What the caller asked for this message alone. A tool result names none. */ + readonly options?: AskOptions; +} + +/** What every piece of a round names: the entries it is answering. */ +interface Answering { + /** + * The queued entries this round is answering, in the order they joined the + * line. One for a message; one or more for tool results that ran together. + * It is how a caller tells one queued message's answer from another's. + */ + readonly answering: readonly string[]; +} + +/** + * One thing that happened in a round the caller did not ask for. + * + * A round either says something or fails, so this is a union rather than a + * stream that fails. A failed round is one message's bad news and not the end + * of the feed: the session goes on running rounds for everything else in the + * line, and a caller who lost the stream to the first refused round would never + * hear about any of them. Narrow with `'failed' in one`. + */ +type Continued = + | (Answering & { readonly event: ModelEvent }) + | (Answering & { readonly failed: ContinuedError }); + +/** The line, the bell that tells the driver something joined it, and the names. */ +interface Pending { + readonly waiting: Ref.Ref; + /** + * One token per entry added, carrying its identifier. The driver waits on + * this rather than polling. A token whose entry was cancelled before the + * driver reached it finds an empty run and starts no round. + */ + readonly arrived: Queue.Queue; + /** Where an entry's identifier comes from. The line makes its own names. */ + readonly entropy: EntropySourceService; +} + +const makePending = (entropy: EntropySourceService): Effect.Effect => + Effect.all({ + waiting: Ref.make([]), + arrived: Queue.unbounded(), + entropy: Effect.succeed(entropy), + }); + +/** Joins the line, at the back. The identifier is what cancels it. */ +const enqueue = (pending: Pending, entry: Omit): Effect.Effect => + Effect.flatMap(makeId(pending.entropy, idPrefix), id => + Ref.update(pending.waiting, held => [...held, { ...entry, id }]).pipe( + Effect.zipRight(Queue.offer(pending.arrived, id)), + Effect.as(id) + ) + ); + +/** A caller's message, as a caller writes one. */ +const enqueueMessage = ( + pending: Pending, + input: string | readonly PartDraft[], + options: AskOptions | undefined +): Effect.Effect => + enqueue(pending, { + kind: 'message', + parts: partsOf(input), + ...(options === undefined ? {} : { options }), + }); + +/** + * Takes a message back out of the line. + * + * True when it was still there. False when it was not: it has already been + * asked, or it was never here, and neither is an error — a caller racing their + * own cancel button against the session losing that race is the ordinary case, + * not a failure. + */ +const cancelQueued = (pending: Pending, id: string): Effect.Effect => + Ref.modify(pending.waiting, held => { + const left = held.filter(one => one.id !== id); + return [left.length !== held.length, left]; + }); + +/** How many tool results are waiting at the front, before anything else. */ +const resultsAtFront = (held: readonly Waiting[]): number => { + const ends = held.findIndex(one => one.kind !== 'toolResult'); + return ends === -1 ? held.length : ends; +}; + +/** + * The next round's worth of the line. + * + * A message is a round of its own: a caller who wrote two of them meant two + * turns. Tool results run together, as many as are waiting at the front, + * because the model asked for those calls in one turn and is waiting on all of + * them — answering them one round at a time would put the model through a + * request per result and tell it less each time. + */ +const takeRun = (pending: Pending): Effect.Effect => + Ref.modify(pending.waiting, held => { + const taken = held[0]?.kind === 'message' ? 1 : resultsAtFront(held); + return [held.slice(0, taken), held.slice(taken)]; + }); + +/** + * Rings the bell again for a line that still holds something. + * + * A round the driver gave up on took nothing out of the line, but the token + * that pointed at it is spent. Without this the entries wait for whatever joins + * next, which for the last message a caller sends is forever. + */ +const wake = (pending: Pending): Effect.Effect => + Effect.flatMap(Ref.get(pending.waiting), held => { + const [first] = held; + return first === undefined + ? Effect.void + : Effect.asVoid(Queue.offer(pending.arrived, first.id)); + }); + +export type { Answering, Continued, Pending, Waiting }; +export { cancelQueued, enqueue, enqueueMessage, makePending, takeRun, wake }; diff --git a/packages/harness-sdk/src/core/reasoning.test.ts b/packages/harness-sdk/src/core/reasoning.test.ts new file mode 100644 index 0000000000..8f1af88557 --- /dev/null +++ b/packages/harness-sdk/src/core/reasoning.test.ts @@ -0,0 +1,181 @@ +import { Chunk, Effect, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { assemble } from '../plugins/prompt/default.js'; +import { run } from './session-fixture.js'; + +/** + * What a session does with the model's thinking: keeps it, seals it, and hands + * it back on the next question. How each shape renders it is + * `plugins/gateway/wire/replay.test.ts`. + */ + +const thinking = { + deltas: ['the answer'], + reasoning: ['first', ' second'], + signature: 'sig_abc', +}; + +it('keeps the reasoning and its signature, ahead of what the model then said', async () => { + const { value } = await run([thinking], session => + Effect.zipRight(Stream.runDrain(session.ask('why')), session.history) + ); + + const [, answer] = value; + expect(answer?.parts).toMatchObject([ + { kind: 'reasoning', body: 'first second', signature: 'sig_abc' }, + { kind: 'text', body: 'the answer' }, + ]); +}); + +it('adds no reasoning part when the model did none', async () => { + const { value } = await run([{ deltas: ['plain'] }], session => + Effect.zipRight(Stream.runDrain(session.ask('why')), session.history) + ); + + expect(value[1]?.parts).toMatchObject([{ kind: 'text', body: 'plain' }]); +}); + +it('keeps a thinking block that carries a signature and no words', async () => { + /* A provider returns the thinking as a summary, and defaults to no summary + at all. The block is then empty, is still billed, and still has to go back + exactly as it came. Dropping it here would drop every block on that + default. */ + const { value } = await run([{ deltas: ['said'], signature: 'sig_empty' }], session => + Effect.zipRight(Stream.runDrain(session.ask('why')), session.history) + ); + + expect(value[1]?.parts).toMatchObject([ + { kind: 'reasoning', body: '', signature: 'sig_empty' }, + { kind: 'text', body: 'said' }, + ]); +}); + +it('does not open a second block on the empty event that ends the first', async () => { + /* The gateway ends a thinking block with a reasoning event carrying no words + and no signature, after the signature has already arrived. Opening a block + on it leaves an unsigned block behind the signed one, and the wire drops + what it cannot sign: the thinking would go back with a hole in it, and only + a live run would ever say so. */ + const { value } = await run( + [ + { + deltas: ['the answer'], + events: [ + { kind: 'reasoning', text: 'first' }, + { kind: 'reasoning', text: '', signature: 'sig_abc' }, + { kind: 'reasoning', text: '' }, + { kind: 'delta', text: 'the answer' }, + ], + }, + ], + session => Effect.zipRight(Stream.runDrain(session.ask('why')), session.history) + ); + + expect(value[1]?.parts).toMatchObject([ + { kind: 'reasoning', body: 'first', signature: 'sig_abc' }, + { kind: 'text', body: 'the answer' }, + ]); +}); + +it('sends the thinking back on the next question, unchanged', async () => { + const { calls } = await run([thinking], session => + Effect.zipRight(Stream.runDrain(session.ask('why')), Stream.runDrain(session.ask('and then'))) + ); + + const answer = calls[1]?.prompt.messages[1]; + expect(answer?.parts).toEqual([ + { kind: 'reasoning', text: 'first second', signature: 'sig_abc' }, + { kind: 'text', text: 'the answer' }, + ]); +}); + +it('streams the reasoning to the caller, marked as reasoning', async () => { + const { value } = await run([thinking], session => Stream.runCollect(session.ask('why'))); + + expect( + Chunk.toReadonlyArray(value) + .filter(event => event.kind === 'reasoning') + .map(event => event.text) + ).toEqual(['first', ' second', '']); +}); + +it('carries the signature through the store and back into the prompt', async () => { + const { value } = await run([thinking], session => + Effect.zipRight(Stream.runDrain(session.ask('why')), session.history) + ); + + const prompt = assemble({ system: 'sys', turns: value }); + expect(prompt.messages[1]?.parts[0]).toEqual({ + kind: 'reasoning', + text: 'first second', + signature: 'sig_abc', + }); +}); + +it('keeps thinking the provider encrypted, and hands it back byte for byte', async () => { + /* A redacted block is thinking the provider would not show. It carries no + signature and no words, and dropping it breaks the chain exactly as + dropping a signed block does. */ + const { value, calls } = await run( + [{ deltas: ['said'], redacted: ['ENCRYPTED_ONE', 'ENCRYPTED_TWO'] }, { deltas: ['after'] }], + session => + Effect.zipRight( + Effect.zipRight(Stream.runDrain(session.ask('why')), Stream.runDrain(session.ask('and'))), + session.history + ) + ); + + expect(value[1]?.parts).toMatchObject([ + { kind: 'redacted', body: 'ENCRYPTED_ONE' }, + { kind: 'redacted', body: 'ENCRYPTED_TWO' }, + { kind: 'text', body: 'said' }, + ]); + expect(calls[1]?.prompt.messages[1]?.parts).toEqual([ + { kind: 'redacted', data: 'ENCRYPTED_ONE' }, + { kind: 'redacted', data: 'ENCRYPTED_TWO' }, + { kind: 'text', text: 'said' }, + ]); +}); + +it('keeps an encrypted block where it arrived, between the thinking around it', async () => { + /* The provider refuses a turn whose thinking blocks do not come back in the + order it produced them, and an encrypted block is one of those blocks. A + model that has part of its reasoning redacted returns thinking, then the + encrypted block, then more thinking. Collecting the words in one field and + the encrypted blocks in another loses which came first, and the next + request is rejected for rearranging what the model said. */ + const { value, calls } = await run( + [ + { + deltas: [], + events: [ + { kind: 'reasoning', text: 'before' }, + { kind: 'reasoning', text: '', signature: 'sig_one' }, + { kind: 'redacted', data: 'ENCRYPTED' }, + { kind: 'reasoning', text: 'after' }, + { kind: 'reasoning', text: '', signature: 'sig_two' }, + { kind: 'delta', text: 'said' }, + ], + }, + { deltas: ['next'] }, + ], + session => + Effect.zipRight( + Effect.zipRight(Stream.runDrain(session.ask('why')), Stream.runDrain(session.ask('and'))), + session.history + ) + ); + + expect(value[1]?.parts).toMatchObject([ + { kind: 'reasoning', body: 'before', signature: 'sig_one' }, + { kind: 'redacted', body: 'ENCRYPTED' }, + { kind: 'reasoning', body: 'after', signature: 'sig_two' }, + { kind: 'text', body: 'said' }, + ]); + expect(calls[1]?.prompt.messages[1]?.parts).toEqual([ + { kind: 'reasoning', text: 'before', signature: 'sig_one' }, + { kind: 'redacted', data: 'ENCRYPTED' }, + { kind: 'reasoning', text: 'after', signature: 'sig_two' }, + { kind: 'text', text: 'said' }, + ]); +}); diff --git a/packages/harness-sdk/src/core/resume-fixture.ts b/packages/harness-sdk/src/core/resume-fixture.ts new file mode 100644 index 0000000000..eaf66c0d30 --- /dev/null +++ b/packages/harness-sdk/src/core/resume-fixture.ts @@ -0,0 +1,74 @@ +import { DatabaseSync } from 'node:sqlite'; +import { Effect, Layer, Stream } from 'effect'; +import { layerTableCatalog } from '../plugins/catalog/table.js'; +import { layerSeededEntropy } from '../plugins/entropy/seeded.js'; +import { fakeModel, type FakeReply } from '../plugins/model/fake.js'; +import { layerAssembler } from '../plugins/prompt/default.js'; +import { layerNodeStore } from '../plugins/store/node.js'; +import type { ModelRequest } from './model.js'; +import type { ResumeContext } from './resume.js'; +import type { SessionHandle } from './handle.js'; + +/** + * What the continue and clone tests share. + * + * They run against the real SQLite store rather than a double, because what is + * under test is whether a session survives being written down and read back. A + * double would only prove that the double remembers. + */ + +const options = { system: 'the system prompt', model: 'claude-opus-5', maxTokens: 1024 }; + +interface Bench { + /** Runs one program against the same database, with the layers built afresh. */ + readonly run: ( + use: Effect.Effect + ) => Promise<{ readonly value: A; readonly calls: readonly ModelRequest[] }>; +} + +/** What a bench varies: the model's answer, and the window it is measured against. */ +interface Setup { + readonly reply?: FakeReply; + /** Without one the catalog names no window, and nothing ever compacts. */ + readonly window?: number; +} + +/** + * One database, many sessions. Each run builds its own layers, which is what a + * second start of an application does. + */ +const bench = (setup: Setup = {}): Bench => { + const database = new DatabaseSync(':memory:'); + return { + run: use => { + const model = fakeModel([setup.reply ?? { deltas: ['an answer'] }]); + const layers = Layer.mergeAll( + layerAssembler, + layerTableCatalog( + {}, + { + apiKinds: ['messages'], + ...(setup.window === undefined ? {} : { contextWindow: setup.window }), + } + ), + layerSeededEntropy(1), + model.layer, + layerNodeStore(database) + ); + return Effect.runPromise(Effect.scoped(Effect.provide(use, layers))).then(value => ({ + value, + calls: model.calls, + })); + }, + }; +}; + +/** Asks one question and keeps nothing, so a test can read what was sent. */ +const asked = (session: SessionHandle, text: string): Effect.Effect => + Stream.runDrain(session.ask(text)); + +/** What the model was sent, as plain text, so two prompts compare byte for byte. */ +const prompted = (request: ModelRequest | undefined): string => + JSON.stringify(request?.prompt ?? {}); + +export { asked, bench, options, prompted }; diff --git a/packages/harness-sdk/src/core/resume.test.ts b/packages/harness-sdk/src/core/resume.test.ts new file mode 100644 index 0000000000..ec8ba81c0d --- /dev/null +++ b/packages/harness-sdk/src/core/resume.test.ts @@ -0,0 +1,225 @@ +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { asked, bench, options, prompted } from './resume-fixture.js'; +import { cloneSession, continueSession, SessionNotFoundError } from './resume.js'; +import { openSession } from './run.js'; +import { texts } from './session-fixture.js'; +import { textIn } from './prompt.js'; + +it('carries the turns of an earlier run into the next one', async () => { + const desk = bench(); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => + Effect.as(asked(session, 'the first question'), session.id) + ) + ); + + const carried = await desk.run( + Effect.flatMap(continueSession(opened.value), session => session.history) + ); + + expect(texts(carried.value)).toEqual(['user:the first question', 'assistant:an answer']); +}); + +it('reopens with the options it was stored with, not the ones a caller has now', async () => { + const desk = bench(); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + + const resumed = await desk.run( + Effect.flatMap(continueSession(opened.value), session => asked(session, 'second')) + ); + + /* The system prompt is the front of the cached prefix. A resumed session that + took a system prompt from its caller would drop the prefix on this call. */ + expect(resumed.calls[0]?.prompt.system[0]?.text).toBe(options.system); + expect(resumed.calls[0]?.model).toBe(options.model); +}); + +it('asks the next question with the whole restored conversation in front of it', async () => { + const desk = bench(); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + + const resumed = await desk.run( + Effect.flatMap(continueSession(opened.value), session => asked(session, 'second')) + ); + + expect(resumed.calls[0]?.prompt.messages.map(textIn)).toEqual(['first', 'an answer', 'second']); +}); + +it('refuses an identifier the store has never held', async () => { + const desk = bench(); + + const failed = await desk.run(Effect.flip(continueSession('ses_nothing'))); + + expect(failed.value).toBeInstanceOf(SessionNotFoundError); +}); + +it('gives a clone its own identifier and its own turns', async () => { + const desk = bench(); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + + const cloned = await desk.run( + Effect.flatMap(cloneSession(opened.value), session => + Effect.map(session.history, turns => ({ id: session.id, turns })) + ) + ); + + expect(cloned.value.id).not.toBe(opened.value); + /* A copied turn identifier would collide on the primary key, and every + identifier also carries the order the turns are read back in. */ + const ids = cloned.value.turns.map(turn => turn.id); + expect(new Set(ids).size).toBe(2); + expect(texts(cloned.value.turns)).toEqual(['user:first', 'assistant:an answer']); +}); + +it('sends a clone the same prompt bytes as the session it came from', async () => { + const desk = bench(); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + + /* The clone goes first. Continuing the original would append to it, and the + two would then be compared at different lengths. */ + const clone = await desk.run( + Effect.flatMap(cloneSession(opened.value), session => asked(session, 'next')) + ); + const original = await desk.run( + Effect.flatMap(continueSession(opened.value), session => asked(session, 'next')) + ); + + /* This is what makes a clone cheap: the prefix is identical, so the model + reads it from its cache instead of building it again. */ + expect(prompted(clone.calls[0])).toBe(prompted(original.calls[0])); +}); + +it('leaves the original alone when the clone is asked something', async () => { + const desk = bench(); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + + await desk.run( + Effect.flatMap(cloneSession(opened.value), session => asked(session, 'only on the branch')) + ); + const original = await desk.run( + Effect.flatMap(continueSession(opened.value), session => session.history) + ); + + expect(texts(original.value)).toEqual(['user:first', 'assistant:an answer']); +}); + +/** + * Moving a conversation to another model. + * + * A session freezes its model, so this is the only way there is: the turns are + * copied onto a session opened on the other one. What cannot come with them is + * the thinking, because a provider reads back the signature it issued and + * refuses one it did not. + */ +it('opens the copy on the model it was moved onto', async () => { + const desk = bench(); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + + const moved = await desk.run( + Effect.flatMap(cloneSession(opened.value, { model: 'z-ai/glm-5.3-flash' }), session => + asked(session, 'next') + ) + ); + + expect(moved.calls[0]?.model).toBe('z-ai/glm-5.3-flash'); + /* The system prompt still comes from the store: the model moves, and + everything the caller never asked to change stays as it was. */ + expect(moved.calls[0]?.prompt.system[0]?.text).toBe(options.system); +}); + +it('leaves the thinking behind when the copy moves to another model', async () => { + const desk = bench({ reply: { deltas: ['an answer'], reasoning: ['because of this'] } }); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + + const moved = await desk.run( + Effect.flatMap( + cloneSession(opened.value, { model: 'z-ai/glm-5.3-flash' }), + session => session.history + ) + ); + + const kinds = moved.value.flatMap(turn => turn.parts.map(part => part.kind)); + expect(kinds).not.toContain('reasoning'); + /* Everything else is still there. The conversation moved; only the signed + thinking, which no other model can read back, did not. */ + expect(texts(moved.value)).toEqual(['user:first', 'assistant:an answer']); +}); + +it('keeps the thinking when the copy stays on the same model', async () => { + const desk = bench({ reply: { deltas: ['an answer'], reasoning: ['because of this'] } }); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + + const branched = await desk.run( + Effect.flatMap(cloneSession(opened.value), session => session.history) + ); + + const kinds = branched.value.flatMap(turn => turn.parts.map(part => part.kind)); + expect(kinds).toContain('reasoning'); +}); + +/** + * A resumed session knows how full it is. + * + * The compaction trigger is the provider's own count of the last request, and + * nothing here estimates one. So the count is stored beside the session and + * read back with it: a session reopened onto a conversation that already fills + * the window compacts before it asks anything, rather than sending the whole + * thing once and learning from the answer. + */ +it('compacts before its first question when the stored count fills the window', async () => { + const desk = bench({ + window: 1000, + reply: { deltas: ['an answer'], usage: { inputTokens: 900, cacheReadTokens: 0 } }, + }); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + + const resumed = await desk.run( + Effect.flatMap(continueSession(opened.value), session => asked(session, 'second')) + ); + + /* Two calls: the summary, then the question. The first answer reported 900 + of a 1000 window, and that is what the store handed back. */ + expect(resumed.calls).toHaveLength(2); + const sent = resumed.calls[1]?.prompt.messages.map(textIn) ?? []; + expect(sent).toHaveLength(2); + expect(sent[0]).toContain('Summary of the conversation'); + expect(sent[1]).toBe('second'); +}); + +it('reopens a compacted session without compacting it again', async () => { + const desk = bench({ + window: 1000, + reply: { deltas: ['an answer'], usage: { inputTokens: 900, cacheReadTokens: 0 } }, + }); + const opened = await desk.run( + Effect.flatMap(openSession(options), session => Effect.as(asked(session, 'first'), session.id)) + ); + await desk.run(Effect.flatMap(continueSession(opened.value), session => session.compact)); + + const resumed = await desk.run( + Effect.flatMap(continueSession(opened.value), session => asked(session, 'second')) + ); + + /* One call. A compaction records zero, because the next request starts from + the summary; a store that kept the old count would summarise the summary. */ + expect(resumed.calls).toHaveLength(1); +}); diff --git a/packages/harness-sdk/src/core/resume.ts b/packages/harness-sdk/src/core/resume.ts new file mode 100644 index 0000000000..4ff88accc7 --- /dev/null +++ b/packages/harness-sdk/src/core/resume.ts @@ -0,0 +1,188 @@ +import { Data, Effect, Option } from 'effect'; +import { EntropySource, type EntropySourceService } from './entropy.js'; +import type { Effort } from './model.js'; +import { makeSession } from './session.js'; +import { + SessionStore, + type SessionStoreService, + type StoredSession, + type StoreError, +} from './storage.js'; +import type { ToolMissingError } from './tool.js'; +import { draftOf, makeTurn, type PartDraft, type Turn } from './turn.js'; +import { handleOf, type SessionHandle } from './handle.js'; +import { type SessionContext, type SessionOptions, wiringFor } from './wiring.js'; + +/** The store holds no session under that identifier. */ +class SessionNotFoundError extends Data.TaggedError('harness/SessionNotFoundError')<{ + readonly sessionId: string; +}> {} + +/** Everything a resumed session needs. A store is required, not optional. */ +type ResumeContext = SessionContext | SessionStore; + +type ResumeError = StoreError | SessionNotFoundError | ToolMissingError; + +/** What the session was opened with, as `wiringFor` wants it. */ +const optionsOf = (stored: StoredSession): SessionOptions => ({ + system: stored.system, + model: stored.model, + ...(stored.effort === undefined ? {} : { effort: stored.effort }), + ...(stored.maxTokens === undefined ? {} : { maxTokens: stored.maxTokens }), + ...(stored.tools === undefined ? {} : { tools: stored.tools }), +}); + +/** + * What a clone may be opened with instead of what the store holds. + * + * These two and nothing else, because these two are why a caller clones rather + * than continues: a session freezes its model and its effort for the life of + * the cached prefix, so moving a conversation to another model is a copy of it. + * Everything else still comes from the store, and for the usual reason: a + * system prompt that differs by one byte drops the whole prefix. + */ +interface CloneOptions { + readonly model?: string; + readonly effort?: Effort; +} + +const storedOf = ( + store: SessionStoreService, + sessionId: string +): Effect.Effect => + Effect.flatMap( + store.read(sessionId), + Option.match({ + onNone: () => Effect.fail(new SessionNotFoundError({ sessionId })), + onSome: (stored: StoredSession) => Effect.succeed(stored), + }) + ); + +/** + * Reopens a session the store already holds. + * + * The options come from the store, never from the caller. A system prompt that + * differs by one byte from the one the session was opened with would drop the + * whole cached prefix on the first question, and the only symptom is the bill. + */ +const continueSession = ( + sessionId: string +): Effect.Effect => + Effect.gen(function* () { + const store = yield* SessionStore; + const stored = yield* storedOf(store, sessionId); + const turns = yield* store.load(sessionId); + const wiring = yield* wiringFor(optionsOf(stored), { id: sessionId, turns }, stored.prompted); + return yield* handleOf(wiring); + }); + +/** + * The stored options with what the caller is moving the copy onto. A field the + * caller left out is the stored one: `undefined` here means "as it was", never + * "unset it". + */ +const movedOnto = (options: SessionOptions, onto: CloneOptions | undefined): SessionOptions => ({ + ...options, + ...(onto?.model === undefined ? {} : { model: onto.model }), + ...(onto?.effort === undefined ? {} : { effort: onto.effort }), +}); + +/** + * The parts of one turn a copy may carry. + * + * All of them, unless the copy runs on another model. A thinking block is + * signed by the model that made it and read back by the same one, so it is the + * one thing that cannot move: what the model said, the images it was shown, and + * the tools it called all replay anywhere, and its thinking replays nowhere + * else. The conversation is unchanged by dropping it — a summary of its own + * reasoning is not what the next model is asked to build on. + */ +const replayable = (turn: Turn, moved: boolean): readonly PartDraft[] => + turn.parts + .filter(part => !moved || (part.kind !== 'reasoning' && part.kind !== 'redacted')) + .map(draftOf); + +/** + * Copies the turns onto a new session, in order. + * + * Each copy is a new turn with new parts, because an identifier names one row + * and carries its order. The content is unchanged, and the content is what the + * model sees, so the copy renders to the same bytes and inherits the warm + * cache. + */ +const copyTurns = ( + store: SessionStoreService, + entropy: EntropySourceService, + into: { + readonly sessionId: string; + readonly source: readonly Turn[]; + /** The source's count: the copy renders to the same bytes, so it is as full. */ + readonly prompted: number; + /** True when the copy runs on another model, which is what drops thinking. */ + readonly moved: boolean; + } +): Effect.Effect => + Effect.forEach(into.source, turn => + makeTurn(entropy, { + sessionId: into.sessionId, + role: turn.role, + parts: replayable(turn, into.moved), + }) + ).pipe( + Effect.tap(copies => + store.append({ sessionId: into.sessionId, turns: copies, prompted: into.prompted }) + ) + ); + +/** The turns to copy, and what the copy must know about them. */ +interface Copying { + readonly source: readonly Turn[]; + readonly moved: boolean; + readonly prompted: number; +} + +/** Records the copy, writes its turns, and hands back the handle over them. */ +const copyOnto = ( + store: SessionStoreService, + options: SessionOptions, + from: Copying +): Effect.Effect => + Effect.gen(function* () { + const entropy = yield* EntropySource; + const opened = yield* makeSession(entropy); + const { prompted } = from; + yield* store.create({ ...options, id: opened.id, prompted }); + const turns = yield* copyTurns(store, entropy, { sessionId: opened.id, ...from }); + return yield* handleOf(yield* wiringFor(options, { id: opened.id, turns }, prompted)); + }); + +/** + * Opens a new session holding a copy of another session's turns. + * + * The two sessions then diverge: a question asked of one leaves the other + * alone. This is how a conversation is branched without paying to build its + * prefix again, and — with `onto` — how one moves to another model at all. + * + * The count comes across with the turns, so a copy of a conversation that + * nearly fills the window compacts before its first question rather than after + * it. On another model it is the source's number rather than the copy's own, + * which the first answer replaces with the truth. + */ +const cloneSession = ( + sessionId: string, + onto?: CloneOptions +): Effect.Effect => + Effect.gen(function* () { + const store = yield* SessionStore; + const stored = yield* storedOf(store, sessionId); + const source = yield* store.load(sessionId); + const options = movedOnto(optionsOf(stored), onto); + return yield* copyOnto(store, options, { + source, + moved: options.model !== stored.model, + prompted: stored.prompted ?? 0, + }); + }); + +export type { CloneOptions, ResumeContext, ResumeError }; +export { cloneSession, continueSession, SessionNotFoundError }; diff --git a/packages/harness-sdk/src/core/retry.ts b/packages/harness-sdk/src/core/retry.ts new file mode 100644 index 0000000000..2da584b28d --- /dev/null +++ b/packages/harness-sdk/src/core/retry.ts @@ -0,0 +1,21 @@ +import { Context, type Schedule } from 'effect'; +import type { ModelError } from './model.js'; + +/** + * Decides whether and how a failed call is tried again. + * + * This is a plugin because the right policy belongs to the caller, not to the + * package. A phone on a slow link wants patience. A batch job wants to fail + * fast and move on. A caller behind its own rate limiter wants no retry at all. + * + * The schedule sees the error, so it decides both how long to wait and whether + * the error is worth waiting for. + */ +interface RetryPolicyService { + readonly schedule: Schedule.Schedule; +} + +class RetryPolicy extends Context.Tag('harness/RetryPolicy')() {} + +export type { RetryPolicyService }; +export { RetryPolicy }; diff --git a/packages/harness-sdk/src/core/run.test.ts b/packages/harness-sdk/src/core/run.test.ts new file mode 100644 index 0000000000..8b114cd52f --- /dev/null +++ b/packages/harness-sdk/src/core/run.test.ts @@ -0,0 +1,159 @@ +import { Chunk, Effect, Fiber, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { layerSeededEntropy } from '../plugins/entropy/seeded.js'; +import { fakeModel } from '../plugins/model/fake.js'; +import { layerAssembler } from '../plugins/prompt/default.js'; +import { ModelError } from './model.js'; +import { textIn } from './prompt.js'; +import { openSession } from './run.js'; +import { catalogWindowed, options, run, runWith, silentCatalog, texts } from './session-fixture.js'; +import { hitRatio } from './usage.js'; + +it('keeps the question and the answer as two turns, in order', async () => { + const { value } = await run([{ deltas: ['he', 'llo'] }], session => + Effect.zipRight(Stream.runDrain(session.ask('hi')), session.history) + ); + expect(texts(value)).toEqual(['user:hi', 'assistant:hello']); +}); + +it('takes the question back out when the stream fails part way', async () => { + const failure = new ModelError({ reason: 'transport', cause: 'cut' }); + const { value } = await run([{ deltas: ['par'], fail: failure }], session => + Effect.zipRight( + Effect.ignore(Stream.runDrain(session.ask('hi'))), + Effect.map(session.history, texts) + ) + ); + + /* A transcript that ends on an unanswered question sends it again with + every later request, and the model may answer it late. */ + expect(value).toEqual([]); +}); + +it('takes the question back out when the caller walks away mid-stream', async () => { + const { value } = await run([{ deltas: ['par'], stall: true }, { deltas: ['ok'] }], session => + Effect.gen(function* () { + const reading = yield* Effect.fork(Stream.runDrain(session.ask('a'))); + yield* Effect.sleep('10 millis'); + yield* Fiber.interrupt(reading); + /* The session must still take a question: a `busy` flag left set by the + interrupt would strand it, and the abandoned question would ride along + on every later request. */ + yield* Stream.runDrain(session.ask('b')); + return yield* Effect.map(session.history, texts); + }) + ); + expect(value).toEqual(['user:b', 'assistant:ok']); +}); + +/** A window small enough that the call below fills it, and a call that does. */ +const window = 1000; +const full = { deltas: ['answered'], usage: { inputTokens: 900, cacheReadTokens: 0 } }; + +it('takes the next question after a caller walks away mid-compaction', async () => { + /* The session is held before the summariser is called, and the stream that + releases it does not exist yet. An interrupt in that window must still + free the session, or every later question is refused as busy. */ + const { value } = await runWith({ + replies: [ + full, + { deltas: ['the notes'], stall: true }, + { deltas: ['the notes'] }, + { deltas: ['ok'] }, + ], + catalog: catalogWindowed(window), + use: session => + Effect.gen(function* () { + yield* Stream.runDrain(session.ask('one')); + const reading = yield* Effect.fork(Stream.runDrain(session.ask('two'))); + yield* Effect.sleep('10 millis'); + yield* Fiber.interrupt(reading); + yield* Stream.runDrain(session.ask('three')); + return yield* Effect.map(session.history, texts); + }), + }); + + /* The abandoned summary wrote nothing, so the window is still full and the + next question summarises before it is asked. What it must not do is fail. */ + expect(value).toEqual(['user:one', 'assistant:answered', 'user:', 'user:three', 'assistant:ok']); +}); + +it('asks the second question with the first exchange already in the prompt', async () => { + const { calls } = await run([{ deltas: ['one'] }, { deltas: ['two'] }], session => + Effect.zipRight(Stream.runDrain(session.ask('a')), Stream.runDrain(session.ask('b'))) + ); + expect(calls[0]?.prompt.messages.map(textIn)).toEqual(['a']); + expect(calls[1]?.prompt.messages.map(textIn)).toEqual(['a', 'one', 'b']); +}); + +it('adds up the token counts of every call', async () => { + const usage = { inputTokens: 5, cacheReadTokens: 95 }; + const { value } = await run([{ deltas: ['x'], usage }], session => + Effect.zipRight( + Effect.zipRight(Stream.runDrain(session.ask('a')), Stream.runDrain(session.ask('b'))), + session.usage + ) + ); + expect(value).toMatchObject({ inputTokens: 10, cacheReadTokens: 190 }); + expect(hitRatio(value)).toBeCloseTo(0.95); +}); + +it('asks with the same effort on every question of a session', async () => { + const model = fakeModel([{ deltas: ['x'] }]); + const layers = Layer.mergeAll(layerAssembler, silentCatalog, layerSeededEntropy(1), model.layer); + await Effect.runPromise( + Effect.provide( + Effect.scoped( + Effect.flatMap(openSession({ ...options, effort: 'low' }), session => + Effect.zipRight(Stream.runDrain(session.ask('a')), Stream.runDrain(session.ask('b'))) + ) + ), + layers + ) + ); + expect(model.calls.map(call => call.effort)).toEqual(['low', 'low']); +}); + +it('refuses a second question asked while the first is still streaming', async () => { + const { value } = await run([{ deltas: ['one'] }, { deltas: ['two'] }], session => + Stream.runCollect( + Stream.merge( + Stream.map(session.ask('a'), () => 'a'), + Stream.catchTag( + Stream.map(session.ask('b'), () => 'b'), + 'harness/SessionBusyError', + () => Stream.succeed('refused') + ) + ) + ).pipe(Effect.map(chunk => [...new Set(Chunk.toReadonlyArray(chunk))].toSorted())) + ); + + expect(value).toEqual(['a', 'refused']); +}); + +it('takes the next question once the first stream has ended', async () => { + const { value } = await run([{ deltas: ['one'] }, { deltas: ['two'] }], session => + Effect.zipRight( + Effect.zipRight(Stream.runDrain(session.ask('a')), Stream.runDrain(session.ask('b'))), + Effect.map(session.history, texts) + ) + ); + + expect(value).toEqual(['user:a', 'assistant:one', 'user:b', 'assistant:two']); +}); + +it('takes the next question after one fails part way', async () => { + const failure = new ModelError({ reason: 'transport', cause: 'cut' }); + const { value } = await run([{ deltas: ['par'], fail: failure }, { deltas: ['ok'] }], session => + Effect.zipRight( + Effect.zipRight( + Effect.ignore(Stream.runDrain(session.ask('a'))), + Stream.runDrain(session.ask('b')) + ), + Effect.map(session.history, texts) + ) + ); + + /* The failed question is gone, so the transcript still alternates. */ + expect(value).toEqual(['user:b', 'assistant:ok']); +}); diff --git a/packages/harness-sdk/src/core/run.ts b/packages/harness-sdk/src/core/run.ts new file mode 100644 index 0000000000..d1fbfeac3b --- /dev/null +++ b/packages/harness-sdk/src/core/run.ts @@ -0,0 +1,27 @@ +import { Effect } from 'effect'; +import { EntropySource } from './entropy.js'; +import { makeSession } from './session.js'; +import { onStore, type StoreError } from './storage.js'; +import type { ToolMissingError } from './tool.js'; +import { handleOf, type SessionHandle } from './handle.js'; +import { type SessionContext, type SessionOptions, wiringFor } from './wiring.js'; + +/** + * Opens a new session and, when a store is in the context, records it. + * + * The record is written before the first question, so a session that is + * interrupted mid-answer can still be found afterwards. Without a store the + * session runs in memory and cannot be continued. + */ +const openSession = ( + options: SessionOptions +): Effect.Effect => + Effect.gen(function* () { + const entropy = yield* EntropySource; + const opened = yield* makeSession(entropy); + const wiring = yield* wiringFor(options, opened); + yield* onStore(wiring.store, plugin => plugin.create({ ...options, id: opened.id })); + return yield* handleOf(wiring); + }); + +export { openSession }; diff --git a/packages/harness-sdk/src/core/said.test.ts b/packages/harness-sdk/src/core/said.test.ts new file mode 100644 index 0000000000..99dd735dfc --- /dev/null +++ b/packages/harness-sdk/src/core/said.test.ts @@ -0,0 +1,49 @@ +import { Effect, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { type ModelEvent, said, zeroUsage } from './model.js'; + +/** + * The fold from a stream of events down to the answer. + * + * It exists because twelve of this package's own live runs had written it out + * by hand, and because it is the first thing anybody wants that `ask` does not + * already give them. What it must get right is what it leaves out. + */ + +const streamed = (events: readonly ModelEvent[]) => + Effect.runPromise(said(Stream.fromIterable(events))); + +it('keeps the words and nothing else', async () => { + const answer = await streamed([ + { kind: 'reasoning', text: 'let me think' }, + { kind: 'delta', text: 'there are ' }, + { kind: 'redacted', data: 'ZW5jcnlwdGVk' }, + { kind: 'delta', text: 'nine files' }, + { kind: 'toolCall', call: { id: 'tc_1', name: 'look', arguments: '{}' } }, + { kind: 'toolResult', result: { callId: 'tc_1', body: 'nine', failed: false } }, + { kind: 'done', usage: zeroUsage, stop: 'end' }, + ]); + + /* Thinking is not the answer, and neither is what a tool said. A reader shown + either as the model's words would be shown something it never said. */ + expect(answer).toBe('there are nine files'); +}); + +it('gives back what the model said after its tools, not before', async () => { + const answer = await streamed([ + { kind: 'delta', text: 'looking' }, + { kind: 'toolCall', call: { id: 'tc_1', name: 'look', arguments: '{}' } }, + { kind: 'done', usage: zeroUsage, stop: 'tools' }, + { kind: 'toolResult', result: { callId: 'tc_1', body: 'nine', failed: false } }, + { kind: 'delta', text: 'there are nine' }, + { kind: 'done', usage: zeroUsage, stop: 'end' }, + ]); + + /* Both rounds, because both are what the model said in answer to the one + question. A caller that wants only the last round watches `done` itself. */ + expect(answer).toBe('lookingthere are nine'); +}); + +it('answers with nothing when the model said nothing', async () => { + expect(await streamed([{ kind: 'done', usage: zeroUsage, stop: 'refusal' }])).toBe(''); +}); diff --git a/packages/harness-sdk/src/core/session-fixture.ts b/packages/harness-sdk/src/core/session-fixture.ts new file mode 100644 index 0000000000..17f14a19ca --- /dev/null +++ b/packages/harness-sdk/src/core/session-fixture.ts @@ -0,0 +1,138 @@ +import { Effect, Layer, Option } from 'effect'; +import { layerTableCatalog } from '../plugins/catalog/table.js'; +import type { ModelCatalog } from './catalog.js'; +import { layerSeededEntropy } from '../plugins/entropy/seeded.js'; +import { fakeModel, type FakeReply } from '../plugins/model/fake.js'; +import { layerAssembler } from '../plugins/prompt/default.js'; +import type { SessionBusyError } from './ask.js'; +import type { ModelError, ModelRequest } from './model.js'; +import { openSession } from './run.js'; +import type { SessionHandle } from './handle.js'; +import type { SessionOptions } from './wiring.js'; +import { SessionStore, StoreError, type StoredExchange } from './storage.js'; +import type { ToolRegistry } from './tool.js'; +import { textOf, type Turn } from './turn.js'; + +/** + * What the session tests share. It lives outside a `*.test.ts` file so three + * test files can use it without repeating it, and it is excluded from `dist/` + * along with the other test doubles. + */ + +const options = { system: 'sys', model: 'claude-opus-5', maxTokens: 1024 }; + +/** A catalog that names no output limit, so the package falls back to 4096. */ +const silentCatalog = layerTableCatalog({}, { apiKinds: ['messages'] }); + +/** A catalog that does name one, which is what a caller who names none gets. */ +const catalogSaying = (maxOutputTokens: number): Layer.Layer => + layerTableCatalog({}, { apiKinds: ['messages'], maxOutputTokens }); + +/** A catalog that answers for nothing, which is what an empty table does. */ +const emptyCatalog = layerTableCatalog({}); + +const recordingStore = (): { + readonly seen: string[]; + readonly layer: Layer.Layer; +} => { + const seen: string[] = []; + const layer = Layer.succeed(SessionStore, { + create: () => Effect.void, + read: () => Effect.succeed(Option.none()), + append: ({ turns }: StoredExchange) => + Effect.sync(() => { + for (const turn of turns) { + seen.push(`${turn.role}:${textOf(turn)}`); + } + }), + load: () => Effect.succeed([] as readonly Turn[]), + flush: () => Effect.sync(() => void seen.push('flush')), + }); + return { seen, layer }; +}; + +/** + * A store that fails the operation named, and records what it was told about. + * `seen` is what the store actually holds; the session's own history is what + * it thinks the store holds. The two diverging is the defect under test. + */ +const brokenStore = ( + broken: 'append' | 'flush' +): { readonly seen: string[]; readonly layer: Layer.Layer } => { + const seen: string[] = []; + const refuse = (operation: 'append' | 'flush') => + Effect.fail(new StoreError({ operation, cause: 'the disk is full' })); + const layer = Layer.succeed(SessionStore, { + create: () => Effect.void, + read: () => Effect.succeed(Option.none()), + append: ({ turns }: StoredExchange) => + broken === 'append' + ? refuse('append') + : Effect.sync(() => { + for (const turn of turns) { + seen.push(`${turn.role}:${textOf(turn)}`); + } + }), + load: () => Effect.succeed([] as readonly Turn[]), + flush: () => + broken === 'flush' ? refuse('flush') : Effect.sync(() => void seen.push('flush')), + }); + return { seen, layer }; +}; + +const run = ( + replies: readonly FakeReply[], + use: (session: SessionHandle) => Effect.Effect, + store?: Layer.Layer +): Promise<{ readonly value: A; readonly calls: readonly ModelRequest[] }> => + runWith({ replies, use, ...(store === undefined ? {} : { store }) }); + +/** What one session test needs. `run` is this with the usual catalog and options. */ +interface Setup { + readonly replies: readonly FakeReply[]; + readonly use: ( + session: SessionHandle + ) => Effect.Effect; + readonly store?: Layer.Layer; + readonly catalog?: Layer.Layer; + readonly options?: SessionOptions; + /** The tools the session may name. A test that names none provides none. */ + readonly tools?: Layer.Layer; +} + +const runWith = ( + setup: Setup +): Promise<{ readonly value: A; readonly calls: readonly ModelRequest[] }> => { + const model = fakeModel(setup.replies); + const layers = Layer.mergeAll( + layerAssembler, + setup.catalog ?? silentCatalog, + layerSeededEntropy(1), + model.layer, + setup.tools ?? Layer.empty + ); + const program = Effect.scoped(Effect.flatMap(openSession(setup.options ?? options), setup.use)); + return Effect.runPromise( + Effect.provide(program, setup.store === undefined ? layers : Layer.merge(layers, setup.store)) + ).then(value => ({ value, calls: model.calls })); +}; + +/** A catalog that names a window, which is what makes a session compact itself. */ +const catalogWindowed = (contextWindow: number): Layer.Layer => + layerTableCatalog({}, { apiKinds: ['messages'], contextWindow }); + +const texts = (turns: readonly Turn[]): readonly string[] => + turns.map(turn => `${turn.role}:${textOf(turn)}`); + +export { + brokenStore, + catalogSaying, + catalogWindowed, + emptyCatalog, + options, + recordingStore, + run, + runWith, + silentCatalog, + texts, +}; diff --git a/packages/harness-sdk/src/core/session.test.ts b/packages/harness-sdk/src/core/session.test.ts new file mode 100644 index 0000000000..c037583f5a --- /dev/null +++ b/packages/harness-sdk/src/core/session.test.ts @@ -0,0 +1,17 @@ +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { seededEntropy } from '../plugins/entropy/seeded.js'; +import { makeSession } from './session.js'; + +const entropy = seededEntropy(1); + +const run = (effect: Effect.Effect): A => Effect.runSync(effect); + +it('makes an identifier of the form ses_{ulid}', () => { + expect(run(makeSession(entropy)).id).toMatch(/^ses_[0-9A-HJKMNP-TV-Z]{26}$/); +}); + +it('orders two identifiers by the order they were made in', () => { + const [first, second] = run(Effect.all([makeSession(entropy), makeSession(entropy)])); + expect(first.id < second.id).toBe(true); +}); diff --git a/packages/harness-sdk/src/core/session.ts b/packages/harness-sdk/src/core/session.ts new file mode 100644 index 0000000000..1a0df7a3f6 --- /dev/null +++ b/packages/harness-sdk/src/core/session.ts @@ -0,0 +1,50 @@ +import { Effect } from 'effect'; +import type { EntropySourceService } from './entropy.js'; +import { makeId } from './id.js'; +import type { Turn } from './turn.js'; + +/** + * A session is one conversation between a user and an agent. Everything else + * plugs into it. The turns are append only: an earlier turn is never rewritten, + * because a rewrite changes the prompt prefix and drops the model cache. + */ +interface Session { + readonly id: string; + readonly turns: readonly Turn[]; +} + +const idPrefix = 'ses'; + +/** Makes an empty session. */ +const makeSession = (entropy: EntropySourceService): Effect.Effect => + Effect.map(makeId(entropy, idPrefix), id => ({ id, turns: [] })); + +/** + * Appends a turn. A copy, so nothing that already read the turns can see it + * change. Building a 200 turn session this way costs 15 us in total, which is + * less than the same session costs through `Chunk`, so the plain array is both + * the simpler and the faster one. + */ +const appendTurn = (session: Session, turn: Turn): Session => ({ + ...session, + turns: [...session.turns, turn], +}); + +/** + * The turns a prompt is built from: everything from the last summary onward. + * + * Compaction replaces the conversation with a summary and replays nothing + * before it. Keeping the earlier turns and only summarising the old ones is the + * shape that breaks: a thinking block was signed against the whole history that + * stood when it was made, so replaying it after a summary is refused. + * + * The earlier turns are still in memory and still in the store. They are the + * record of what happened; they are simply not what the model is asked with. + */ +const sinceSummary = (turns: readonly Turn[]): readonly Turn[] => { + const at = turns.findLastIndex(turn => turn.parts.some(part => part.kind === 'summary')); + return at <= 0 ? turns : turns.slice(at); +}; + +export type { Session }; +export { appendTurn, makeSession, sinceSummary }; diff --git a/packages/harness-sdk/src/core/stop.test.ts b/packages/harness-sdk/src/core/stop.test.ts new file mode 100644 index 0000000000..1934664a32 --- /dev/null +++ b/packages/harness-sdk/src/core/stop.test.ts @@ -0,0 +1,94 @@ +import { Chunk, Effect, Option, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { completionsWire } from '../plugins/gateway/wire/completions.js'; +import { messagesWire } from '../plugins/gateway/wire/messages.js'; +import { responsesWire } from '../plugins/gateway/wire/responses.js'; +import type { ModelEvent, StopReason } from './model.js'; +import { run } from './session-fixture.js'; + +/** + * The stop reason is the difference between a finished answer and one the wall + * cut off mid-sentence. A caller that cannot tell them apart stores half a + * thought and builds every later request on it. + */ + +const stopOf = (events: Chunk.Chunk): StopReason | undefined => { + const last = Chunk.last(events).pipe(Option.getOrUndefined); + return last?.kind === 'done' ? last.stop : undefined; +}; + +it('reports a finished answer as finished', async () => { + const { value } = await run([{ deltas: ['all of it'] }], session => + Stream.runCollect(session.ask('why')) + ); + expect(stopOf(value)).toBe('end'); +}); + +it('reports an answer the ceiling cut off', async () => { + const { value } = await run([{ deltas: ['half a th'], stop: 'maxTokens' }], session => + Stream.runCollect(session.ask('why')) + ); + expect(stopOf(value)).toBe('maxTokens'); +}); + +it('reads the stop reason of the messages shape', () => { + expect(messagesWire.toStop({ delta: { stop_reason: 'end_turn' } })).toBe('end'); + expect(messagesWire.toStop({ delta: { stop_reason: 'stop_sequence' } })).toBe('end'); + expect(messagesWire.toStop({ delta: { stop_reason: 'max_tokens' } })).toBe('maxTokens'); + expect(messagesWire.toStop({ delta: { stop_reason: 'refusal' } })).toBe('refusal'); + /* The other wall. The provider's guidance is to treat it as truncated, and + truncated is what `maxTokens` means here. */ + expect(messagesWire.toStop({ delta: { stop_reason: 'model_context_window_exceeded' } })).toBe( + 'maxTokens' + ); + expect(messagesWire.toStop({ delta: { stop_reason: 'tool_use' } })).toBe('tools'); + /* A name this package has never seen is `unknown`, never a guess. */ + expect(messagesWire.toStop({ delta: { stop_reason: 'pause_turn' } })).toBe('unknown'); + /* Every other frame says nothing, so it must not overwrite what was said. */ + expect(messagesWire.toStop({ delta: { text: 'he' } })).toBeUndefined(); + expect(messagesWire.toStop({ usage: { output_tokens: 4 } })).toBeUndefined(); +}); + +it('reads the stop reason of the responses shape', () => { + expect(responsesWire.toStop({ type: 'response.completed', response: {} })).toBe('end'); + expect( + responsesWire.toStop({ + type: 'response.incomplete', + response: { status: 'incomplete', incomplete_details: { reason: 'max_output_tokens' } }, + }) + ).toBe('maxTokens'); + expect( + responsesWire.toStop({ + type: 'response.incomplete', + response: { status: 'incomplete', incomplete_details: { reason: 'content_filter' } }, + }) + ).toBe('refusal'); + expect(responsesWire.toStop({ type: 'response.output_text.delta', delta: 'he' })).toBeUndefined(); +}); + +it('reads the stop reason of the chat shape', () => { + expect(completionsWire.toStop({ choices: [{ finish_reason: 'stop' }] })).toBe('end'); + expect(completionsWire.toStop({ choices: [{ finish_reason: 'length' }] })).toBe('maxTokens'); + expect(completionsWire.toStop({ choices: [{ finish_reason: 'content_filter' }] })).toBe( + 'refusal' + ); + expect(completionsWire.toStop({ choices: [{ delta: { content: 'he' } }] })).toBeUndefined(); +}); + +it('says unknown rather than guessing when the shape reported nothing', async () => { + /* A gateway that relays a provider which sends no reason at all still has to + answer the question, and `unknown` is the honest answer. */ + const { value } = await run([{ deltas: ['said'], stop: 'unknown' }], session => + Stream.runCollect(session.ask('why')) + ); + expect(stopOf(value)).toBe('unknown'); +}); + +it('keeps the answer of a call the ceiling cut off', async () => { + /* A truncated answer is still the model's turn. Dropping it would lose what + was paid for and shorten every prompt that follows. */ + const { value } = await run([{ deltas: ['half a th'], stop: 'maxTokens' }], session => + Effect.zipRight(Stream.runDrain(session.ask('why')), session.history) + ); + expect(value[1]?.parts).toMatchObject([{ kind: 'text', body: 'half a th' }]); +}); diff --git a/packages/harness-sdk/src/core/storage.ts b/packages/harness-sdk/src/core/storage.ts new file mode 100644 index 0000000000..0520290794 --- /dev/null +++ b/packages/harness-sdk/src/core/storage.ts @@ -0,0 +1,101 @@ +import { Context, Data, Effect, Option } from 'effect'; +import type { Effort } from './model.js'; +import type { Turn } from './turn.js'; + +/** A store failed to read or to write. */ +class StoreError extends Data.TaggedError('harness/StoreError')<{ + readonly operation: 'create' | 'read' | 'append' | 'load' | 'flush'; + readonly cause: unknown; +}> {} + +/** + * Everything the store must give back for a session to be reopened as it stood. + * + * Most of it is what `SessionOptions` freezes, stored because a continued + * session must be opened with the same values. Resuming under a system prompt + * that differs by one byte drops the whole cached prefix, so the store + * remembers rather than trusting the caller to pass them again. + * + * `prompted` is the exception: it changes with every answer. It is the + * provider's own count of the last request, and it is what decides whether the + * next question compacts first. Absent means no request has been recorded yet, + * which is a new session, or one written before this package stored the count. + */ +interface StoredSession { + readonly id: string; + readonly system: string; + readonly model: string; + readonly effort?: Effort; + readonly maxTokens?: number; + readonly prompted?: number; + /** + * The tools the session offers, by name, in the order the model sees them. + * Absent means none: a session that named none is reopened offering none. + */ + readonly tools?: readonly string[]; +} + +/** + * One completed exchange, as the store takes it. + * + * The turns and the count go together because they describe the same request: + * a store that wrote the turns and lost the count would hand back a session + * that does not know how full it is, and a store that wrote the count without + * the turns would hand back one that thinks it is fuller than it is. + */ +interface StoredExchange { + /** The session this belongs to, named as every other method here names it. */ + readonly sessionId: string; + /** + * The turns to record. It is a list because the store must never hold a + * question with no answer: such a question goes back out with every later + * request, so the caller pays for it again each time and the model may answer + * it late. + */ + readonly turns: readonly Turn[]; + /** + * What the request that produced them put in front of the model, from the + * provider's own count. A compaction records zero: the next request starts + * from the summary, so what the last one cost says nothing about it. + */ + readonly prompted: number; +} + +/** + * Holds the sessions and their turns. The plugin has two jobs: take what it is + * given and store it, and read it back and parse it. Every read must validate + * what it finds, because a store is an edge. + * + * The session notifies the plugin on every change and on close. When to write, + * whether to batch, and how to recover is the plugin's decision. + * + * A store is optional. A session with no `SessionStore` in its context keeps + * its turns in memory only, and cannot be continued or cloned. + */ +interface SessionStoreService { + /** Records a new session. The session calls this once, at open. */ + readonly create: (session: StoredSession) => Effect.Effect; + /** `None` when the store has never heard of the identifier. */ + readonly read: (sessionId: string) => Effect.Effect, StoreError>; + /** Records one completed exchange, turns and count together, or neither. */ + readonly append: (exchange: StoredExchange) => Effect.Effect; + /** The turns of one session, oldest first. */ + readonly load: (sessionId: string) => Effect.Effect; + /** Writes whatever the plugin still holds. The session calls this on close. */ + readonly flush: () => Effect.Effect; +} + +class SessionStore extends Context.Tag('harness/SessionStore')< + SessionStore, + SessionStoreService +>() {} + +/** Runs the work when there is a store, and does nothing when there is not. */ +const onStore = ( + store: Option.Option, + use: (plugin: SessionStoreService) => Effect.Effect +): Effect.Effect => + Option.match(store, { onNone: () => Effect.void, onSome: use }); + +export type { SessionStoreService, StoredExchange, StoredSession }; +export { onStore, SessionStore, StoreError }; diff --git a/packages/harness-sdk/src/core/store.test.ts b/packages/harness-sdk/src/core/store.test.ts new file mode 100644 index 0000000000..540769d59e --- /dev/null +++ b/packages/harness-sdk/src/core/store.test.ts @@ -0,0 +1,64 @@ +import { Effect, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { brokenStore, recordingStore, run, texts } from './session-fixture.js'; + +it('tells the store about every turn and asks it to flush on close', async () => { + const store = recordingStore(); + await run([{ deltas: ['hello'] }], session => Stream.runDrain(session.ask('hi')), store.layer); + expect(store.seen).toEqual(['user:hi', 'assistant:hello', 'flush']); +}); + +it('runs with no store at all', async () => { + const { value } = await run([{ deltas: ['hello'] }], session => + Effect.zipRight(Stream.runDrain(session.ask('hi')), Effect.map(session.history, texts)) + ); + expect(value).toEqual(['user:hi', 'assistant:hello']); +}); + +it('keeps memory and the store agreeing when the store refuses the write', async () => { + const store = brokenStore('append'); + const { value } = await run( + [{ deltas: ['x'] }], + session => + Effect.zipRight( + Effect.ignore(Stream.runDrain(session.ask('hi'))), + Effect.map(session.history, texts) + ), + store.layer + ); + + /* Neither holds the exchange. A session that kept a turn the store refused + would load back from a different turn and miss the cache for good. */ + expect(value).toEqual([]); + expect(store.seen).toEqual(['flush']); +}); + +it('reports the store failure to the caller rather than swallowing it', async () => { + const store = brokenStore('append'); + const failure = await run( + [{ deltas: ['x'] }], + session => Effect.either(Stream.runDrain(session.ask('hi'))), + store.layer + ); + + expect(failure.value).toMatchObject({ + _tag: 'Left', + left: { _tag: 'harness/StoreError', operation: 'append' }, + }); +}); + +it('closes the session even when the final flush fails', async () => { + const store = brokenStore('flush'); + const { value } = await run( + [{ deltas: ['x'] }], + session => Effect.zipRight(Stream.runDrain(session.ask('hi')), Effect.succeed('closed')), + store.layer + ); + + /* `run.ts` ignores a failing flush on purpose: a close that throws would + hide the answer the caller already has. The cost is that whatever the + store still buffered is lost silently, which is the plugin's risk to + manage. Pinned here so that trade is a decision, not an accident. */ + expect(value).toBe('closed'); + expect(store.seen).toEqual(['user:hi', 'assistant:x']); +}); diff --git a/packages/harness-sdk/src/core/token.ts b/packages/harness-sdk/src/core/token.ts new file mode 100644 index 0000000000..ee4ac7640f --- /dev/null +++ b/packages/harness-sdk/src/core/token.ts @@ -0,0 +1,31 @@ +import { Context, Data, type Effect } from 'effect'; + +class TokenError extends Data.TaggedError('harness/TokenError')<{ + readonly cause: unknown; +}> {} + +/** + * Supplies the credential for one call. + * + * This is a plugin because a token is not a constant. The kilo token carries an + * expiry, so a session that outlives it starts failing with 401 while holding a + * string it still believes in. Asking per call lets a plugin refresh, read a + * keychain, or mint a short-lived token, without the package knowing how. + * + * The call is on the request path, so a plugin that fetches must cache; the + * package asks every time and does not cache on the plugin's behalf. + * + * **Return the work, do not do it.** A failed call is retried by re-running + * the effect this returned, `get` included, so a plugin that reads its state + * while building the effect hands the same stale credential to every attempt. + * `Effect.suspend`, `Effect.promise` and `Effect.sync` all defer to run time; + * a bare `Effect.succeed(cache.token)` does not. + */ +interface TokenSourceService { + readonly get: () => Effect.Effect; +} + +class TokenSource extends Context.Tag('harness/TokenSource')() {} + +export type { TokenSourceService }; +export { TokenError, TokenSource }; diff --git a/packages/harness-sdk/src/core/tool.test.ts b/packages/harness-sdk/src/core/tool.test.ts new file mode 100644 index 0000000000..89f04dcb2b --- /dev/null +++ b/packages/harness-sdk/src/core/tool.test.ts @@ -0,0 +1,42 @@ +import { Effect, Layer } from 'effect'; +import { expect, it } from 'vitest'; +import { resolveTools, type Tool, ToolMissingError, ToolRegistry } from './tool.js'; + +/** + * The tools a session offers are resolved when it opens, not when a question + * happens to want one. A name nothing holds is a session that would send a + * prefix promising a tool it cannot run, and the model would call it. + */ + +const named = (name: string): Tool => ({ + definition: { name, description: name, parameters: { type: 'object', properties: {} } }, + run: () => Effect.succeed('done'), +}); + +const registry = (...tools: readonly Tool[]) => Layer.succeed(ToolRegistry, { tools }); + +const resolve = (names: readonly string[], ...tools: readonly Tool[]) => + Effect.runPromise(Effect.either(Effect.provide(resolveTools(names), registry(...tools)))); + +it('resolves the names in the order the session asked for them', async () => { + const resolved = await resolve(['b', 'a'], named('a'), named('b')); + + /* The order is the order of the prefix, not the order of the registry. */ + expect(resolved._tag === 'Right' && resolved.right.map(tool => tool.definition.name)).toEqual([ + 'b', + 'a', + ]); +}); + +it('refuses to open a session naming a tool nothing holds', async () => { + const resolved = await resolve(['a', 'missing'], named('a')); + + expect(resolved._tag === 'Left' && resolved.left).toBeInstanceOf(ToolMissingError); +}); + +it('asks for no registry when the session names no tool', async () => { + /* No layer at all, which is what a caller with no tools provides. */ + const resolved = await Effect.runPromise(resolveTools([])); + + expect(resolved).toEqual([]); +}); diff --git a/packages/harness-sdk/src/core/tool.ts b/packages/harness-sdk/src/core/tool.ts new file mode 100644 index 0000000000..b6a07759c0 --- /dev/null +++ b/packages/harness-sdk/src/core/tool.ts @@ -0,0 +1,229 @@ +import { Context, Data, Duration, Effect, Option } from 'effect'; + +/** + * A tool is something the model may ask for, and the code that answers. + * + * The package splits a tool in two, and the split is not cosmetic. What the + * model is told — the name, the description, the schema — sits in front of + * every message of every request, so it is part of the cached prefix and is + * frozen for the life of a session, exactly as the system prompt is. What the + * tool does is a plugin, resolved when the session opens. + * + * A session names the tools it offers, in order, and the order is part of the + * prefix too. The definitions come from the registry rather than from the + * store: a definition lives in code and ships with the build, where a system + * prompt is a value a caller made at run time. See AGENTS.md, "A session names + * its tools; the registry defines them". + */ + +/** + * The JSON Schema of a tool's arguments. Every shape takes one, under a name of + * its own, and none of them looks inside it. + * + * It is stated rather than left as `unknown` so a tool this package ships is + * checked when it is written. It is not validated at run time: a definition is + * a caller's own value, not an edge. See AGENTS.md, principle 10. + */ +interface JsonSchema { + readonly type: 'object'; + readonly properties: Readonly>; + readonly required?: readonly string[]; + readonly additionalProperties?: boolean; +} + +/** What the model is told a tool is. Frozen: it sits in the cached prefix. */ +interface ToolDefinition { + readonly name: string; + readonly description: string; + readonly parameters: JsonSchema; +} + +/** + * What the model asked for. + * + * `arguments` is the JSON text the model wrote, not a parsed object. It is text + * because that is how every shape streams it, because the tool is the only + * thing that knows what shape it should be, and because a model that writes + * malformed JSON must be told so rather than crash the session. + */ +interface ToolCall { + readonly id: string; + readonly name: string; + readonly arguments: string; +} + +/** What goes back to the model, against the call it answers. */ +interface ToolResult { + readonly callId: string; + readonly body: string; + /** True when the tool did not do what it was asked. The model reads it and retries. */ + readonly failed: boolean; +} + +/** + * A tool did not do what it was asked. + * + * This is not a failed session. The runner turns it into a failed result and + * hands it to the model, which is the only party that can decide what to do + * about it. Nothing a tool does fails the stream. + */ +class ToolFailure extends Data.TaggedError('harness/ToolFailure')<{ + readonly cause: unknown; +}> {} + +/** A session named a tool that the registry does not hold. */ +class ToolMissingError extends Data.TaggedError('harness/ToolMissingError')<{ + readonly tool: string; +}> {} + +/** + * A tool, as the registry holds it. + * + * `run` takes no context: a tool comes from a layer, so whatever it needs was + * provided when that layer was built. + */ +interface Tool { + readonly definition: ToolDefinition; + /** + * How long the model waits for this tool before the call goes to the + * background. The session's own limit applies when this names none. + * + * Zero backgrounds every call to it at once, which is what a tool that waits + * on a person wants: no model should sit on an open request while somebody + * reads a question. + * + * It bounds the waiting; it does not decide whether there is any. `wait` + * decides that, and the model's own answer beats both. + */ + readonly inlineFor?: Duration.DurationInput; + /** + * Whether the model waits for this tool, as the model is told by default. + * + * The tool knows something the harness cannot. A question the model asked + * because it cannot go on without the answer is worth waiting for, so + * `question` says true. A subagent the model handed a task to is not: the + * whole point of handing it over is to carry on, so `subagent` says false. + * + * It reaches the model as the schema's `default`, so a model that says + * nothing gets what the tool expects and a model that knows better overrides + * it. Without it the default is read from `inlineFor`: a tool that waits no + * time at all advertises false, and everything else advertises true. + */ + readonly wait?: boolean; + readonly run: (call: ToolCall) => Effect.Effect; +} + +/** + * Every tool a session may be opened with. It is one service and not one per + * tool, because a caller assembles the set once and the session picks from it + * by name. + */ +interface ToolRegistryService { + readonly tools: readonly Tool[]; +} + +class ToolRegistry extends Context.Tag('harness/ToolRegistry')< + ToolRegistry, + ToolRegistryService +>() {} + +/** The tool of that name, or nothing. */ +const toolNamed = (tools: readonly Tool[], name: string): Option.Option => + Option.fromNullable(tools.find(tool => tool.definition.name === name)); + +/** + * The name of the field the model sets to choose whether it waits. + * + * It is the harness's field and not the tool's: every tool carries it, no tool + * author writes it, and the runner takes it off again before the tool ever sees + * the arguments. + */ +const waitField = 'wait'; + +/** + * How long the model waits for a tool that names no deadline of its own, and + * whose session names none either. Half a minute is long enough for anything + * that reads a file or asks a server, and short enough that a request is not + * left open on something slower. + */ +const defaultInlineFor = Duration.seconds(30); + +const waitProperty = { + type: 'boolean', + description: + 'Whether to wait for this call. False hands you a note saying it is ' + + 'still running, so you can carry on with what does not depend on it, and ' + + 'the result reaches you in a later message. True waits for it. The default ' + + 'is what this tool expects — leave it out unless this call is different.', +}; + +/** + * Whether the model waits for this tool when it says nothing. + * + * The tool's own answer if it gave one. Otherwise it is read from the deadline: + * a tool that waits no time at all is a tool nobody waits for. + */ +const waitsFor = (tool: Tool, session?: Duration.DurationInput): boolean => + tool.wait ?? !Duration.isZero(Duration.decode(tool.inlineFor ?? session ?? defaultInlineFor)); + +/** One tool as the model is told it, with the field the harness adds to all of them. */ +const asOffered = (tool: Tool, session?: Duration.DurationInput): ToolDefinition => ({ + ...tool.definition, + parameters: { + ...tool.definition.parameters, + properties: { + ...tool.definition.parameters.properties, + [waitField]: { ...waitProperty, default: waitsFor(tool, session) }, + }, + }, +}); + +/** + * What the model is told about the tools, in the order the session named them. + * + * Every one of them gains `wait`, because whether the model waits for a call is + * the model's decision to make and not the tool author's. What the tool author + * decided reaches the model as that field's default, so a model that says + * nothing gets it. + */ +const definitionsOf = ( + tools: readonly Tool[], + session?: Duration.DurationInput +): readonly ToolDefinition[] => tools.map(tool => asOffered(tool, session)); + +/** + * Resolves the names a session was opened with, in that order, against the + * registry. A name nothing holds fails here, when the session opens, rather + * than at the first question that happens to want it. + * + * A session that names no tool never asks for the registry, so a caller with no + * tools needs no layer for them. + */ +const resolveTools = (names: readonly string[]): Effect.Effect => + names.length === 0 + ? Effect.succeed([]) + : Effect.flatMap(Effect.serviceOption(ToolRegistry), registry => + Effect.forEach(names, name => + Option.match( + Option.flatMap(registry, held => toolNamed(held.tools, name)), + { + onNone: () => Effect.fail(new ToolMissingError({ tool: name })), + onSome: (tool: Tool) => Effect.succeed(tool), + } + ) + ) + ); + +export type { JsonSchema, Tool, ToolCall, ToolDefinition, ToolRegistryService, ToolResult }; +export { + asOffered, + defaultInlineFor, + definitionsOf, + resolveTools, + ToolFailure, + ToolMissingError, + waitField, + waitsFor, + ToolRegistry, + toolNamed, +}; diff --git a/packages/harness-sdk/src/core/tools.ts b/packages/harness-sdk/src/core/tools.ts new file mode 100644 index 0000000000..ea2a1d2214 --- /dev/null +++ b/packages/harness-sdk/src/core/tools.ts @@ -0,0 +1,177 @@ +import { Cause, Deferred, Duration, Effect, type Exit, Fiber, Option } from 'effect'; +import { enqueue } from './queue.js'; +import { type Tool, type ToolCall, type ToolFailure, type ToolResult, toolNamed } from './tool.js'; +import { makeTurn, type PartDraft, type Turn } from './turn.js'; +import { waited, waitFor, wanted, whileWaiting } from './waiting.js'; +import type { Wiring } from './wiring.js'; + +/** + * Runs the tools one turn asked for, and turns what they say into a turn. + * + * Nothing here fails. A tool that throws, a tool that does not exist, arguments + * that are not JSON: each of those is a failed result handed back to the model, + * which is the only party that can decide what to do about it. A session that + * ended because a tool did not like its arguments would be a session that gives + * up where a person would try again. + * + * The calls of one turn run at once. The model asks for several because they + * are independent, and running them one after another spends the wall clock for + * nothing. Nothing here serialises anything: a tool that holds one thing holds a + * permit beside it, because the thing is the caller's and the session cannot see + * it. See "A session is independent of every other" in AGENTS.md. + */ + +/** What the model is told when it names a tool the session does not offer. */ +const noSuchTool = (name: string): string => + `There is no tool named ${name} in this session. Use one of the tools you were given.`; + +const refusal = (call: ToolCall, body: string): ToolResult => ({ + callId: call.id, + body, + failed: true, +}); + +/** + * What went wrong, in the words the model gets. + * + * A `ToolFailure` says it in its cause, which is what the tool's author wrote + * for exactly this. Anything else is a defect, and the thrown value says as + * much as there is to say. Neither is rendered with `Cause.pretty`: that + * carries a stack trace, and a stack trace in a tool result is paid for on + * every request of the session from then on. + */ +const reasonOf = (cause: Cause.Cause): string => + Option.match(Cause.failureOption(cause), { + onSome: (failure: ToolFailure) => String(failure.cause), + onNone: () => String(Cause.squash(cause)), + }); + +/** What the model is told about a call it will hear about later. */ +const stillRunning = (call: ToolCall): ToolResult => ({ + callId: call.id, + body: + 'This call is still running. Its result will reach you in a later message. ' + + 'Carry on with whatever does not depend on it.', + /* Not a failure. Nothing went wrong; the answer is simply not here yet, and a + model told this had failed would start again rather than wait. */ + failed: false, +}); + +/** + * The work itself, without the waiting. + * + * Interruption passes through rather than becoming a result: a session the + * caller stopped has nobody left to tell, and a result written after the stream + * was dropped would land in a transcript nobody asked for. + * + * Nothing here serialises anything. A tool that must not be re-entered holds + * its own permit — see "A session is independent of every other" in AGENTS.md. + */ +const working = (tool: Tool, call: ToolCall): Effect.Effect => + tool.run(call).pipe( + Effect.map((body): ToolResult => ({ callId: call.id, body, failed: false })), + Effect.catchAllCause(cause => + Cause.isInterruptedOnly(cause) + ? Effect.interrupt + : Effect.succeed(refusal(call, reasonOf(cause))) + ) + ); + +/** + * One call, run under a deadline it can outlive. + * + * Every call is forked, so the deadline moves the model on rather than + * cancelling anything: the work keeps running in the session's own scope, and + * what it eventually says goes on the queue that `background.ts` drains. That + * is why any tool at all can be backgrounded — the harness decides when to stop + * waiting, not the tool, and a tool that always outlives a request says + * `inlineFor: 0` and is backgrounded from the start. Zero is read rather than + * timed: a zero-length deadline raced against the work is a race the work + * usually wins, and a tool that asked never to be waited for would be waited + * for anyway. + */ +const under = (wiring: Wiring, tool: Tool, asked: ToolCall): Effect.Effect => + Effect.gen(function* () { + const { wait: wanting, call } = wanted(asked); + const fiber = yield* Effect.forkIn(working(tool, call), wiring.scope); + const release = yield* Deferred.make(); + const wait = Duration.decode(waitFor(wiring, tool, wanting)); + const ended = yield* whileWaiting(wiring, { call, release }, waited(fiber, release, wait)); + return yield* Option.match(ended, { + onSome: (exit: Exit.Exit) => exit, + onNone: () => later(wiring, call, fiber), + }); + }); + +/** + * How the answer reads to the model when it arrives out of turn. + * + * It goes back as words the conversation says, not as a second tool result: the + * call it belongs to was already answered, and every shape refuses a second + * result for one call. + */ +const lateText = (call: ToolCall, result: ToolResult): string => + `The ${call.name} call you made earlier (id ${call.id}) has ` + + `${result.failed ? 'failed' : 'finished'}:\n\n${result.body}`; + +/** Hands the model on, and puts what the call says in the line when it says it. */ +const later = ( + wiring: Wiring, + call: ToolCall, + fiber: Fiber.RuntimeFiber +): Effect.Effect => + Effect.as( + Effect.forkIn( + Effect.flatMap(Fiber.join(fiber), result => + enqueue(wiring.pending, { + kind: 'toolResult', + parts: [{ kind: 'text', body: lateText(call, result) }], + }) + ), + wiring.scope + ), + stillRunning(call) + ); + +/** One call, whatever it asked for. A name the session does not offer is a refusal. */ +const runOne = (wiring: Wiring, call: ToolCall): Effect.Effect => + Option.match(toolNamed(wiring.tools, call.name), { + onNone: () => Effect.succeed(refusal(call, noSuchTool(call.name))), + onSome: (tool: Tool) => under(wiring, tool, call), + }); + +/** Every call of one turn, at once. The permits hold back what must not overlap. */ +const runCalls = ( + wiring: Wiring, + calls: readonly ToolCall[] +): Effect.Effect => + Effect.forEach(calls, call => runOne(wiring, call), { concurrency: 'unbounded' }); + +const partOf = (result: ToolResult): PartDraft => ({ + kind: 'toolResult', + body: result.body, + callId: result.callId, + failed: result.failed, +}); + +/** + * The results as one turn, in the order the model asked. + * + * The role is `user`, because a result is something the conversation tells the + * model rather than something the model said. Every shape agrees on that, even + * though each writes it differently. + */ +const resultsTurn = (wiring: Wiring, results: readonly ToolResult[]): Effect.Effect => + makeTurn(wiring.entropy, { + sessionId: wiring.id, + role: 'user', + parts: results.map(partOf), + }); + +/** The calls in a turn, in order, ready to run. */ +const callsIn = (turn: Turn): readonly ToolCall[] => + turn.parts + .filter(part => part.kind === 'toolCall') + .map(part => ({ id: part.callId, name: part.name, arguments: part.body })); + +export { callsIn, resultsTurn, runCalls }; diff --git a/packages/harness-sdk/src/core/turn.test.ts b/packages/harness-sdk/src/core/turn.test.ts new file mode 100644 index 0000000000..c63d5317c0 --- /dev/null +++ b/packages/harness-sdk/src/core/turn.test.ts @@ -0,0 +1,40 @@ +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { seededEntropy } from '../plugins/entropy/seeded.js'; +import { appendTurn, makeSession } from './session.js'; +import { makeTurn, textOf } from './turn.js'; + +const entropy = seededEntropy(1); + +const run = (effect: Effect.Effect): A => Effect.runSync(effect); + +it('makes a turn that carries its fields and a trn_{ulid} identifier', () => { + const turn = run( + makeTurn(entropy, { + sessionId: 'ses_1', + role: 'user', + parts: [{ kind: 'text', body: 'hello' }], + }) + ); + expect(turn).toMatchObject({ sessionId: 'ses_1', role: 'user' }); + expect(textOf(turn)).toBe('hello'); + expect(turn.id).toMatch(/^trn_[0-9A-HJKMNP-TV-Z]{26}$/); +}); + +it('appends turns in order and leaves the earlier session untouched', () => { + const [session, first, second] = run( + Effect.all([ + makeSession(entropy), + makeTurn(entropy, { sessionId: 'ses_1', role: 'user', parts: [{ kind: 'text', body: 'a' }] }), + makeTurn(entropy, { + sessionId: 'ses_1', + role: 'assistant', + parts: [{ kind: 'text', body: 'b' }], + }), + ]) + ); + const appended = appendTurn(appendTurn(session, first), second); + + expect(appended.turns.map(textOf)).toEqual(['a', 'b']); + expect(session.turns).toEqual([]); +}); diff --git a/packages/harness-sdk/src/core/turn.ts b/packages/harness-sdk/src/core/turn.ts new file mode 100644 index 0000000000..c5dcd44da1 --- /dev/null +++ b/packages/harness-sdk/src/core/turn.ts @@ -0,0 +1,139 @@ +import { Effect } from 'effect'; +import type { EntropySourceService } from './entropy.js'; +import { makeId } from './id.js'; + +/** + * One piece of a turn. A turn is a list of these, in the order they arrived. + * + * Every kind carries exactly one payload, so `body` is the only field that + * holds content and a part is one flat row. An image keeps its bytes as base64 + * rather than as a blob: base64 is what every gateway shape wants on the wire, + * so storing it that way costs a third more space and saves encoding the image + * again on every single request. + */ +type TurnPart = + | { readonly id: string; readonly kind: 'text'; readonly body: string } + /** Everything before it, in one part. See `sinceSummary`. */ + | { readonly id: string; readonly kind: 'summary'; readonly body: string } + | { + readonly id: string; + readonly kind: 'reasoning'; + readonly body: string; + /** + * What the provider issued with the thinking, and reads back to know the + * thinking is its own. It is opaque: nothing here parses it or builds + * one. A reasoning part without it cannot be replayed, so the shape that + * renders the prompt leaves it out. + */ + readonly signature?: string; + } + /** + * Thinking the provider encrypted rather than showed. `body` is its opaque + * bytes, not words: it is a kind of its own so nothing renders it as text. + */ + | { readonly id: string; readonly kind: 'redacted'; readonly body: string } + | { + readonly id: string; + readonly kind: 'image'; + readonly body: string; + /** The media type, such as `image/png`. */ + readonly media: string; + } + /** + * A tool the model asked for. `body` is the arguments, as the JSON text the + * model wrote: nothing here parses it, because only the tool knows what shape + * it should be. + */ + | { + readonly id: string; + readonly kind: 'toolCall'; + readonly body: string; + /** What the provider called the call. The result names the same one. */ + readonly callId: string; + readonly name: string; + } + /** + * What a tool gave back. Every shape refuses a call with no result, so a + * stored call without one is a session that can never be continued: the two + * are written together or neither is. + */ + | { + readonly id: string; + readonly kind: 'toolResult'; + readonly body: string; + readonly callId: string; + /** True when the tool did not do what it was asked. */ + readonly failed: boolean; + }; + +/** A part before it has an identifier. */ +type PartDraft = + | { readonly kind: 'text'; readonly body: string } + | { readonly kind: 'summary'; readonly body: string } + | { readonly kind: 'reasoning'; readonly body: string; readonly signature?: string } + | { readonly kind: 'redacted'; readonly body: string } + | { readonly kind: 'image'; readonly body: string; readonly media: string } + | { + readonly kind: 'toolCall'; + readonly body: string; + readonly callId: string; + readonly name: string; + } + | { + readonly kind: 'toolResult'; + readonly body: string; + readonly callId: string; + readonly failed: boolean; + }; + +/** + * One turn of a conversation. Both identifiers are monotonic ULIDs, so each is + * both the primary key and the sort order; a separate timestamp column would + * repeat what the identifier already holds. + */ +interface Turn { + readonly id: string; + readonly sessionId: string; + readonly role: TurnRole; + readonly parts: readonly TurnPart[]; +} + +type TurnRole = 'user' | 'assistant'; + +const turnPrefix = 'trn'; +const partPrefix = 'prt'; + +/** What a turn is made of, before it has an identifier. */ +interface TurnDraft { + readonly sessionId: string; + readonly role: TurnRole; + readonly parts: readonly PartDraft[]; +} + +const makePart = (entropy: EntropySourceService, draft: PartDraft): Effect.Effect => + Effect.map(makeId(entropy, partPrefix), id => ({ id, ...draft })); + +const makeTurn = (entropy: EntropySourceService, draft: TurnDraft): Effect.Effect => + Effect.all({ + id: makeId(entropy, turnPrefix), + parts: Effect.forEach(draft.parts, part => makePart(entropy, part)), + }).pipe( + Effect.map(({ id, parts }) => ({ id, sessionId: draft.sessionId, role: draft.role, parts })) + ); + +/** The plain text of a turn, which is what a caller who sends no image writes. */ +const textOf = (turn: Turn): string => + turn.parts + .filter(part => part.kind === 'text') + .map(part => part.body) + .join(''); + +/** A part without its identifier, so a copy of it becomes a part of its own. */ +const draftOf = ({ id: _id, ...draft }: TurnPart): PartDraft => draft; + +/** What a caller means by a bare string: one turn of one text part. */ +const partsOf = (input: string | readonly PartDraft[]): readonly PartDraft[] => + typeof input === 'string' ? [{ kind: 'text', body: input }] : input; + +export type { PartDraft, Turn, TurnDraft, TurnPart, TurnRole }; +export { draftOf, makePart, makeTurn, partsOf, textOf }; diff --git a/packages/harness-sdk/src/core/usage.ts b/packages/harness-sdk/src/core/usage.ts new file mode 100644 index 0000000000..744dbb8380 --- /dev/null +++ b/packages/harness-sdk/src/core/usage.ts @@ -0,0 +1,41 @@ +import type { ModelUsage } from './model.js'; + +const add = (held: ModelUsage, part: ModelUsage): ModelUsage => ({ + inputTokens: held.inputTokens + part.inputTokens, + outputTokens: held.outputTokens + part.outputTokens, + cacheReadTokens: held.cacheReadTokens + part.cacheReadTokens, + cacheWriteTokens: held.cacheWriteTokens + part.cacheWriteTokens, +}); + +/** + * Folds one frame's counts into what this reply has reported so far. + * + * Every shape reports the counts of the whole reply in each frame, not the + * counts of that frame. So a count only ever rises, and a later zero says + * nothing rather than correcting an earlier number. Overwriting instead of + * raising loses the input counts to any provider that echoes zeros in its + * last frame, which reads as a cache hit ratio of exactly zero. + */ +const raise = (held: ModelUsage, part: Partial): ModelUsage => ({ + inputTokens: Math.max(held.inputTokens, part.inputTokens ?? 0), + outputTokens: Math.max(held.outputTokens, part.outputTokens ?? 0), + cacheReadTokens: Math.max(held.cacheReadTokens, part.cacheReadTokens ?? 0), + cacheWriteTokens: Math.max(held.cacheWriteTokens, part.cacheWriteTokens ?? 0), +}); + +/** + * The share of the input that came from the cache. Returns zero when nothing has + * been read yet. + * + * There is no floor to hold this to, and one used to be written here. The + * number is partly the provider's: the same prompt through the same code read + * 0.79 on one run and 0.99 on the next, and the model run spans 0.28 to 0.9997 + * across providers. A live run asserts `cacheReadTokens > 0` instead — see + * "Many models, a longer conversation" in AGENTS.md. + */ +const hitRatio = (usage: ModelUsage): number => { + const total = usage.cacheReadTokens + usage.inputTokens; + return total === 0 ? 0 : usage.cacheReadTokens / total; +}; + +export { add, hitRatio, raise }; diff --git a/packages/harness-sdk/src/core/wait-flag.test.ts b/packages/harness-sdk/src/core/wait-flag.test.ts new file mode 100644 index 0000000000..a32388e6ea --- /dev/null +++ b/packages/harness-sdk/src/core/wait-flag.test.ts @@ -0,0 +1,176 @@ +import { Duration, Effect, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import type { ModelEvent, ModelRequest } from './model.js'; +import { runWith } from './session-fixture.js'; +import { type Tool, ToolRegistry } from './tool.js'; + +/** + * The model saying, per call, whether it wants to wait. + * + * Every tool names a default, because a tool that always outlives a request + * knows that about itself. The model knows something the tool cannot: whether + * this call is the one it is stuck on. So it answers `wait` on the call, and its + * answer wins in both directions — it can give up on a call the tool expected it + * to wait for, and wait for one the tool expected it to abandon. + * + * Waiting costs nothing at the provider. Tools run between requests, never + * during one, so what a waiting model spends is the caller's own stream, and the + * caller can cut that short whenever it likes with `session.background`. + */ + +/* Long enough that nothing here reaches it. Every decision in this file is the + model's, never the clock's. */ +const options = { + system: 'sys', + model: 'claude-opus-5', + maxTokens: 1024, + tools: ['look'], + inlineFor: Duration.minutes(5), +}; + +const asking = (wait?: boolean) => ({ + id: 'tc_1', + name: 'look', + arguments: JSON.stringify(wait === undefined ? { path: 'x' } : { path: 'x', wait }), +}); + +/** What the tool says about waiting, in the two ways a tool can say it. */ +interface Says { + readonly inlineFor?: Duration.DurationInput; + readonly wait?: boolean; +} + +/** A tool that answers at once, so anything unanswered is a decision, not a race. */ +const tool = (says: Says = {}, seen?: string[]): Layer.Layer => + Layer.succeed(ToolRegistry, { + tools: [ + { + definition: { + name: 'look', + description: 'look', + parameters: { type: 'object', properties: { path: { type: 'string' } } }, + }, + ...says, + run: (call): Effect.Effect => + Effect.sync(() => { + seen?.push(call.arguments); + return 'nine'; + }), + } satisfies Tool, + ], + }); + +/** What the model was told about the call, which is the whole question here. */ +const bodyOf = (events: Iterable): string => { + const found = [...events].find(event => event.kind === 'toolResult'); + return found === undefined ? '' : found.result.body; +}; + +/** What the model was told about waiting for this tool, as the schema says it. */ +const shown = (held: { readonly calls: readonly ModelRequest[] }): unknown => + held.calls[0]?.tools?.[0]?.parameters.properties?.['wait']; + +const answered = (tools: Layer.Layer, wait?: boolean) => + runWith({ + options, + tools, + replies: [{ deltas: [], calls: [asking(wait)], stop: 'tools' }, { deltas: ['nine files'] }], + use: session => Stream.runCollect(session.ask('how many files')), + }); + +it('gives up on a call the tool expected it to wait for', async () => { + /* No deadline of its own, so the session's five minutes would apply. */ + const { value } = await answered(tool(), false); + + expect(bodyOf(value)).toContain('still running'); +}); + +it('waits for a call the tool expected it to abandon', async () => { + const { value } = await answered(tool({ inlineFor: Duration.zero }), true); + + expect(bodyOf(value)).toBe('nine'); +}); + +it('leaves the tool to decide when the model does not answer', async () => { + const { value } = await answered(tool({ inlineFor: Duration.zero })); + + expect(bodyOf(value)).toContain('still running'); +}); + +it('offers the field to the model and keeps it away from the tool', async () => { + const seen: string[] = []; + const { calls } = await answered(tool({ inlineFor: Duration.zero }, seen), true); + + expect(calls[0]?.tools?.[0]?.parameters.properties).toMatchObject({ + path: { type: 'string' }, + wait: { type: 'boolean' }, + }); + /* The tool is handed its own arguments and nothing else. A tool that + validates them strictly would refuse a key its author never wrote. */ + expect(seen).toEqual(['{"path":"x"}']); +}); + +it('leaves arguments that are not an object for the tool to complain about', async () => { + const seen: string[] = []; + const { value } = await runWith({ + options, + tools: tool({ inlineFor: Duration.zero }, seen), + replies: [ + { deltas: [], calls: [{ id: 'tc_1', name: 'look', arguments: 'not json' }], stop: 'tools' }, + { deltas: ['nine files'] }, + ], + use: session => Stream.runCollect(session.ask('how many files')), + }); + + /* Untouched: a runner that rewrote it would change the words of the + complaint the tool is about to make. */ + expect(seen).toEqual(['not json']); + expect(bodyOf(value)).toContain('still running'); +}); + +it('survives arguments that parse to nothing at all', async () => { + const seen: string[] = []; + const { value } = await runWith({ + options, + tools: tool({ inlineFor: Duration.zero }, seen), + replies: [ + { deltas: [], calls: [{ id: 'tc_1', name: 'look', arguments: 'null' }], stop: 'tools' }, + { deltas: ['nine files'] }, + ], + use: session => Stream.runCollect(session.ask('how many files')), + }); + + /* `null` parses, and taking a field off it throws. The session would die of a + defect on a call the model was entitled to make. */ + expect(seen).toEqual(['null']); + expect(bodyOf(value)).toContain('still running'); +}); + +it('takes the tool at its word when the model says nothing', async () => { + const { value } = await answered(tool({ wait: false })); + + /* The tool named no deadline, so the session's five minutes would hold the + model here. `wait: false` is the tool saying nobody waits for this one. */ + expect(bodyOf(value)).toContain('still running'); +}); + +it('lets the model wait for a tool that says nobody does', async () => { + const { value } = await answered(tool({ wait: false }), true); + + expect(bodyOf(value)).toBe('nine'); +}); + +it('tells the model what each tool expects, in the schema', async () => { + const waits = await answered(tool({ wait: true })); + const goes = await answered(tool({ wait: false })); + const zero = await answered(tool({ inlineFor: Duration.zero })); + const plain = await answered(tool()); + + /* A tool that said so, either way. And one that said nothing: the deadline + answers for it, because a tool nobody waits any time for is a tool nobody + waits for. */ + expect(shown(waits)).toMatchObject({ default: true }); + expect(shown(goes)).toMatchObject({ default: false }); + expect(shown(zero)).toMatchObject({ default: false }); + expect(shown(plain)).toMatchObject({ default: true }); +}); diff --git a/packages/harness-sdk/src/core/waiting.ts b/packages/harness-sdk/src/core/waiting.ts new file mode 100644 index 0000000000..63a5b71afb --- /dev/null +++ b/packages/harness-sdk/src/core/waiting.ts @@ -0,0 +1,162 @@ +import { Clock, Deferred, Duration, Effect, type Exit, Fiber, Option, Ref } from 'effect'; +import { defaultInlineFor, type Tool, type ToolCall, type ToolResult, waitField } from './tool.js'; +import type { Running, Wiring } from './wiring.js'; + +/** + * How long the model waits for one call, and who gets to decide. + * + * Four parties have a say, and the most specific one wins. The tool author + * names a default with `inlineFor`, because a tool that always outlives a + * request knows that about itself. The session names a fallback. The model + * itself answers `wait` on the call, in either direction. And a caller watching + * a call that is already running can stop the waiting at any moment. + * + * None of them changes what the tool does. Every call is forked into the + * session's scope and runs to the end whatever is decided here: what is under + * discussion is only whether the model sits and waits for it, or is handed a + * note and told the answer later. + */ + +/** + * What the model asked for about waiting, and the call without it. + * + * The field is the harness's, so the tool never sees it: a tool that validates + * its own arguments strictly would refuse a key it does not know, and a tool + * author should not have to know this exists. Arguments that are not an object + * pass through untouched — malformed JSON is the tool's to complain about, and + * a runner that rewrote it would change the words of the complaint. + */ +const wanted = (call: ToolCall): { readonly wait?: boolean; readonly call: ToolCall } => { + const held: unknown = tried(() => JSON.parse(call.arguments)); + if (!isFields(held)) { + return { call }; + } + const { [waitField]: wait, ...rest } = held; + if (typeof wait !== 'boolean') { + return { call }; + } + return { wait, call: { ...call, arguments: JSON.stringify(rest) } }; +}; + +/** Arguments the field can be taken out of: a JSON object and nothing else. */ +const isFields = (held: unknown): held is Record => + typeof held === 'object' && held !== null && !Array.isArray(held); + +/** `undefined` rather than a throw, so a malformed call reaches its own tool. */ +const tried = (parse: () => unknown): unknown => { + try { + return parse(); + } catch { + return undefined; + } +}; + +/** + * How long the model waits for one call. + * + * Three answers, and the most specific one wins. The model's own `wait` beats + * everything, in both directions: it may give up on a call the tool expected it + * to wait for, and it may wait for one the tool expected it to abandon — and + * when it waits it waits under the session's limit, never the tool's zero. The + * tool's own `wait` decides when the model said nothing. And when neither said + * anything the deadline decides, as it always did. + * + * Waiting costs nothing at the provider: tools run between requests, not during + * one. What it spends is the caller's own stream, and the caller can cut that + * short at any time with `session.background`. + */ +const waitFor = (wiring: Wiring, tool: Tool, asked?: boolean): Duration.DurationInput => { + if ((asked ?? tool.wait) === false) { + return Duration.zero; + } + return asked === true + ? (wiring.inlineFor ?? defaultInlineFor) + : (tool.inlineFor ?? wiring.inlineFor ?? defaultInlineFor); +}; + +/** One call the model is waiting on, as a caller reads it. */ +interface RunningCall { + readonly id: string; + readonly name: string; + readonly since: number; +} + +const shown = (one: Running): RunningCall => ({ + id: one.call.id, + name: one.call.name, + since: one.since, +}); + +/** What the model is waiting on now, in the order the calls started. */ +const runningIn = (wiring: Wiring): Effect.Effect => + Effect.map(Ref.get(wiring.running), held => + [...held.values()].map(shown).sort((one, other) => one.since - other.since) + ); + +/** + * Stops the model waiting for one call, now. True when it was still waiting. + * + * False means the call has already been answered, has already gone to the + * background, or was never here. None of those is an error: a person pressing + * the key as the answer lands is ordinary. + * + * The same call serves a person and an agent. Which of them decided is the + * caller's business, and the session does not need to know. + */ +const backgroundNow = (wiring: Wiring, callId: string): Effect.Effect => + Effect.flatMap(Ref.get(wiring.running), held => { + const one = held.get(callId); + /* `Deferred.succeed` answers false when it was already completed, which is + a call somebody sent away twice. Both facts read the same to a caller: + it is no longer waiting on this one. */ + return one === undefined ? Effect.succeed(false) : Deferred.succeed(one.release, true); + }); + +/** Holds the call in `running` for as long as the model is waiting on it. */ +const whileWaiting = ( + wiring: Wiring, + one: Omit, + wait: Effect.Effect +): Effect.Effect => + Effect.acquireUseRelease( + Effect.flatMap(Clock.currentTimeMillis, since => + Ref.update(wiring.running, held => new Map(held).set(one.call.id, { ...one, since })) + ), + () => wait, + () => + Ref.update(wiring.running, held => { + const left = new Map(held); + left.delete(one.call.id); + return left; + }) + ); + +/** + * Waits for the call, unless the deadline passes or somebody sends it away. + * + * `None` means the model stops waiting: the answer is not here, and the call + * carries on without it. + */ +const waited = ( + fiber: Fiber.RuntimeFiber, + release: Deferred.Deferred, + wait: Duration.Duration +): Effect.Effect>> => + Duration.isZero(wait) + ? Effect.succeed(Option.none()) + : Effect.map( + Effect.timeoutOption( + Effect.raceFirst( + Effect.map( + Fiber.await(fiber), + (exit): Option.Option> => Option.some(exit) + ), + Effect.as(Deferred.await(release), Option.none>()) + ), + wait + ), + Option.flatten + ); + +export type { RunningCall }; +export { backgroundNow, runningIn, waited, waitFor, wanted, whileWaiting }; diff --git a/packages/harness-sdk/src/core/wake.test.ts b/packages/harness-sdk/src/core/wake.test.ts new file mode 100644 index 0000000000..8c4bfe228a --- /dev/null +++ b/packages/harness-sdk/src/core/wake.test.ts @@ -0,0 +1,41 @@ +import { Effect, Queue } from 'effect'; +import { expect, it } from 'vitest'; +import { seededEntropy } from '../plugins/entropy/seeded.js'; +import { cancelQueued, enqueueMessage, makePending, wake } from './queue.js'; + +/** + * The driver waits on one token per entry that joined the line. A round it gives + * up on took nothing out of the line, but the token that pointed at it is spent, + * so somebody has to ring the bell again — otherwise the entries wait on + * whatever joins next, which for a caller's last message is forever. + */ + +it('rings the bell again for a line that still holds something', async () => { + const taken = await Effect.runPromise( + Effect.gen(function* () { + const pending = yield* makePending(seededEntropy(1)); + yield* enqueueMessage(pending, 'still waiting', {}); + /* The driver's take, spending the token the message brought with it. */ + yield* Queue.take(pending.arrived); + yield* wake(pending); + return yield* Queue.take(pending.arrived); + }) + ); + + expect(taken).toMatch(/^que_/); +}); + +it('rings no bell for a line that emptied while the driver waited', async () => { + const woken = await Effect.runPromise( + Effect.gen(function* () { + const pending = yield* makePending(seededEntropy(1)); + const id = yield* enqueueMessage(pending, 'cancelled', {}); + yield* Queue.take(pending.arrived); + yield* cancelQueued(pending, id); + yield* wake(pending); + return yield* Queue.size(pending.arrived); + }) + ); + + expect(woken).toBe(0); +}); diff --git a/packages/harness-sdk/src/core/wiring.ts b/packages/harness-sdk/src/core/wiring.ts new file mode 100644 index 0000000000..8b563af24d --- /dev/null +++ b/packages/harness-sdk/src/core/wiring.ts @@ -0,0 +1,207 @@ +import { type Deferred, type Duration, Effect, type Option, PubSub, Ref, type Scope } from 'effect'; +import type { SessionBusyError } from './ask.js'; +import { ModelCatalog, type ModelCatalogService } from './catalog.js'; +import { EntropySource, type EntropySourceService } from './entropy.js'; +import { + type Effort, + ModelClient, + type ModelClientService, + type ModelError, + type ModelUsage, + zeroUsage, +} from './model.js'; +import { PromptAssembler, type PromptAssemblerService } from './prompt.js'; +import { type Continued, makePending, type Pending } from './queue.js'; +import type { Session } from './session.js'; +import { onStore, SessionStore, type SessionStoreService, type StoreError } from './storage.js'; +import { resolveTools, type Tool, type ToolCall, type ToolMissingError } from './tool.js'; + +/** + * What a session is opened with. Every value is frozen for the life of the + * session, and for the same reason: the system prompt is the front of the + * cached prefix, a cache belongs to one model, and effort is part of the key. + * Changing any of them mid-session throws the cache away. + * + * A store records these, so a session that is continued later is reopened with + * the same ones rather than with whatever the caller passes the second time. + */ +interface SessionOptions { + readonly system: string; + readonly model: string; + /** + * The default ceiling on one answer. Without one the catalog's output limit + * decides. One question may raise or lower it either way. + */ + readonly maxTokens?: number; + /** How hard the model should think. Frozen: a change invalidates the cache. */ + readonly effort?: Effort; + /** + * The share of the model's context window a session may fill before it + * compacts itself. `0.8` by default. A catalog that names no window for the + * model never compacts, whatever this says. + * + * A share above 1 never compacts, and the session ends when the provider + * refuses the request. A share at or below 0 compacts before every question, + * which costs a summary call each time. Neither is checked: both are what + * the number asks for, and the range is 0 to 1. + */ + readonly compactAt?: number; + /** The ceiling on one summary. 2048 by default. */ + readonly summaryTokens?: number; + /** + * The tools this session offers, by name, in the order the model sees them. + * Frozen for the same reason as the system prompt: they sit in front of every + * message, so a change to the set or to the order moves the whole prefix. + * + * The names are resolved against the `ToolRegistry` when the session opens. A + * name the registry does not hold fails there rather than at the first + * question, and a session that names none never mentions tools at all. + */ + readonly tools?: readonly string[]; + /** + * How long the model waits for a tool before the call goes to the background. + * 30 seconds by default. A tool may name its own, and its own wins. + */ + readonly inlineFor?: Duration.DurationInput; + /** + * How many times one question may go back to the model before the loop stops + * offering tools and asks for an answer in words. 24 by default. + */ + readonly maxRounds?: number; +} + +/** + * Everything one session holds. Every plugin here is already resolved, and so + * is every tool: the options name them and this holds the code behind them. + */ +interface Wiring extends Omit { + readonly id: string; + readonly catalog: ModelCatalogService; + readonly entropy: EntropySourceService; + readonly assembler: PromptAssemblerService; + readonly client: ModelClientService; + readonly store: Option.Option; + readonly state: Ref.Ref; + readonly totals: Ref.Ref; + /** What the last request put in front of the model. Drives compaction. */ + readonly prompted: Ref.Ref; + /** True while a question is streaming. See `SessionBusyError`. */ + readonly busy: Ref.Ref; + /** + * The tools this session offers, resolved once, in the order it named them. + * Empty when it named none, which is the only test anything makes. + */ + readonly tools: readonly Tool[]; + /** + * The calls the model is waiting on right now, by identifier. A call is here + * from the moment it starts until the model stops waiting for it, whether + * that is because it answered, because the deadline passed, or because + * somebody sent it to the background. + */ + readonly running: Ref.Ref>; + /** + * The session's own scope. Work that has to outlive the question that started + * it is forked here: a backgrounded tool, and the thing that drives what it + * eventually says. Closing the session stops all of it. + */ + readonly scope: Scope.Scope; + /** + * What the session has been given to say and has not said yet: the messages a + * caller queued, and the results of tools the model stopped waiting for. See + * `queue.ts` and `background.ts`. + */ + readonly pending: Pending; + /** + * What the session did without being asked, for a caller that wants to show + * it. Values are dropped rather than held when nobody is listening: these are + * for display, and the transcript is the record. + */ + readonly continued: PubSub.PubSub; +} + +/** + * A call the model is still waiting on, and the way to stop it waiting. + * + * The deadline is not the only thing that can end the waiting. A person + * watching a call take too long, or an agent that decides it has waited enough, + * completes `release`, and the model is moved on at once. The work itself is + * untouched either way: it keeps running, and what it says arrives in a round + * of its own. + */ +interface Running { + readonly call: ToolCall; + /** When the call started, from the session's clock. */ + readonly since: number; + readonly release: Deferred.Deferred; +} + +/** + * Why a round the session started on its own did not happen. `SessionBusyError` + * is here because such a round waits for the question in flight, and gives up + * rather than waiting forever on a session that never goes quiet. + */ +type ContinuedError = ModelError | StoreError | SessionBusyError; + +/** + * How many events of unwatched rounds the session keeps before it drops the + * oldest. It is a display buffer, not a log, so it is small and it slides rather + * than blocking the round that is publishing. + * + * The same number is the replay window, so a caller that queues a message and + * only then reads `continued` still sees the answer. Without it the order of + * two lines of a caller's own code would decide whether they see anything at + * all, and the round can start before the subscription does. + */ +const continuedCapacity = 256; + +/** Everything a session needs from its context, whether it is new or resumed. */ +type SessionContext = PromptAssembler | ModelClient | ModelCatalog | EntropySource | Scope.Scope; + +/** + * Bridges to every plugin and resolves each one once, so the handle carries no + * requirement of its own. What each plugin then does is the plugin's decision. + * + * The session is scoped. Closing the scope tells the store to write whatever it + * still holds. + */ +const wiringFor = ( + options: SessionOptions, + session: Session, + /** + * What the session's last request put in front of the model. A new session + * has made none, and a resumed one takes what the store recorded: without it + * a full conversation would go back out whole before anything compacts. + */ + prompted = 0 +): Effect.Effect => + Effect.gen(function* () { + const tools = yield* resolveTools(options.tools ?? []); + const wiring: Wiring = { + ...options, + id: session.id, + entropy: yield* EntropySource, + assembler: yield* PromptAssembler, + client: yield* ModelClient, + catalog: yield* ModelCatalog, + store: yield* Effect.serviceOption(SessionStore), + state: yield* Ref.make(session), + totals: yield* Ref.make(zeroUsage), + prompted: yield* Ref.make(prompted), + busy: yield* Ref.make(false), + tools, + running: yield* Ref.make>(new Map()), + scope: yield* Effect.scope, + pending: yield* makePending(yield* EntropySource), + continued: yield* PubSub.sliding({ + capacity: continuedCapacity, + replay: continuedCapacity, + }), + }; + yield* Effect.addFinalizer(() => + Effect.ignore(onStore(wiring.store, plugin => plugin.flush())) + ); + return wiring; + }); + +export type { ContinuedError, Running, SessionContext, SessionOptions, Wiring }; +export { wiringFor }; diff --git a/packages/harness-sdk/src/index.test.ts b/packages/harness-sdk/src/index.test.ts new file mode 100644 index 0000000000..776fb42bdc --- /dev/null +++ b/packages/harness-sdk/src/index.test.ts @@ -0,0 +1,123 @@ +import { expect, it } from 'vitest'; +import * as sdk from './index.js'; +import * as core from './core/index.js'; + +/** + * What a consumer can reach. + * + * A module left out of a barrel is invisible from outside the package and + * nothing else notices: every test here imports by path, so the whole package + * passes while a consumer cannot call half of it. That has happened twice — + * compaction and the composed layer were both unreachable — so the surface the + * README documents is asserted rather than assumed. + * + * The two store plugins are deliberately absent. Each names a platform, so + * exporting them from the root would pull `node:sqlite` or `expo-sqlite` into + * every bundle. They have subpaths of their own. + * + * So are the conformance checks and the shipped `fetch`. An entry point is what + * a consumer bundles, and neither is run in production: `checkStore` and + * `checkAssembler` belong to a plugin author's test suite, and a caller with a + * `fetch` adapter of their own should not carry this one. + */ + +/** Every layer the README's plugin table names. */ +const layers = [ + 'layerKilo', + 'layerKiloGateway', + 'layerAssembler', + 'layerTableCatalog', + 'layerStaticToken', + 'layerBackoff', + 'layerNoRetry', + 'layerWebCrypto', + 'layerSeededEntropy', +] as const; + +/** What a caller opens, continues, or reads a session with. */ +const functions = [ + 'openSession', + 'continueSession', + 'cloneSession', + 'hitRatio', + 'said', + 'textOf', + 'questionTool', + 'subagentTool', + 'timeTool', + 'todoTool', +] as const; + +/** What has an entry point of its own, and must not be reachable from the root. */ +const elsewhere = ['checkStore', 'checkAssembler', 'webFetch'] as const; + +const tags = [ + 'ModelClient', + 'ModelCatalog', + 'PromptAssembler', + 'SessionStore', + 'TokenSource', + 'RetryPolicy', + 'EntropySource', + 'ToolRegistry', +] as const; + +it('exports every layer a consumer wires', () => { + const missing = layers.filter(name => !(name in sdk)); + + expect(missing).toStrictEqual([]); +}); + +it('exports every call a consumer makes', () => { + const missing = [...functions, ...tags].filter(name => !(name in sdk)); + + expect(missing).toStrictEqual([]); +}); + +it('keeps what has its own entry point out of the root', () => { + const leaked = elsewhere.filter(name => name in sdk); + + expect(leaked).toStrictEqual([]); + /* And they are still reachable. A name in neither place is a name nobody can + call, which is the failure this whole file exists to catch. */ + expect(['checkStore', 'checkAssembler'].filter(name => !(name in core))).toStrictEqual([]); +}); + +it('keeps the core entry point free of plugins', () => { + const plugins = layers.filter(name => name in core); + + expect(plugins).toStrictEqual([]); +}); + +it('keeps the machinery of a session out of the root', () => { + /* The root is what a consumer calls. These run a session from the inside and + are reached through `/core`, so a caller reading `history` is not offered + them. Deleting a name from this list is fine; adding one to the root by + accident is what the test is for. */ + const machinery = [ + 'appendTurn', + 'cancelQueued', + 'compactIfFull', + 'definitionsOf', + 'draftOf', + 'enqueue', + 'enqueueMessage', + 'handleOf', + 'makeId', + 'makePending', + 'makePart', + 'makeSession', + 'makeTurn', + 'onStore', + 'promptedOf', + 'resolveTools', + 'sinceSummary', + 'takeRun', + 'toolNamed', + 'wiringFor', + ]; + const leaked = machinery.filter(name => name in sdk); + + expect(leaked).toStrictEqual([]); + expect(machinery.filter(name => !(name in core))).toStrictEqual([]); +}); diff --git a/packages/harness-sdk/src/index.ts b/packages/harness-sdk/src/index.ts new file mode 100644 index 0000000000..39448477f5 --- /dev/null +++ b/packages/harness-sdk/src/index.ts @@ -0,0 +1,60 @@ +/** + * What a consumer imports. + * + * The modules a caller uses whole are re-exported whole. The ones named below + * are not: they hold the machinery a session runs on, and a caller who reads + * `history` has no use for `wiringFor`, `makeId` or `sinceSummary`. Everything + * left out here is still reachable from `@kilocode/harness-sdk/core`, which is + * where a plugin author goes. + * + * No count is written here on purpose. One was, twice, and both were wrong + * within two commits. `src/index.test.ts` is what holds this honest. + * + * `conformance.ts` is not here either, and for a different reason: `checkStore` + * and `checkAssembler` are run by a plugin author in their own test suite, and + * nobody runs them in production. Three hundred lines in the entry every + * consumer imports is three hundred lines every consumer bundles. They live at + * `@kilocode/harness-sdk/testing`, and in `/core` with the rest. + * + * `plugins/fetch` is left out for the same kind of reason, the other way round: + * a caller who brings their own adapter should not carry this one. + */ + +export type { AskOptions } from './core/ask.js'; +export { SessionBusyError } from './core/ask.js'; +export * from './core/catalog.js'; +export * from './core/entropy.js'; +export * from './core/fetch.js'; +export * from './core/model.js'; +export * from './core/prompt.js'; +export type { Answering, Continued, Waiting } from './core/queue.js'; +export * from './core/retry.js'; +export * from './core/resume.js'; +export * from './core/run.js'; +export type { SessionStoreService, StoredSession } from './core/storage.js'; +export { SessionStore, StoreError } from './core/storage.js'; +export * from './core/token.js'; +export type { + JsonSchema, + Tool, + ToolCall, + ToolDefinition, + ToolRegistryService, + ToolResult, +} from './core/tool.js'; +export { ToolFailure, ToolMissingError, ToolRegistry } from './core/tool.js'; +export type { Session } from './core/session.js'; +export type { PartDraft, Turn, TurnPart, TurnRole } from './core/turn.js'; +export { textOf } from './core/turn.js'; +export type { SessionHandle } from './core/handle.js'; +export type { SessionContext, SessionOptions } from './core/wiring.js'; +export { hitRatio } from './core/usage.js'; +export * from './plugins/catalog/table.js'; +export * from './plugins/entropy/seeded.js'; +export * from './plugins/entropy/web-crypto.js'; +export * from './plugins/gateway/index.js'; +export * from './plugins/kilo.js'; +export * from './plugins/prompt/default.js'; +export * from './plugins/retry/backoff.js'; +export * from './plugins/token/static.js'; +export * from './plugins/tools/index.js'; diff --git a/packages/harness-sdk/src/perf.perf.test.ts b/packages/harness-sdk/src/perf.perf.test.ts new file mode 100644 index 0000000000..f79d45cd8f --- /dev/null +++ b/packages/harness-sdk/src/perf.perf.test.ts @@ -0,0 +1,220 @@ +import { Effect, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { layerTableCatalog } from './plugins/catalog/table.js'; +import { messagesWire } from './plugins/gateway/wire/messages.js'; +import { seededEntropy, layerSeededEntropy } from './plugins/entropy/seeded.js'; +import { fakeFetch, type Reply, sse } from './plugins/gateway/fake.js'; +import { testGateway } from './plugins/gateway/test-gateway.js'; +import { assemble, layerAssembler } from './plugins/prompt/default.js'; +import { makeId } from './core/id.js'; +import { openSession } from './core/run.js'; +import { appendTurn, makeSession } from './core/session.js'; +import { makeTurn } from './core/turn.js'; + +/** + * These guard against a regression in order of magnitude, not against noise. + * Every ceiling is roughly five times the number measured on 2026-09-03 (see + * the Performance section of AGENTS.md), so a change that doubles a cost still + * passes and a change that breaks the shape of the work does not. + * + * A timing test that fails on a busy laptop is worse than no timing test, so + * each figure is the median of several runs and the ceilings are generous. + */ +const spin = (reps: number, run: () => void): void => { + for (let index = 0; index < reps; index += 1) { + run(); + } +}; + +const medianMicros = (reps: number, run: () => void): number => { + spin(reps, run); + const taken: number[] = []; + for (let round = 0; round < 5; round += 1) { + const started = performance.now(); + spin(reps, run); + taken.push(((performance.now() - started) * 1000) / reps); + } + return taken.toSorted((a, b) => a - b)[2] ?? 0; +}; + +const entropy = seededEntropy(11); + +const sessionOf = (turns: number) => { + let held = Effect.runSync(makeSession(entropy)); + for (let index = 0; index < turns; index += 1) { + held = appendTurn( + held, + Effect.runSync( + makeTurn(entropy, { + sessionId: held.id, + role: index % 2 === 0 ? 'user' : 'assistant', + parts: [ + { + kind: 'text', + body: `message number ${String(index)} with enough text to weigh something`, + }, + ], + }) + ) + ); + } + return held; +}; + +it('assembles a 200 turn prompt in well under 20 us', () => { + const { turns } = sessionOf(200); + const cost = medianMicros(2000, () => void assemble({ system: 'sys', turns })); + expect(cost).toBeLessThan(20); +}); + +it('assembles in time linear in the turn count, not quadratic', () => { + const small = sessionOf(100).turns; + const large = sessionOf(800).turns; + const smallCost = medianMicros(2000, () => void assemble({ system: 'sys', turns: small })); + const largeCost = medianMicros(500, () => void assemble({ system: 'sys', turns: large })); + + /* Eight times the turns. Linear lands near 8x; quadratic lands near 64x. + Twenty catches the shape change without failing on a noisy machine. */ + expect(largeCost / smallCost).toBeLessThan(20); +}); + +it('builds a whole 200 turn request in well under 250 us', () => { + const { turns } = sessionOf(200); + const system = Array.from({ length: 200 }, (_, index) => `Rule ${String(index)}: be terse.`).join( + '\n' + ); + + /* Everything one question costs before the socket: the prompt, the body of + the shape, and the JSON. Measured at 48 us — 17 to assemble, 6 for the + body, 32 for `JSON.stringify` of 27 kilobytes. The provider's own first + token took between 849 and 4064 ms on the ten model matrix, so this whole + path is a ten-thousandth of the wait and is not worth optimising. The + ceiling is here to catch a rewrite that makes it matter. */ + const cost = medianMicros( + 500, + () => + void JSON.stringify( + messagesWire.toBody({ + prompt: assemble({ system, turns }), + model: 'm', + maxTokens: 1024, + cacheKey: 'ses_1', + }) + ) + ); + expect(cost).toBeLessThan(250); +}); + +it('makes an identifier in under 5 us', () => { + const cost = medianMicros(20_000, () => void Effect.runSync(makeId(entropy, 'trn'))); + expect(cost).toBeLessThan(5); +}); + +it('draws randomness once per millisecond, not once per identifier', () => { + const drawn = { count: 0 }; + const counting = { + bytes: (size: number) => { + drawn.count += 1; + return entropy.bytes(size); + }, + }; + const started = Date.now(); + for (let index = 0; index < 50_000; index += 1) { + Effect.runSync(makeId(counting, 'trn')); + } + const spanned = Date.now() - started + 1; + + /* The monotonic counter refills only when the clock moves, so the cost of + entropy is bounded by wall time: 44 draws for 50000 identifiers when this + was written. A refill per identifier would be 50000, so any bound near the + elapsed milliseconds separates the two by three orders of magnitude. */ + expect(drawn.count).toBeLessThan(spanned * 2 + 20); +}); + +/** Sequential, because concurrent rounds would skew the time being measured. */ +const repeat = async (times: number, run: () => Promise): Promise => { + if (times <= 0) { + return; + } + await run(); + await repeat(times - 1, run); +}; + +/** One whole answer of `tokens` deltas, timed over `rounds`, in microseconds. */ +const perToken = async (tokens: number, rounds: number): Promise => { + const chunks = sse( + { type: 'message_start', message: { usage: { input_tokens: 5 } } }, + ...Array.from({ length: tokens }, () => ({ + type: 'content_block_delta', + delta: { text: 'word ' }, + })), + { type: 'message_delta', usage: { output_tokens: tokens } } + ); + const answer: Reply = { ok: true, status: 200, body: '', chunks }; + + const once = async () => { + const { fetch } = fakeFetch([answer]); + await Effect.runPromise( + Effect.provide( + Effect.scoped( + Effect.flatMap(openSession({ system: 'sys', model: 'm', maxTokens: 64 }), session => + Stream.runDrain(session.ask('hi')) + ) + ), + Layer.mergeAll( + layerAssembler, + layerTableCatalog({}, { apiKinds: ['messages'] }), + layerSeededEntropy(5), + testGateway({ fetch }) + ) + ) + ); + }; + + await once(); + const started = performance.now(); + await repeat(rounds, once); + return ((performance.now() - started) * 1000) / (rounds * tokens); +}; + +it('streams a token for under 90 us end to end', async () => { + /* Measured at 18.1 us per token through the whole session, which is more + than the 7 us of the gateway path alone because this counts the session, + the store hook and the turn recording too. The ceiling catches a rewrite + that adds an order of magnitude, which an accidental await or copy would. */ + expect(await perToken(200, 20)).toBeLessThan(90); +}); + +it('costs no more per token on a long answer than on a short one', async () => { + /* The answer is built up a delta at a time, so a copy of the whole answer + per delta would be quadratic and would only show on a long one. It is not: + the median of five rounds is 13.4 us per token over 200 and 7.1 over 5000, + because the fixed cost of opening the session is spread over more tokens. + The comparison is the guard, not either figure, and the two are 1.9 times + apart. */ + const short = await perToken(200, 20); + const long = await perToken(5000, 4); + + expect(long).toBeLessThan(short); +}); + +it('holds a 2000 turn session in memory linear in the turn count', () => { + const before = sessionOf(1000); + const after = sessionOf(2000); + + /* An append copies the array of turns, so the cost of growing a session is + quadratic in principle. What the number below says is that it does not + matter at any length a context window allows: a hundred appends cost less + than one round trip's first byte by four orders of magnitude. */ + expect(before.turns).toHaveLength(1000); + expect(after.turns).toHaveLength(2000); + + const hundred = before.turns.slice(0, 100); + const growth = medianMicros(200, () => { + let held = sessionOf(0); + for (const turn of hundred) { + held = appendTurn(held, turn); + } + }); + expect(growth).toBeLessThan(500); +}); diff --git a/packages/harness-sdk/src/plugins/catalog/table.test.ts b/packages/harness-sdk/src/plugins/catalog/table.test.ts new file mode 100644 index 0000000000..585b7c8081 --- /dev/null +++ b/packages/harness-sdk/src/plugins/catalog/table.test.ts @@ -0,0 +1,36 @@ +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { ModelCatalog } from '../../core/catalog.js'; +import { layerTableCatalog } from './table.js'; + +const facts = ( + table: Parameters[0], + fallback: Parameters[1], + model: string +) => + Effect.runSync( + Effect.either( + Effect.flatMap(ModelCatalog, catalog => catalog.facts(model)).pipe( + Effect.provide(layerTableCatalog(table, fallback)) + ) + ) + ); + +it('answers from the table for a model it names', () => { + const known = { apiKinds: ['messages'] } as const; + expect(facts({ 'a/b': known }, undefined, 'a/b')).toMatchObject({ _tag: 'Right', right: known }); +}); + +it('answers from the fallback for a model it does not name', () => { + const other = { apiKinds: ['responses'] } as const; + expect(facts({}, other, 'a/b')).toMatchObject({ _tag: 'Right', right: other }); +}); + +it('refuses a model it does not name when there is no fallback', () => { + /* A table meant to be complete must say so rather than guess a shape. The + gateway turns this into a failed call, not a call to the wrong endpoint. */ + expect(facts({}, undefined, 'a/b')).toMatchObject({ + _tag: 'Left', + left: { _tag: 'harness/CatalogError', model: 'a/b' }, + }); +}); diff --git a/packages/harness-sdk/src/plugins/catalog/table.ts b/packages/harness-sdk/src/plugins/catalog/table.ts new file mode 100644 index 0000000000..271ff27bcd --- /dev/null +++ b/packages/harness-sdk/src/plugins/catalog/table.ts @@ -0,0 +1,25 @@ +import { Effect, Layer } from 'effect'; +import { CatalogError, ModelCatalog, type ModelFacts } from '../../core/catalog.js'; + +/** + * A catalog the caller writes down. It costs no request and never surprises, + * and it goes stale the day a provider adds a shape. + * + * `fallback` answers for a model the table does not name. Without one an + * unknown model fails, which is the honest answer when the table is meant to + * be complete. + */ +const layerTableCatalog = ( + table: Readonly>, + fallback?: ModelFacts +): Layer.Layer => + Layer.succeed(ModelCatalog, { + facts: model => { + const known = table[model] ?? fallback; + return known === undefined + ? Effect.fail(new CatalogError({ model, cause: 'the catalog does not name this model' })) + : Effect.succeed(known); + }, + }); + +export { layerTableCatalog }; diff --git a/packages/harness-sdk/src/plugins/conformance.test.ts b/packages/harness-sdk/src/plugins/conformance.test.ts new file mode 100644 index 0000000000..6715355677 --- /dev/null +++ b/packages/harness-sdk/src/plugins/conformance.test.ts @@ -0,0 +1,144 @@ +import { DatabaseSync } from 'node:sqlite'; +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { layerAssembler } from './prompt/default.js'; +import { layerNodeStore } from './store/node.js'; +import { checkAssembler, checkStore } from '../core/conformance.js'; +import { + PromptAssembler, + type PromptAssemblerService, + type PromptMessage, +} from '../core/prompt.js'; +import { SessionStore, type SessionStoreService, StoreError } from '../core/storage.js'; +import type { Turn } from '../core/turn.js'; + +/** A turn a broken store can hand back for a session that has none. */ +const laterTurn: Turn = { + id: 'trn_x', + sessionId: 'ses_x', + role: 'user', + parts: [{ id: 'prt_x', kind: 'text', body: 'not yours' }], +}; + +/** + * The checks a plugin author runs, run against the plugins this package ships + * and against plugins broken on purpose. + * + * Both halves matter. A check that passes the package's own plugins and nothing + * else is a check that says yes to everything, and an author would trust it to + * the day their store silently reordered a turn. + */ + +/** The shipped store, on a database that lives as long as the one check. */ +const nodeStore = (use: (store: SessionStoreService) => Effect.Effect) => + Effect.runPromise( + Effect.scoped( + Effect.provide( + Effect.flatMap(SessionStore, use), + layerNodeStore(new DatabaseSync(':memory:')) + ) + ) + ); + +const assembler = Effect.runSync(Effect.provide(PromptAssembler, layerAssembler)); + +it('passes the store this package ships', async () => { + const wrong = await nodeStore(checkStore); + + expect(wrong).toEqual([]); +}); + +it('passes the assembler this package ships', () => { + expect(checkAssembler(assembler)).toEqual([]); +}); + +it('catches a store that hands the turns back in the wrong order', async () => { + const wrong = await nodeStore(store => + checkStore({ ...store, load: id => Effect.map(store.load(id), turns => turns.toReversed()) }) + ); + + expect(wrong.join('\n')).toContain('in the order it was given them'); +}); + +/** Gives every signature back changed, which is the same as losing it. */ +const bend = (turn: Turn): Turn => ({ + ...turn, + parts: turn.parts.map(part => + part.kind === 'reasoning' ? { ...part, signature: 'sig_other' } : part + ), +}); + +it('catches a store that gives a signature back changed', async () => { + const wrong = await nodeStore(store => + checkStore({ ...store, load: id => Effect.map(store.load(id), turns => turns.map(bend)) }) + ); + + /* A signature that comes back changed cannot be replayed, and nothing but a + check like this notices until the provider refuses a request. */ + expect(wrong.join('\n')).toContain('not the ones append was given'); +}); + +it('catches a store that answers for a session it never heard of', async () => { + const wrong = await nodeStore(store => + checkStore({ + ...store, + load: (id: string) => Effect.succeed([{ ...laterTurn, sessionId: id }]), + }) + ); + + expect(wrong.join('\n')).toContain('never created'); +}); + +it('catches a store that keeps the first prompted count rather than the last', async () => { + const wrong = await nodeStore(store => + checkStore({ ...store, append: exchange => store.append({ ...exchange, prompted: 11 }) }) + ); + + expect(wrong.join('\n')).toContain('prompted count'); +}); + +it('catches a store that refuses a write', async () => { + const wrong = await nodeStore(store => + checkStore({ + ...store, + append: () => Effect.fail(new StoreError({ operation: 'append', cause: 'the disk is full' })), + }) + ); + + expect(wrong.join('\n')).toContain('append refused the call'); +}); + +it('catches an assembler that does not give the same bytes twice', () => { + let count = 0; + const drifting: PromptAssemblerService = { + assemble: input => { + count += 1; + const built = assembler.assemble(input); + return { ...built, system: [{ text: String(count), cache: true }, ...built.system] }; + }, + }; + + expect(checkAssembler(drifting).join('\n')).toContain('different bytes for the same input'); +}); + +it('catches an assembler that rewrites what came before an appended turn', () => { + const rewriting: PromptAssemblerService = { + assemble: input => { + const built = assembler.assemble(input); + /* Numbers each message out of how many there are, which is the shape of + every accidental rewrite: harmless-looking, and it moves every byte + after the first message the moment a turn is added. */ + const numbered = (message: PromptMessage, at: number): PromptMessage => ({ + role: message.role, + cache: message.cache, + parts: [ + { kind: 'text', text: `${String(at)}/${String(built.messages.length)} ` }, + ...message.parts, + ], + }); + return { ...built, messages: built.messages.map(numbered) }; + }, + }; + + expect(checkAssembler(rewriting).join('\n')).toContain('rewrote what came before'); +}); diff --git a/packages/harness-sdk/src/plugins/entropy/seeded.test.ts b/packages/harness-sdk/src/plugins/entropy/seeded.test.ts new file mode 100644 index 0000000000..ca1b62557d --- /dev/null +++ b/packages/harness-sdk/src/plugins/entropy/seeded.test.ts @@ -0,0 +1,21 @@ +import { expect, it } from 'vitest'; +import { seededEntropy } from './seeded.js'; + +it('gives the same bytes for the same seed', () => { + expect([...seededEntropy(42).bytes(32)]).toEqual([...seededEntropy(42).bytes(32)]); +}); + +it('gives different bytes for different seeds', () => { + expect([...seededEntropy(1).bytes(32)]).not.toEqual([...seededEntropy(2).bytes(32)]); +}); + +it('keeps going rather than repeating one byte', () => { + const drawn = new Set(seededEntropy(7).bytes(256)); + /* A generator stuck on one value would still pass a determinism check, and + would then hand every identifier the same random part. */ + expect(drawn.size).toBeGreaterThan(64); +}); + +it('accepts a zero seed without collapsing', () => { + expect(new Set(seededEntropy(0).bytes(64)).size).toBeGreaterThan(16); +}); diff --git a/packages/harness-sdk/src/plugins/entropy/seeded.ts b/packages/harness-sdk/src/plugins/entropy/seeded.ts new file mode 100644 index 0000000000..331baf7786 --- /dev/null +++ b/packages/harness-sdk/src/plugins/entropy/seeded.ts @@ -0,0 +1,33 @@ +import { Layer } from 'effect'; +import { EntropySource, type EntropySourceService } from '../../core/entropy.js'; + +/** + * Randomness from a seed, so a run repeats. This is the second implementation + * that earns `EntropySource` its place as a plugin point: it gives a test the + * same identifiers every time, and it gives a runtime with no `crypto` a way + * to run at all. + * + * It is not for anything that must be unguessable. Identifiers are not + * secrets here — they name a turn, and the ordering is what matters — but do + * not reach for this plugin outside a test or a replay. + */ +const modulus = 4_294_967_296; +const multiplier = 1_664_525; +const increment = 1_013_904_223; + +const seededEntropy = (seed: number): EntropySourceService => { + /* A linear congruential generator. Every product stays under 2^53, so it is + exact in a double and needs no bit twiddling to stay in range. */ + const state = { value: Math.abs(Math.trunc(seed)) % modulus }; + const next = (): number => { + state.value = (state.value * multiplier + increment) % modulus; + // The low bits of an LCG cycle quickly; the high ones do not. + return Math.floor(state.value / 65_536) % 256; + }; + return { bytes: count => Uint8Array.from({ length: count }, next) }; +}; + +const layerSeededEntropy = (seed: number): Layer.Layer => + Layer.sync(EntropySource, () => seededEntropy(seed)); + +export { layerSeededEntropy, seededEntropy }; diff --git a/packages/harness-sdk/src/plugins/entropy/web-crypto.ts b/packages/harness-sdk/src/plugins/entropy/web-crypto.ts new file mode 100644 index 0000000000..c81b391a5f --- /dev/null +++ b/packages/harness-sdk/src/plugins/entropy/web-crypto.ts @@ -0,0 +1,42 @@ +import { Effect, Layer } from 'effect'; +import { EntropyError, EntropySource, type EntropySourceService } from '../../core/entropy.js'; + +/** + * The Web Crypto surface this plugin needs. It is declared here rather than + * taken from the DOM library, so the package still compiles with `"types": []`. + */ +interface WebCrypto { + readonly getRandomValues: (array: Uint8Array) => Uint8Array; +} + +/** What this plugin needs of the global object, and nothing more. */ +interface CryptoHost { + readonly crypto?: WebCrypto | undefined; +} + +const host: CryptoHost = globalThis; + +/** + * Randomness from the global `crypto`, which Node 19 and later, Bun, Deno, + * every browser and every worker runtime all provide. It is the default + * because it is the one source that is the same everywhere it exists. + * + * A React Native release build has no global `crypto` until a polyfill is + * installed, so this layer fails there rather than at the first identifier. + * Install `react-native-get-random-values`, or supply another plugin. + */ +const layerWebCrypto: Layer.Layer = Layer.effect( + EntropySource, + Effect.suspend(() => { + const source = host.crypto; + return source === undefined + ? Effect.fail( + new EntropyError({ cause: 'this runtime has no global crypto.getRandomValues' }) + ) + : Effect.succeed({ + bytes: count => source.getRandomValues(new Uint8Array(count)), + }); + }) +); + +export { layerWebCrypto }; diff --git a/packages/harness-sdk/src/plugins/fetch/web.ts b/packages/harness-sdk/src/plugins/fetch/web.ts new file mode 100644 index 0000000000..1b2b759dd4 --- /dev/null +++ b/packages/harness-sdk/src/plugins/fetch/web.ts @@ -0,0 +1,73 @@ +import type { FetchLike } from '../../core/fetch.js'; + +/** + * The adapter for a runtime that has a WHATWG `fetch`: Node, a browser, a + * Cloudflare Worker, React Native. It is what every caller was writing by hand. + * + * `core/fetch.ts` explains why the core declares its own `fetch` shape and + * never calls a runtime's. That reason holds for the core and not for a plugin, + * which is what this is. Nothing here names a platform: `fetch`, `TextDecoder` + * and the streamed body are web globals, and a runtime that lacks them wants an + * adapter of its own anyway. + * + * The types are declared below rather than pulled from the DOM library, because + * `tsconfig.json` sets `"lib": ["esnext"]` and `"types": []` and that is what + * keeps the package honest about what it depends on. Only the members used are + * named, which is also what makes the signal fit: the core's `AbortLike` is + * structurally what `signal` below asks for, so nothing is cast. + */ + +/** Only what this file reads. A runtime's own types are wider and compatible. */ +interface Body { + readonly [Symbol.asyncIterator]: () => AsyncIterator; +} + +interface Reply { + readonly ok: boolean; + readonly status: number; + readonly text: () => Promise; + readonly body: Body | null; +} + +interface Sent { + readonly method: string; + readonly headers: Record; + readonly body: string; + readonly signal?: unknown; +} + +interface Decoder { + readonly decode: (chunk: Uint8Array, options: { readonly stream: boolean }) => string; +} + +declare const fetch: (url: string, request: Sent) => Promise; +declare const TextDecoder: new () => Decoder; + +const decoded = async function* decoded(body: Body): AsyncIterable { + const decoder = new TextDecoder(); + for await (const chunk of body) { + yield decoder.decode(chunk, { stream: true }); + } +}; + +const webFetch: FetchLike = async (url, request) => { + const response = await fetch(url, { + method: request.method, + headers: { ...request.headers }, + body: request.body, + /* The runtime stops the call when the caller stops listening. Dropping this + would leave a cancelled call still running, and still being charged for, + on the provider. */ + ...(request.signal === undefined ? {} : { signal: request.signal }), + }); + /* Named once, so the narrowing holds inside the closure below. */ + const { body } = response; + return { + ok: response.ok, + status: response.status, + text: () => response.text(), + ...(body === null ? {} : { stream: () => decoded(body) }), + }; +}; + +export { webFetch }; diff --git a/packages/harness-sdk/src/plugins/gateway/api-kind.ts b/packages/harness-sdk/src/plugins/gateway/api-kind.ts new file mode 100644 index 0000000000..e6d48bfe50 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/api-kind.ts @@ -0,0 +1,22 @@ +import type { ApiKind } from '../../core/catalog.js'; + +/** + * Ranked by what each shape lets a caller control, best first. `messages` takes + * an explicit cache breakpoint and takes its thinking back signed. `responses` + * names a cache key and takes its thinking back encrypted. A completion + * controls the cache not at all and can replay no thinking, because the + * providers relayed through it report it under two names and take neither back. + * + * The cache half of that order is not what holds the hit ratio above 95 + * percent. Measured on 2026-09-04, this gateway places its own breakpoints and + * every shape cached the same prefix identically with what this package sends + * and without it. What holds the ratio is the prefix never moving. See + * AGENTS.md, "The kilo gateway". + */ +const ranked: readonly ApiKind[] = ['messages', 'responses', 'chat_completions']; + +/** Picks the best kind a model speaks. Returns undefined when it speaks none. */ +const pickKind = (supported: readonly ApiKind[]): ApiKind | undefined => + ranked.find(kind => supported.includes(kind)); + +export { pickKind }; diff --git a/packages/harness-sdk/src/plugins/gateway/cancel.test.ts b/packages/harness-sdk/src/plugins/gateway/cancel.test.ts new file mode 100644 index 0000000000..d693aa6a9b --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/cancel.test.ts @@ -0,0 +1,88 @@ +import { Chunk, type Duration, Effect, Fiber, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import type { AbortLike, FetchLike } from '../../core/fetch.js'; +import { ModelClient } from '../../core/model.js'; +import { sampleRequest } from './fake.js'; +import { testGateway } from './test-gateway.js'; + +const frame = 'data: {"delta":{"text":"and"}}\n\n'; + +const pause = (): Promise => Effect.runPromise(Effect.sleep('5 millis')); + +/** + * A model that never stops talking. It is the case cancellation exists for: a + * caller who walks away from a long answer must stop the generation, or the + * provider keeps producing it and keeps charging for it. + */ +const endless = async function* endless(): AsyncIterable { + yield frame; + await pause(); + yield* endless(); +}; + +const once = async function* once(): AsyncIterable { + yield frame; +}; + +/** Records the signal each call was handed, so a test can read it afterwards. */ +const watched = ( + body: () => AsyncIterable +): { readonly fetch: FetchLike; readonly signals: (AbortLike | undefined)[] } => { + const signals: (AbortLike | undefined)[] = []; + const fetch: FetchLike = (_url, request) => { + signals.push(request.signal); + return Promise.resolve({ + ok: true, + status: 200, + text: () => Promise.resolve('{}'), + stream: body, + }); + }; + return { fetch, signals }; +}; + +const interrupted = (fetch: FetchLike, after: Duration.DurationInput): Promise => + Effect.runPromise( + Effect.gen(function* () { + const client = yield* ModelClient; + const reading = yield* Effect.fork(Stream.runDrain(client.stream(sampleRequest()))); + yield* Effect.sleep(after); + yield* Fiber.interrupt(reading); + }).pipe(Effect.provide(testGateway({ fetch }))) + ); + +it('stops a stream the caller has walked away from', async () => { + const { fetch, signals } = watched(endless); + + await interrupted(fetch, '50 millis'); + + /* The handle is scoped to the stream, not to the request. A streamed call + resolves as soon as the headers arrive and produces for a long time after, + so a handle released when the request resolved would cancel nothing. */ + expect(signals).toHaveLength(1); + expect(signals[0]?.aborted).toBeTruthy(); +}); + +it('leaves the signal alone while the caller is still reading', async () => { + const { fetch, signals } = watched(endless); + const reading = interrupted(fetch, '200 millis'); + + await Effect.runPromise(Effect.sleep('40 millis')); + expect(signals[0]?.aborted).toBeFalsy(); + + await reading; + expect(signals[0]?.aborted).toBeTruthy(); +}); + +it('gives back every event of a stream that ends on its own', async () => { + const { fetch, signals } = watched(once); + + const events = await Effect.runPromise( + Effect.flatMap(ModelClient, client => Stream.runCollect(client.stream(sampleRequest()))).pipe( + Effect.provide(testGateway({ fetch })) + ) + ); + + expect(Chunk.toReadonlyArray(events).map(event => event.kind)).toEqual(['delta', 'done']); + expect(signals[0]?.aborted).toBeTruthy(); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/effort.test.ts b/packages/harness-sdk/src/plugins/gateway/effort.test.ts new file mode 100644 index 0000000000..ed3446625e --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/effort.test.ts @@ -0,0 +1,34 @@ +import { Effect, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import type { ApiKind } from '../../core/catalog.js'; +import { fakeFetch, type Reply, sampleRequest } from './fake.js'; +import { testGateway } from './test-gateway.js'; +import { ModelClient } from '../../core/model.js'; + +/** These tests read the request, so the reply only has to arrive. */ +const reply: Reply = { ok: true, status: 200, body: '', chunks: [] }; + +const bodyOf = async (kinds: readonly ApiKind[], effort: 'low' | 'high') => { + const { calls, fetch } = fakeFetch([reply]); + await ModelClient.pipe( + Effect.map(client => client.stream({ ...sampleRequest(), effort })), + Stream.unwrap, + Stream.runDrain, + Effect.either, + Effect.provide(testGateway({ fetch, kinds })), + Effect.runPromise + ); + return JSON.parse(calls[0]?.request.body ?? '') as unknown; +}; + +it('names the effort as output_config on the messages shape', async () => { + expect(await bodyOf(['messages'], 'high')).toMatchObject({ output_config: { effort: 'high' } }); +}); + +it('names the effort as reasoning on the responses shape', async () => { + expect(await bodyOf(['responses'], 'low')).toMatchObject({ reasoning: { effort: 'low' } }); +}); + +it('names the effort as reasoning on the completions shape', async () => { + expect(await bodyOf(['chat_completions'], 'low')).toMatchObject({ reasoning: { effort: 'low' } }); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/fake.ts b/packages/harness-sdk/src/plugins/gateway/fake.ts new file mode 100644 index 0000000000..6464490dee --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/fake.ts @@ -0,0 +1,65 @@ +import type { FetchLike, HttpRequest } from '../../core/fetch.js'; +import { assemble } from '../prompt/default.js'; +import type { ModelRequest } from '../../core/model.js'; +import type { Turn } from '../../core/turn.js'; + +interface Call { + readonly url: string; + readonly request: HttpRequest; +} + +interface Reply { + readonly ok: boolean; + readonly status: number; + readonly body: string; + readonly chunks?: readonly string[]; +} + +const notFound: Reply = { ok: false, status: 404, body: 'no reply configured' }; + +const turn = (role: Turn['role'], content: string): Turn => ({ + id: `trn_${content}`, + sessionId: 'ses_1', + role, + parts: [{ id: `prt_${content}`, kind: 'text', body: content }], +}); + +/** A request over a two turn session, used by every gateway test. */ +const sampleRequest = (): ModelRequest => ({ + prompt: assemble({ + system: 'sys', + turns: [turn('user', 'a'), turn('assistant', 'b')], + }), + model: 'claude-opus-5', + maxTokens: 1024, + cacheKey: 'ses_1', +}); + +/** One server-sent event per frame, the way every shape sends them. */ +const sse = (...events: readonly unknown[]): readonly string[] => + events.map(event => `data: ${JSON.stringify(event)}\n\n`); + +const toAsync = async function* toAsync(chunks: readonly string[]): AsyncIterable { + for (const chunk of chunks) { + yield chunk; + } +}; + +/** Answers each call with the next reply and records what was sent. */ +const fakeFetch = (replies: readonly Reply[]): { calls: Call[]; fetch: FetchLike } => { + const calls: Call[] = []; + const fetch: FetchLike = (url, request) => { + calls.push({ url, request }); + const reply = replies[Math.min(calls.length - 1, replies.length - 1)] ?? notFound; + return Promise.resolve({ + ok: reply.ok, + status: reply.status, + text: () => Promise.resolve(reply.body), + ...(reply.chunks === undefined ? {} : { stream: () => toAsync(reply.chunks ?? []) }), + }); + }; + return { calls, fetch }; +}; + +export type { Reply }; +export { fakeFetch, sampleRequest, sse, toAsync }; diff --git a/packages/harness-sdk/src/plugins/gateway/gateway.perf.test.ts b/packages/harness-sdk/src/plugins/gateway/gateway.perf.test.ts new file mode 100644 index 0000000000..482174c756 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/gateway.perf.test.ts @@ -0,0 +1,123 @@ +import { Effect, Stream } from 'effect'; +import { afterAll, expect, it } from 'vitest'; +import { ModelClient } from '../../core/model.js'; +import { fakeFetch, type Reply, sampleRequest, sse } from './fake.js'; +import { testGateway } from './test-gateway.js'; + +/** + * What one streamed event costs the gateway, in CPU rather than in wall clock. + * + * Wall clock on this path measures Effect's stream runtime and not this + * package: the work itself — read a frame, parse it, ask the wire three + * questions — is 0.32 us of the 7.6 the whole gateway takes, measured + * 2026-09-04. Nothing worth doing lives in that gap, and a ceiling on it would + * only ever fail because a dependency changed. + * + * So the guards here are the two things a caller actually feels: how much CPU a + * long answer burns, and whether the cost of one event grows with the length of + * the answer. Both catch a rewrite that changes the shape of the work, which is + * what a ceiling is for. + */ + +/** Sequential, because concurrent streams would skew the time being measured. */ +const repeat = async (times: number, run: () => Promise): Promise => { + if (times <= 0) { + return; + } + await run(); + await repeat(times - 1, run); +}; + +/** What one run measured, so a reader sees the number and not only the ceiling. */ +const measured: string[] = []; + +afterAll(() => { + for (const line of measured) { + process.stdout.write(`${line}\n`); + } +}); + +const drained = (chunks: readonly string[]) => { + const answer: Reply = { ok: true, status: 200, body: '', chunks }; + const { fetch } = fakeFetch([answer]); + return Effect.runPromise( + Effect.provide( + Effect.flatMap(ModelClient, client => Stream.runDrain(client.stream(sampleRequest()))), + testGateway({ fetch }) + ) + ); +}; + +/** CPU actually burned per event, user and system, in microseconds. */ +const busyPer = async (chunks: readonly string[], events: number, rounds: number) => { + await drained(chunks); + const started = process.cpuUsage(); + await repeat(rounds, () => drained(chunks)); + const spent = process.cpuUsage(started); + return (spent.user + spent.system) / (rounds * events); +}; + +const deltas = (events: number): readonly string[] => + sse( + { type: 'message_start', message: { usage: { input_tokens: 5 } } }, + ...Array.from({ length: events }, () => ({ + type: 'content_block_delta', + delta: { text: 'word ' }, + })), + { type: 'message_delta', usage: { output_tokens: events } } + ); + +it('burns under 70 us of CPU per streamed event', async () => { + const busy = await busyPer(deltas(2000), 2000, 10); + measured.push(`gateway: ${busy.toFixed(2)} us of CPU per event`); + + /* Measured at 14.4 us on 2026-09-04, which is high for this path because a + single 2000 event run pays for its own warm-up: the same work over 5000 + events costs 5.0. Five times the higher number, like every other ceiling + here — it catches an order of magnitude and not a busy machine. */ + expect(busy).toBeLessThan(70); +}); + +it('costs no more per event on a long answer than on a short one', async () => { + const short = await busyPer(deltas(200), 200, 30); + const long = await busyPer(deltas(5000), 5000, 4); + measured.push(`gateway: ${short.toFixed(2)} us of CPU at 200 events, ${long.toFixed(2)} at 5000`); + + /* The answer is read a frame at a time and nothing holds the whole of it, so + the long one is cheaper: the fixed cost of the call is spread further. A + rewrite that copied what it had so far would invert this. */ + expect(long).toBeLessThan(short); +}); + +/** + * A tool call arrives in fragments and the gateway holds the open one between + * events. It grows the arguments in place; a fold that rebuilt them on every + * fragment would be quadratic, and only a long argument list would show it. + */ +const fragments = (count: number): readonly string[] => + sse( + { type: 'message_start', message: { usage: { input_tokens: 5 } } }, + { + type: 'content_block_start', + content_block: { type: 'tool_use', id: 'tc_1', name: 'weather' }, + }, + ...Array.from({ length: count }, () => ({ + type: 'content_block_delta', + delta: { type: 'input_json_delta', partial_json: '00000000' }, + })), + { type: 'content_block_stop' }, + { type: 'message_delta', usage: { output_tokens: count } } + ); + +it('costs no more per fragment on a long tool call than on a short one', async () => { + const short = await busyPer(fragments(200), 200, 30); + const long = await busyPer(fragments(4000), 4000, 4); + measured.push( + `gateway: ${short.toFixed(2)} us of CPU per fragment at 200, ${long.toFixed(2)} at 4000` + ); + + /* Linear holds the two close, and the long one is cheaper for the same reason + as above. Quadratic makes it many times the short one, so four times + separates the shapes without failing on noise. */ + expect(long).toBeLessThan(short * 4); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/gateway.test.ts b/packages/harness-sdk/src/plugins/gateway/gateway.test.ts new file mode 100644 index 0000000000..3af2c21f0a --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/gateway.test.ts @@ -0,0 +1,247 @@ +import { Effect, Either, Option, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import type { ApiKind } from '../../core/catalog.js'; +import { fakeFetch, type Reply, sampleRequest, sse, toAsync } from './fake.js'; +import { testGateway } from './test-gateway.js'; +import type { FetchLike } from '../../core/fetch.js'; +import type { OrgContext } from './http.js'; +import { ModelClient, type ModelEvent, type ModelRequest, zeroUsage } from '../../core/model.js'; +import { TokenError, type TokenSourceService } from '../../core/token.js'; + +/** These are the transport's tests: one short answer, told the same way twice. */ +const chunks = sse( + { + type: 'message_start', + message: { + usage: { input_tokens: 7, cache_read_input_tokens: 900, cache_creation_input_tokens: 100 }, + }, + }, + { type: 'content_block_delta', delta: { text: 'hi' } }, + { type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 3 } } +); + +const reply: Reply = { ok: true, status: 200, body: '', chunks }; + +/** What the old non-streamed reply used to be: the answer, the cost, the end. */ +const summary = (events: readonly ModelEvent[]) => { + const last = events.at(-1); + const done = last?.kind === 'done' ? last : undefined; + return { + content: events + .filter(event => event.kind === 'delta') + .map(event => event.text) + .join(''), + usage: done?.usage ?? zeroUsage, + stop: done?.stop ?? 'unknown', + }; +}; + +const call = async (options: { + readonly org?: OrgContext; + readonly kinds?: readonly ApiKind[]; + readonly replies?: readonly Reply[]; + readonly token?: TokenSourceService; + readonly request?: ModelRequest; +}) => { + const { calls, fetch } = fakeFetch(options.replies ?? [reply]); + const result = await ModelClient.pipe( + Effect.map(client => client.stream(options.request ?? sampleRequest())), + Stream.unwrap, + Stream.runCollect, + Effect.map(collected => summary([...collected])), + Effect.either, + Effect.provide( + testGateway({ + baseUrl: 'https://app.kilocode.ai/', + fetch, + retries: 2, + ...(options.org === undefined ? {} : { org: options.org }), + ...(options.kinds === undefined ? {} : { kinds: options.kinds }), + ...(options.token === undefined ? {} : { token: options.token }), + }) + ), + Effect.runPromise + ); + return { calls, result }; +}; + +it('posts to the messages endpoint with a bearer token', async () => { + const { calls } = await call({}); + expect(calls[0]?.url).toBe('https://app.kilocode.ai/api/gateway/v1/messages'); + expect(calls[0]?.request.headers).toEqual({ + 'content-type': 'application/json', + authorization: 'Bearer tok', + 'x-kilo-session': 'ses_1', + }); +}); + +/* The gateway hashes this header into the upstream provider's cache key and + into the seed that keeps one conversation on one provider. A call that sends + none gets neither, so the session id travels on every shape. */ +it('names the session on every shape, so the gateway can key the cache by it', async () => { + const sent = await Promise.all( + (['messages', 'responses', 'chat_completions'] as const).map(kind => call({ kinds: [kind] })) + ); + for (const { calls } of sent) { + expect(calls[0]?.request.headers['x-kilo-session']).toBe('ses_1'); + } +}); + +it('sends no session header for a request that names no session', async () => { + const { cacheKey, ...anonymous } = sampleRequest(); + expect(cacheKey).toBe('ses_1'); + const { calls } = await call({ request: anonymous }); + expect(calls[0]?.request.headers['x-kilo-session']).toBeUndefined(); +}); + +it('names the organization when the context is an organization', async () => { + const { calls } = await call({ org: { kind: 'organization', id: 'org_1' } }); + expect(calls[0]?.request.headers['x-kilocode-organizationid']).toBe('org_1'); +}); + +it('marks a cache breakpoint on the system block and on the last message', async () => { + const { calls } = await call({}); + expect(JSON.parse(calls[0]?.request.body ?? '')).toMatchObject({ + system: [{ cache_control: { type: 'ephemeral' } }], + messages: [ + { content: [{ text: 'a' }] }, + { content: [{ cache_control: { type: 'ephemeral' } }] }, + ], + }); +}); + +it('reads the token counts out of the stream', async () => { + const { result } = await call({}); + expect(Either.getOrThrow(result)).toEqual({ + content: 'hi', + usage: { inputTokens: 7, outputTokens: 3, cacheReadTokens: 900, cacheWriteTokens: 100 }, + stop: 'end', + }); +}); + +it('prefers messages over the other two shapes', async () => { + const { calls } = await call({ kinds: ['chat_completions', 'responses', 'messages'] }); + expect(calls[0]?.url).toMatch(/\/messages$/u); +}); + +it('falls back to the completions shape when a model speaks only that', async () => { + const completion: Reply = { + ok: true, + status: 200, + body: '', + chunks: sse( + { choices: [{ delta: { content: 'hi' } }] }, + { + usage: { + prompt_tokens: 10, + completion_tokens: 3, + prompt_tokens_details: { cached_tokens: 9 }, + }, + } + ), + }; + const { calls, result } = await call({ kinds: ['chat_completions'], replies: [completion] }); + expect(calls[0]?.url).toMatch(/\/chat\/completions$/u); + expect(Either.getOrThrow(result).usage).toEqual({ + inputTokens: 1, + outputTokens: 3, + cacheReadTokens: 9, + cacheWriteTokens: 0, + }); +}); + +it('reports that a model speaks no shape the gateway serves', async () => { + const { calls, result } = await call({ kinds: [] }); + expect(calls).toHaveLength(0); + expect(result).toMatchObject({ left: { reason: 'unsupported' } }); +}); + +it('reports the status when the gateway rejects the call', async () => { + const rejected: Reply = { ok: false, status: 402, body: 'no credit' }; + const { calls, result } = await call({ replies: [rejected] }); + expect(calls).toHaveLength(1); + expect(result).toMatchObject({ left: { reason: 'status', status: 402 } }); +}); + +it('tries again after a rate limit and then succeeds', async () => { + const limited: Reply = { ok: false, status: 429, body: 'slow down' }; + const { calls, result } = await call({ replies: [limited, limited, reply] }); + expect(calls).toHaveLength(3); + expect(Either.getOrThrow(result).content).toBe('hi'); +}); + +it('reports a credential it could not get as a transport failure, and asks again', async () => { + /* The token is read inside the retried effect, so a source that refreshes + gets another chance rather than failing the call. That is why a token + failure is a transport failure: it is the one kind of failure this + package cannot tell from a flaky network. */ + let asked = 0; + const refusing: TokenSourceService = { + /* Suspended, so the count follows the runs. A `get` that does its work + while building the effect is asked once however often the call is + retried, because the retry re-runs the effect it was given. */ + get: () => + Effect.suspend(() => { + asked += 1; + return Effect.fail(new TokenError({ cause: 'no session' })); + }), + }; + const { calls, result } = await call({ token: refusing }); + + expect(asked).toBe(3); + expect(calls).toHaveLength(0); + expect(Either.getLeft(result).pipe(Option.map(error => error.reason))).toStrictEqual( + Option.some('transport') + ); +}); + +it('does not try again on a status the gateway will answer the same way', async () => { + /* A bad request, a bad token or an unknown model answers the same on the + fourth attempt as on the first. Retrying one costs three more requests and + three more waits, and hides the real reason behind the last of them. */ + const refused: Reply = { ok: false, status: 400, body: 'max_tokens is required' }; + const { calls, result } = await call({ replies: [refused, reply, reply, reply] }); + + expect(calls).toHaveLength(1); + expect(Either.getLeft(result).pipe(Option.map(error => error.status))).toStrictEqual( + Option.some(400) + ); +}); + +it('gives up after the retry budget and reports the last status', async () => { + const limited: Reply = { ok: false, status: 429, body: 'slow down' }; + const { calls, result } = await call({ replies: [limited] }); + + /* Two retries on top of the first attempt. A budget that never ran out + would hold the caller forever on an outage that does not clear. */ + expect(calls).toHaveLength(3); + expect(result).toMatchObject({ left: { reason: 'status', status: 429 } }); +}); + +it('retries a transport failure, which carries no status to judge', async () => { + const calls: string[] = []; + const failing: FetchLike = (url, request) => { + calls.push(url); + return calls.length < 3 + ? Promise.reject(new Error('socket hang up')) + : Promise.resolve({ + ok: true, + status: 200, + text: () => Promise.resolve(''), + stream: () => toAsync(chunks), + request, + }); + }; + const result = await ModelClient.pipe( + Effect.map(client => client.stream(sampleRequest())), + Stream.unwrap, + Stream.runCollect, + Effect.map(collected => summary([...collected])), + Effect.either, + Effect.provide(testGateway({ fetch: failing, retries: 2 })), + Effect.runPromise + ); + + expect(calls).toHaveLength(3); + expect(Either.getOrThrow(result).content).toBe('hi'); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/http.ts b/packages/harness-sdk/src/plugins/gateway/http.ts new file mode 100644 index 0000000000..ef6228d967 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/http.ts @@ -0,0 +1,142 @@ +import { Effect } from 'effect'; +import type { AbortLike, FetchLike, HttpResponse } from '../../core/fetch.js'; +import { ModelError } from '../../core/model.js'; +import type { RetryPolicyService } from '../../core/retry.js'; +import { TokenError, type TokenSourceService } from '../../core/token.js'; + +/** + * Where a call gets the handle that cancels it. + * + * `AbortController` is a global in every runtime that has `fetch`, and this + * package requires the caller to supply a `fetch`, so it is read off the global + * rather than made into a plugin of its own. A runtime that lacks it still + * works: the call simply cannot be stopped early. + */ +interface AbortHandle { + readonly signal: AbortLike; + readonly abort: () => void; +} + +interface AbortHost { + readonly AbortController?: new () => AbortHandle; +} + +const host: AbortHost = globalThis; + +/** + * A handle for one call, released when the caller stops listening. + * + * The release aborts whether the call ended or was interrupted. Aborting a + * request whose body has already been read does nothing, and the alternative is + * inspecting the exit for a case where the answer is the same. + */ +const abortHandle = (): Effect.Effect => + Effect.sync(() => (host.AbortController === undefined ? undefined : new host.AbortController())); + +/** Whose credit pays for the call. */ +type OrgContext = + | { readonly kind: 'personal' } + | { readonly kind: 'organization'; readonly id: string }; + +interface HttpConfig { + /** The gateway origin, such as `https://app.kilocode.ai`. */ + readonly baseUrl: string; + readonly org: OrgContext; + /** The caller passes `fetch`, so the package needs no runtime of its own. */ + readonly fetch: FetchLike; +} + +/** The plugins one call resolves before it goes out. */ +interface HttpPlugins { + readonly token: TokenSourceService; + readonly retry: RetryPolicyService; +} + +/** Everything one call needs: where to send it, and which plugins shape it. */ +interface HttpCaller extends HttpPlugins { + readonly config: HttpConfig; +} + +const organizationHeader = 'x-kilocode-organizationid'; + +/** + * The gateway's own name for the conversation a call belongs to. + * + * It is what the gateway turns into the upstream provider's cache key — a + * `prompt_cache_key` on the OpenAI shapes, a `session_id` on OpenRouter's — and + * it seeds the routing that keeps one conversation on one provider. Without it + * the gateway has nothing to hash, so it sends neither, and a session's calls + * are as unrelated to each other as two people's are. + * + * `x-kilocode-taskid` is the same field under the editor's name for it. This is + * the one the gateway documents for everybody else, so this is the one a + * harness sends. + */ +const sessionHeader = 'x-kilo-session'; + +const headersOf = ( + org: OrgContext, + token: string, + session: string | undefined +): Record => ({ + 'content-type': 'application/json', + authorization: `Bearer ${token}`, + ...(org.kind === 'organization' ? { [organizationHeader]: org.id } : {}), + ...(session === undefined ? {} : { [sessionHeader]: session }), +}); + +/** A token that cannot be fetched is a transport failure, not a bad reply. */ +const asModelError = (error: TokenError | ModelError): ModelError => + error instanceof TokenError ? new ModelError({ reason: 'transport', cause: error.cause }) : error; + +/** One call: where it goes, what it carries, and what stops it. */ +interface Sending { + readonly path: string; + readonly body: string; + /** The conversation this call belongs to, for the gateway's cache key. */ + readonly session: string | undefined; + /** Absent when the runtime has no `AbortController`. */ + readonly signal: AbortLike | undefined; +} + +/** + * Sends the body and returns the response once the status is good. The retry + * stops here on purpose: the body has not been read yet, so a second try + * repeats nothing the caller has already seen. + * + * The token is read inside the retried effect, so a retry that follows a 401 + * picks up whatever the token plugin supplies next. + */ +const post = (caller: HttpCaller, sending: Sending): Effect.Effect => + caller.token.get().pipe( + Effect.flatMap(token => + Effect.tryPromise({ + try: () => + caller.config.fetch(`${caller.config.baseUrl.replace(/\/+$/u, '')}${sending.path}`, { + method: 'POST', + headers: headersOf(caller.config.org, token, sending.session), + body: sending.body, + ...(sending.signal === undefined ? {} : { signal: sending.signal }), + }), + catch: cause => new ModelError({ reason: 'transport', cause }), + }) + ), + Effect.flatMap(response => + response.ok + ? Effect.succeed(response) + : Effect.tryPromise({ + try: () => response.text(), + catch: cause => cause, + }).pipe( + Effect.merge, + Effect.flatMap(cause => + Effect.fail(new ModelError({ reason: 'status', status: response.status, cause })) + ) + ) + ), + Effect.mapError(asModelError), + Effect.retry(caller.retry.schedule) + ); + +export type { AbortHandle, HttpCaller, HttpConfig, OrgContext }; +export { abortHandle, post }; diff --git a/packages/harness-sdk/src/plugins/gateway/index.ts b/packages/harness-sdk/src/plugins/gateway/index.ts new file mode 100644 index 0000000000..71953be56d --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/index.ts @@ -0,0 +1,259 @@ +import { Effect, Layer, Stream } from 'effect'; +import { abortHandle, post, type AbortHandle, type HttpCaller, type HttpConfig } from './http.js'; +import { ModelCatalog, type ModelCatalogService } from '../../core/catalog.js'; +import { + ModelClient, + ModelError, + type ModelEvent, + type ModelRequest, + type ModelUsage, + type StopReason, + zeroUsage, +} from '../../core/model.js'; +import { RetryPolicy } from '../../core/retry.js'; +import { TokenSource } from '../../core/token.js'; +import { raise } from '../../core/usage.js'; +import { sseReader } from './sse.js'; +import { isFailure, type Wire, type WirePart } from './wire/wire.js'; +import { wireFor } from './wires.js'; + +/** Everything the gateway resolved once, at layer build. */ +interface Gateway extends HttpCaller { + readonly catalog: ModelCatalogService; +} + +/** One call, with the handle that stops it when the caller stops listening. */ +interface Sent { + readonly wire: Wire; + readonly request: ModelRequest; + readonly handle: AbortHandle | undefined; +} + +/** + * Rendering is wrapped because a wire refuses what its shape cannot carry: an + * image in a media type the provider does not take throws here, and that is a + * failed call, not a crash. + */ +const bodyFor = (gateway: Gateway, sent: Sent) => + Effect.try({ + try: () => JSON.stringify(sent.wire.toBody(sent.request)), + catch: cause => new ModelError({ reason: 'unsupported', cause }), + }).pipe( + Effect.flatMap(body => + post(gateway, { + path: sent.wire.path, + body, + session: sent.request.cacheKey, + signal: sent.handle?.signal, + }) + ) + ); + +const chunksOf = (response: { readonly stream?: () => AsyncIterable }) => { + const body = response.stream; + return body === undefined + ? Stream.fail(new ModelError({ reason: 'transport', cause: 'the caller supplied no stream' })) + : Stream.fromAsyncIterable(body(), cause => new ModelError({ reason: 'transport', cause })); +}; + +/** + * What one stream collects on the way past, to report when it ends. + * + * It is mutable, and that is the one place in this package where mutation is + * the right answer. One of these is made per call, inside `stream` below, and + * never leaves it: a `Stream` is consumed by one fiber, so nothing else can see + * a half-written tally and no `Ref` is buying anything. + * + * What it buys instead is measured. This path runs once per streamed event, and + * before this it was five Effect operators over four `Ref`s per event, which is + * an allocation each on the one path a long answer walks thousands of times. + * See "What a streamed token actually costs" in AGENTS.md. + */ +interface Tally { + usage: ModelUsage; + stop: StopReason; + /** The call being read, until the frame that closes it. See `collect`. */ + open: OpenCall | undefined; + /** Whether the model asked for anything. It decides the stop reason. */ + called: boolean; +} + +/** A tool call as it arrives: the name first, then the arguments in fragments. */ +interface OpenCall { + readonly id: string; + readonly name: string; + /** Grown a fragment at a time, in place: a copy per fragment is quadratic. */ + text: string; +} + +/** Nothing to report from this frame. One value, so no frame allocates a list. */ +const nothing: readonly ModelEvent[] = []; + +const closed = (held: OpenCall | undefined): readonly ModelEvent[] => + held === undefined + ? nothing + : [{ kind: 'toolCall', call: { id: held.id, name: held.name, arguments: held.text } }]; + +/** Closes whatever call is open, and leaves nothing open behind it. */ +const ending = (tally: Tally): readonly ModelEvent[] => { + const held = tally.open; + tally.open = undefined; + return closed(held); +}; + +/** + * Collects the pieces of a tool call into one event, and passes everything else + * through untouched. + * + * A call closes on the frame that says so, and on the frame that opens the next + * one: one shape sends no closing frame at all, so opening a second call is + * what ends the first. What is still open when the stream ends is closed by + * `lastOf`. + */ +const collect = (tally: Tally, part: WirePart): readonly ModelEvent[] => { + switch (part.kind) { + case 'callStart': { + const ended = ending(tally); + tally.open = { id: part.id, name: part.name, text: part.text ?? '' }; + tally.called = true; + return ended; + } + case 'callArguments': { + if (tally.open !== undefined) { + tally.open.text += part.text; + } + return nothing; + } + case 'callEnd': { + return ending(tally); + } + case 'delta': + case 'reasoning': + case 'redacted': { + return [part]; + } + } +}; + +/** Everything one frame says: what it cost, why the model stopped, what it said. */ +const read = (wire: Wire, tally: Tally, event: unknown): readonly ModelEvent[] => { + const spent = wire.toUsage(event); + if (spent !== undefined) { + tally.usage = raise(tally.usage, spent); + } + const reason = wire.toStop(event); + if (reason !== undefined) { + tally.stop = reason; + } + const part = wire.toDelta(event); + return part === undefined ? nothing : collect(tally, part); +}; + +/** + * One frame, as the stream sees it. Two operators where there were five, and + * the Effect stays because a body that will not parse and a failure the + * provider reported mid-answer are both the end of the call and have to fail + * it. What was worth removing was the four `Ref`s, not this. + */ +const eventsOf = ( + wire: Wire, + tally: Tally, + data: string +): Effect.Effect => + Effect.try({ + try: (): unknown => JSON.parse(data), + catch: cause => new ModelError({ reason: 'body', cause }), + }).pipe( + Effect.flatMap(event => + isFailure(event) + ? Effect.fail(new ModelError({ reason: 'stream', cause: event })) + : Effect.succeed(read(wire, tally, event)) + ) + ); + +/** + * Why the model really stopped. + * + * One shape names the reason outright. The other two report a finished response + * whether or not the model asked for a tool, so a stream that produced a call + * says so here instead. Only a clean end is corrected: an answer the ceiling cut + * off holds half a call, and running it would run something the model did not + * finish asking for. + */ +const reasonOf = (stop: StopReason, called: boolean): StopReason => + called && stop === 'end' ? 'tools' : stop; + +/** + * The last events of every stream: the call still open, the cost, and the + * reason. Suspended, because this is built when the stream is and read when the + * stream ends. + */ +const lastOf = (tally: Tally): Stream.Stream => + Stream.suspend(() => { + const ended = ending(tally); + const done: ModelEvent = { + kind: 'done', + usage: tally.usage, + stop: reasonOf(tally.stop, tally.called), + }; + return Stream.fromIterable([...ended, done]); + }); + +/** + * The handle lives as long as the stream, not as long as the request. + * + * A streamed call returns as soon as the headers arrive and keeps producing + * afterwards, so a handle released when the request resolved would cancel + * nothing. Scoped to the stream, dropping the stream stops the generation, and + * the provider stops charging for it. + */ +const stream = (gateway: Gateway, request: ModelRequest): Stream.Stream => + Stream.unwrapScoped( + Effect.gen(function* () { + const handle = yield* Effect.acquireRelease(abortHandle(), held => + Effect.sync(() => held?.abort()) + ); + const tally: Tally = { + usage: zeroUsage, + stop: 'unknown', + open: undefined, + called: false, + }; + const wire = yield* wireFor(gateway.catalog, request.model); + const frames = sseReader(); + + return Stream.fromEffect(bodyFor(gateway, { wire, request, handle })).pipe( + Stream.flatMap(chunksOf), + Stream.mapConcat(chunk => frames(chunk)), + Stream.mapConcatEffect(data => eventsOf(wire, tally, data)), + Stream.concat(lastOf(tally)) + ); + }) + ); + +/** + * The kilo gateway plugin. It picks the best shape the model speaks. + * + * The catalog, the token and the retry policy are resolved once here, so the + * request path carries no lookup and the returned client needs no context. + */ +const layerKiloGateway = ( + config: HttpConfig +): Layer.Layer => + Layer.effect( + ModelClient, + Effect.gen(function* () { + const gateway: Gateway = { + config, + catalog: yield* ModelCatalog, + token: yield* TokenSource, + retry: yield* RetryPolicy, + }; + return { + stream: request => stream(gateway, request), + }; + }) + ); + +export type { HttpConfig as KiloGatewayConfig }; +export { layerKiloGateway }; diff --git a/packages/harness-sdk/src/plugins/gateway/sse.test.ts b/packages/harness-sdk/src/plugins/gateway/sse.test.ts new file mode 100644 index 0000000000..8167a0b3b8 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/sse.test.ts @@ -0,0 +1,20 @@ +import { expect, it } from 'vitest'; +import { sseReader } from './sse.js'; + +it('holds an unfinished event until the rest of it arrives', () => { + const read = sseReader(); + expect(read('data: {"a":')).toEqual([]); + expect(read('1}\n\n')).toEqual(['{"a":1}']); +}); + +it('joins a data field written over two lines', () => { + expect(sseReader()('data: one\ndata: two\n\n')).toEqual(['one\ntwo']); +}); + +it('reads two events out of one chunk', () => { + expect(sseReader()('data: a\n\ndata: b\n\n')).toEqual(['a', 'b']); +}); + +it('skips a comment and the done marker but keeps the events around them', () => { + expect(sseReader()('data: a\n\n: ping\n\ndata: [DONE]\n\ndata: b\n\n')).toEqual(['a', 'b']); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/sse.ts b/packages/harness-sdk/src/plugins/gateway/sse.ts new file mode 100644 index 0000000000..431bc94ffa --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/sse.ts @@ -0,0 +1,25 @@ +import { createParser } from 'eventsource-parser'; + +/** + * Reads server-sent events. `eventsource-parser` holds the framing: a chunk may + * stop in the middle of an event, a data field may run over several lines, and + * a comment carries nothing. + * + * The reader holds state, so make one per stream. `[DONE]` is not part of the + * event stream standard; it is how OpenAI marks the end, and it carries nothing. + */ +const sseReader = (): ((chunk: string) => readonly string[]) => { + let events: string[] = []; + const parser = createParser({ + onEvent: event => { + events.push(event.data); + }, + }); + return chunk => { + events = []; + parser.feed(chunk); + return events.filter(data => data !== '' && data !== '[DONE]'); + }; +}; + +export { sseReader }; diff --git a/packages/harness-sdk/src/plugins/gateway/stream.test.ts b/packages/harness-sdk/src/plugins/gateway/stream.test.ts new file mode 100644 index 0000000000..e1aeaf319c --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/stream.test.ts @@ -0,0 +1,247 @@ +import { Effect, Either, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import type { ApiKind } from '../../core/catalog.js'; +import { fakeFetch, type Reply, sampleRequest, sse } from './fake.js'; +import { testGateway } from './test-gateway.js'; +import { ModelClient, type ModelEvent } from '../../core/model.js'; + +const collect = async (kinds: readonly ApiKind[], chunks: readonly string[]) => { + const reply: Reply = { ok: true, status: 200, body: '', chunks }; + const { calls, fetch } = fakeFetch([reply]); + const events = await ModelClient.pipe( + Effect.map(client => client.stream(sampleRequest())), + Stream.unwrap, + Stream.runCollect, + Effect.map(chunk => [...chunk]), + Effect.provide(testGateway({ fetch, kinds })), + Effect.runPromise + ); + return { calls, events }; +}; + +const textOf = (events: readonly ModelEvent[]): string => + events + .filter(event => event.kind === 'delta') + .map(event => event.text) + .join(''); + +it('streams the text of an Anthropic reply and ends with the token counts', async () => { + const { calls, events } = await collect( + ['messages'], + sse( + { + type: 'message_start', + message: { usage: { input_tokens: 5, cache_read_input_tokens: 95 } }, + }, + { type: 'content_block_delta', delta: { text: 'he' } }, + { type: 'content_block_delta', delta: { text: 'llo' } }, + { type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 4 } } + ) + ); + expect(JSON.parse(calls[0]?.request.body ?? '')).toMatchObject({ stream: true }); + expect(textOf(events)).toBe('hello'); + expect(events.at(-1)).toEqual({ + kind: 'done', + usage: { inputTokens: 5, outputTokens: 4, cacheReadTokens: 95, cacheWriteTokens: 0 }, + stop: 'end', + }); +}); + +it('joins a delta that arrives split over two chunks', async () => { + const [first, second] = [ + 'data: {"type":"content_block_delta","delta":{"te', + 'xt":"split"}}\n\ndata: {"type":"message_delta","usage":{"output_tokens":1}}\n\n', + ]; + const { events } = await collect(['messages'], [first ?? '', second ?? '']); + expect(textOf(events)).toBe('split'); +}); + +it('streams a completions reply and asks for its token counts', async () => { + const { calls, events } = await collect( + ['chat_completions'], + sse( + { choices: [{ delta: { content: 'hi' } }] }, + { + choices: [], + usage: { + prompt_tokens: 10, + completion_tokens: 2, + prompt_tokens_details: { cached_tokens: 9 }, + }, + } + ) + ); + expect(JSON.parse(calls[0]?.request.body ?? '')).toMatchObject({ + stream_options: { include_usage: true }, + }); + expect(textOf(events)).toBe('hi'); + expect(events.at(-1)).toMatchObject({ usage: { cacheReadTokens: 9, inputTokens: 1 } }); +}); + +it('streams a responses reply and names the cache key', async () => { + const { calls, events } = await collect( + ['responses'], + sse( + { type: 'response.output_text.delta', delta: 'yo' }, + { + type: 'response.completed', + response: { + usage: { + input_tokens: 20, + output_tokens: 2, + input_tokens_details: { cached_tokens: 19 }, + }, + }, + } + ) + ); + expect(JSON.parse(calls[0]?.request.body ?? '')).toMatchObject({ prompt_cache_key: 'ses_1' }); + expect(textOf(events)).toBe('yo'); + expect(events.at(-1)).toMatchObject({ usage: { cacheReadTokens: 19, inputTokens: 1 } }); +}); + +it('counts a responses reply that stopped at the wall', async () => { + /* The truncated frame carries its counts exactly as the finished one does. + Reading only `response.completed` reports zero for a call that filled the + window, and the session then believes it has room it does not have. */ + const { events } = await collect( + ['responses'], + sse( + { type: 'response.output_text.delta', delta: 'cut' }, + { + type: 'response.incomplete', + response: { + status: 'incomplete', + incomplete_details: { reason: 'max_output_tokens' }, + usage: { + input_tokens: 30, + output_tokens: 8, + input_tokens_details: { cached_tokens: 29 }, + }, + }, + } + ) + ); + + expect(events.at(-1)).toEqual({ + kind: 'done', + usage: { inputTokens: 1, outputTokens: 8, cacheReadTokens: 29, cacheWriteTokens: 0 }, + stop: 'maxTokens', + }); +}); + +const doneUsage = (events: readonly ModelEvent[]) => { + const last = events.at(-1); + return last?.kind === 'done' ? last.usage : undefined; +}; + +it('keeps the input counts when a later frame echoes zeros', async () => { + const { events } = await collect( + ['messages'], + sse( + { + type: 'message_start', + message: { usage: { input_tokens: 3, cache_read_input_tokens: 11_822 } }, + }, + { type: 'content_block_delta', delta: { text: 'hi' } }, + { + type: 'message_delta', + usage: { input_tokens: 0, cache_read_input_tokens: 0, output_tokens: 42 }, + } + ) + ); + + expect(doneUsage(events)).toEqual({ + inputTokens: 3, + outputTokens: 42, + cacheReadTokens: 11_822, + cacheWriteTokens: 0, + }); +}); + +it('ignores a token count that JSON.parse turned into Infinity', async () => { + const { events } = await collect( + ['chat_completions'], + ['data: {"usage":{"prompt_tokens":1e999,"completion_tokens":1}}\n\n'] + ); + + expect(doneUsage(events)).toEqual({ + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }); +}); + +it('fails the stream when the provider reports an error part way through it', async () => { + /* Anthropic's streaming reference: "The API may occasionally send errors in + the event stream", such as an `overloaded_error` that would be a 529 on a + call that was not streamed. Swallowing the frame stores a truncated answer + as a whole one, and every later request is built on it. */ + const reply: Reply = { + ok: true, + status: 200, + body: '', + chunks: sse( + { type: 'message_start', message: { usage: { input_tokens: 5 } } }, + { type: 'content_block_delta', delta: { text: 'half an ans' } }, + { type: 'error', error: { type: 'overloaded_error', message: 'Overloaded' } } + ), + }; + const { fetch } = fakeFetch([reply]); + const result = await ModelClient.pipe( + Effect.map(client => client.stream(sampleRequest())), + Stream.unwrap, + Stream.runCollect, + Effect.either, + Effect.provide(testGateway({ fetch, kinds: ['messages'] })), + Effect.runPromise + ); + + expect(result).toMatchObject({ left: { reason: 'stream' } }); +}); + +/** The responses shape puts its failure one level down, inside `response`. */ +const responsesEnding = async (ending: unknown) => { + const reply: Reply = { + ok: true, + status: 200, + body: '', + chunks: sse({ type: 'response.output_text.delta', delta: 'half an ans' }, ending), + }; + const { fetch } = fakeFetch([reply]); + return ModelClient.pipe( + Effect.map(client => client.stream(sampleRequest())), + Stream.unwrap, + Stream.runCollect, + Effect.either, + Effect.provide(testGateway({ fetch, kinds: ['responses'] })), + Effect.runPromise + ); +}; + +it('fails the stream when the responses shape reports a failed response', async () => { + /* `response.failed` carries its reason as `response.error`, so the frame has + no top-level `error` to read. Letting it pass ends the stream on a `done` + that says `unknown`, and the fragment is stored as a whole answer. */ + const result = await responsesEnding({ + type: 'response.failed', + response: { status: 'failed', error: { code: 'server_error', message: 'went wrong' } }, + }); + + expect(result).toMatchObject({ left: { reason: 'stream' } }); +}); + +it('lets a completed response through, error and all', async () => { + /* This shape carries `error: null` inside `response` on every call that + worked. A reader that takes the key for the failure marks every reply a + failure. The frame is parsed from text because that is how it arrives, and + because the package writes no `null` of its own. */ + const completed: unknown = JSON.parse( + '{"type":"response.completed","response":{"status":"completed","error":null,' + + '"usage":{"input_tokens":1,"output_tokens":1}}}' + ); + const result = await responsesEnding(completed); + + expect(Either.isRight(result)).toBe(true); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/test-gateway.ts b/packages/harness-sdk/src/plugins/gateway/test-gateway.ts new file mode 100644 index 0000000000..af6d918646 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/test-gateway.ts @@ -0,0 +1,41 @@ +import { Layer } from 'effect'; +import type { ApiKind } from '../../core/catalog.js'; +import type { FetchLike } from '../../core/fetch.js'; +import type { OrgContext } from './http.js'; +import { layerKiloGateway } from './index.js'; +import type { ModelClient } from '../../core/model.js'; +import { TokenSource, type TokenSourceService } from '../../core/token.js'; +import { layerBackoff, layerNoRetry } from '../retry/backoff.js'; +import { layerStaticToken } from '../token/static.js'; +import { layerTableCatalog } from '../catalog/table.js'; + +/** + * The gateway with every plugin it needs, wired for a test: the catalog answers + * `kinds` for any model, the token never changes, and nothing is retried unless + * the test asks for it. A test about the credential passes its own `token`. + */ +const testGateway = (options: { + readonly fetch: FetchLike; + readonly kinds?: readonly ApiKind[]; + readonly retries?: number; + readonly org?: OrgContext; + readonly baseUrl?: string; + readonly token?: TokenSourceService; +}): Layer.Layer => + layerKiloGateway({ + baseUrl: options.baseUrl ?? 'https://app.kilocode.ai', + org: options.org ?? { kind: 'personal' }, + fetch: options.fetch, + }).pipe( + Layer.provide( + Layer.mergeAll( + layerTableCatalog({}, { apiKinds: options.kinds ?? ['messages'] }), + options.token === undefined + ? layerStaticToken('tok') + : Layer.succeed(TokenSource, options.token), + options.retries === undefined ? layerNoRetry : layerBackoff(options.retries) + ) + ) + ); + +export { testGateway }; diff --git a/packages/harness-sdk/src/plugins/gateway/tools.test.ts b/packages/harness-sdk/src/plugins/gateway/tools.test.ts new file mode 100644 index 0000000000..ac897867dd --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/tools.test.ts @@ -0,0 +1,162 @@ +import { Effect, Stream } from 'effect'; +import { createAssert } from 'typia'; +import { expect, it } from 'vitest'; +import type { ApiKind } from '../../core/catalog.js'; +import { ModelClient, type ModelEvent, type ModelRequest } from '../../core/model.js'; +import type { ToolDefinition } from '../../core/tool.js'; +import { fakeFetch, type Reply, sampleRequest, sse } from './fake.js'; +import { testGateway } from './test-gateway.js'; + +/** + * A tool call arrives in pieces on every shape, and no shape sends the same + * pieces as another. What is proved here is that all three arrive above the + * transport as one `toolCall` carrying the whole of the arguments, so nothing + * else in the package has to know how a shape spells one. + */ + +const weather: ToolDefinition = { + name: 'weather', + description: 'The weather somewhere.', + parameters: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] }, +}; + +const withTools = (): ModelRequest => ({ ...sampleRequest(), tools: [weather] }); + +/** What went on the wire, so a test can read one field of it by name. */ +const asBody = createAssert>(); + +const collect = async (kind: ApiKind, chunks: readonly string[]) => { + const reply: Reply = { ok: true, status: 200, body: '', chunks }; + const { calls, fetch } = fakeFetch([reply]); + const events = await ModelClient.pipe( + Effect.map(client => client.stream(withTools())), + Stream.unwrap, + Stream.runCollect, + Effect.map(chunk => [...chunk]), + Effect.provide(testGateway({ fetch, kinds: [kind] })), + Effect.runPromise + ); + return { sent: asBody(JSON.parse(calls[0]?.request.body ?? '{}')), events }; +}; + +const callsIn = (events: readonly ModelEvent[]) => + events.filter(event => event.kind === 'toolCall').map(event => event.call); + +const stopOf = (events: readonly ModelEvent[]) => events.find(event => event.kind === 'done')?.stop; + +it('collects a call the messages shape streams as blocks, and says the model wants it', async () => { + const { sent, events } = await collect( + 'messages', + sse( + { + type: 'content_block_start', + content_block: { type: 'tool_use', id: 'tc_1', name: 'weather' }, + }, + { type: 'content_block_delta', delta: { partial_json: '{"city"' } }, + { type: 'content_block_delta', delta: { partial_json: ':"Oslo"}' } }, + { type: 'content_block_stop' }, + { type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 4 } } + ) + ); + + expect(callsIn(events)).toEqual([{ id: 'tc_1', name: 'weather', arguments: '{"city":"Oslo"}' }]); + expect(stopOf(events)).toBe('tools'); + expect(sent['tools']).toEqual([ + { + name: 'weather', + description: 'The weather somewhere.', + input_schema: { + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + }, + }, + ]); +}); + +it('collects a call the responses shape streams as items', async () => { + const { sent, events } = await collect( + 'responses', + sse( + { + type: 'response.output_item.added', + item: { type: 'function_call', call_id: 'tc_2', name: 'weather' }, + }, + { type: 'response.function_call_arguments.delta', delta: '{"city":' }, + { type: 'response.function_call_arguments.delta', delta: '"Oslo"}' }, + { type: 'response.output_item.done', item: { type: 'function_call' } }, + { type: 'response.completed', response: { usage: { input_tokens: 1, output_tokens: 2 } } } + ) + ); + + expect(callsIn(events)).toEqual([{ id: 'tc_2', name: 'weather', arguments: '{"city":"Oslo"}' }]); + /* This shape reports a finished response whether or not the model asked for + anything, so the reason comes from the call having been made. */ + expect(stopOf(events)).toBe('tools'); + expect(sent['tools']).toMatchObject([{ type: 'function', name: 'weather', strict: false }]); +}); + +it('collects two calls the completions shape never closes, and closes the last itself', async () => { + const { sent, events } = await collect( + 'chat_completions', + sse( + { + choices: [ + { delta: { tool_calls: [{ id: 'tc_3', function: { name: 'weather', arguments: '' } }] } }, + ], + }, + { choices: [{ delta: { tool_calls: [{ function: { arguments: '{"city":"Oslo"}' } }] } }] }, + { + choices: [ + { delta: { tool_calls: [{ id: 'tc_4', function: { name: 'weather', arguments: '' } }] } }, + ], + }, + { choices: [{ delta: { tool_calls: [{ function: { arguments: '{"city":"Rome"}' } }] } }] }, + { choices: [{ finish_reason: 'tool_calls', delta: {} }] } + ) + ); + + /* The first call is closed by the frame that opens the second, and the second + by the end of the stream. This shape sends nothing that closes either. */ + expect(callsIn(events)).toEqual([ + { id: 'tc_3', name: 'weather', arguments: '{"city":"Oslo"}' }, + { id: 'tc_4', name: 'weather', arguments: '{"city":"Rome"}' }, + ]); + expect(stopOf(events)).toBe('tools'); + expect(sent['tools']).toMatchObject([{ type: 'function', function: { name: 'weather' } }]); +}); + +it('leaves the tools out of a request that offers none', async () => { + const reply: Reply = { ok: true, status: 200, body: '', chunks: sse({}) }; + const { calls, fetch } = fakeFetch([reply]); + await ModelClient.pipe( + Effect.map(client => client.stream(sampleRequest())), + Stream.unwrap, + Stream.runDrain, + Effect.provide(testGateway({ fetch, kinds: ['messages'] })), + Effect.runPromise + ); + + /* Not an empty list: an empty list would sit in the prefix of every session + that has no tools, and some shapes refuse one outright. */ + expect(JSON.parse(calls[0]?.request.body ?? '{}')).not.toHaveProperty('tools'); +}); + +it('keeps a truncated answer truncated, whatever it half asked for', async () => { + const { events } = await collect( + 'messages', + sse( + { + type: 'content_block_start', + content_block: { type: 'tool_use', id: 'tc_5', name: 'weather' }, + }, + { type: 'content_block_delta', delta: { partial_json: '{"cit' } }, + { type: 'message_delta', delta: { stop_reason: 'max_tokens' }, usage: { output_tokens: 4 } } + ) + ); + + /* The call is reported, because the model did make it and the transcript must + hold it. The reason is not upgraded, so nothing runs half a call. */ + expect(callsIn(events)).toEqual([{ id: 'tc_5', name: 'weather', arguments: '{"cit' }]); + expect(stopOf(events)).toBe('maxTokens'); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/wire/body.test.ts b/packages/harness-sdk/src/plugins/gateway/wire/body.test.ts new file mode 100644 index 0000000000..738f014301 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/body.test.ts @@ -0,0 +1,62 @@ +import { assert } from 'typia'; +import { expect, it } from 'vitest'; +import type { Prompt } from '../../../core/prompt.js'; +import { completionsWire } from './completions.js'; +import { messagesWire } from './messages.js'; +import { responsesWire } from './responses.js'; + +/** + * The same prompt through each shape, read back as text. + * + * Every shape spells the system prompt differently — a block list, a string, a + * message with a role of its own — and until 2026-09-04 nothing here asserted + * two of the three. A shape that dropped or replaced the system prompt passed + * the whole suite and failed only against a live model, which is late and + * expensive to read. + */ + +const prompt: Prompt = { + system: [{ text: 'the system prompt', cache: true }], + messages: [ + { role: 'user', parts: [{ kind: 'text', text: 'the question' }], cache: false }, + { role: 'assistant', parts: [{ kind: 'text', text: 'the answer' }], cache: true }, + ], +}; + +it('sends the system prompt and both turns on the messages shape', () => { + const body = assert<{ + system: { text: string }[]; + messages: { role: string; content: { text?: string }[] }[]; + }>(messagesWire.toBody({ prompt, model: 'm', maxTokens: 8 })); + + expect(body.system.map(part => part.text)).toEqual(['the system prompt']); + expect(body.messages.map(message => [message.role, message.content[0]?.text])).toEqual([ + ['user', 'the question'], + ['assistant', 'the answer'], + ]); +}); + +it('sends the system prompt as instructions on the responses shape', () => { + const body = assert<{ + instructions: string; + input: { role: string; content: { text?: string }[] }[]; + }>(responsesWire.toBody({ prompt, model: 'm', maxTokens: 8 })); + + expect(body.instructions).toBe('the system prompt'); + expect(body.input.map(item => [item.role, item.content[0]?.text])).toEqual([ + ['user', 'the question'], + ['assistant', 'the answer'], + ]); +}); + +it('sends the system prompt as a system message on the completions shape', () => { + const body = assert<{ messages: { role: string; content: { text?: string }[] }[] }>( + completionsWire.toBody({ prompt, model: 'm', maxTokens: 8 }) + ); + + expect(body.messages.map(message => [message.role, message.content[0]?.text])).toEqual([ + ['system', 'the system prompt'], + ['user', 'the question'], + ['assistant', 'the answer'], + ]); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/wire/completions.ts b/packages/harness-sdk/src/plugins/gateway/wire/completions.ts new file mode 100644 index 0000000000..ad8b8c5611 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/completions.ts @@ -0,0 +1,267 @@ +import type OpenAI from 'openai'; +import { createIs } from 'typia'; +import type { Effort, ModelRequest, ModelUsage, StopReason } from '../../../core/model.js'; +import type { PromptMessage, PromptPart } from '../../../core/prompt.js'; +import type { ToolDefinition } from '../../../core/tool.js'; +import { dataUri, resultText } from './parts.js'; +import { stopFrom, type Wire, type WirePart } from './wire.js'; +import { readCached, type TokenCount } from './usage.js'; + +/** + * The OpenAI chat shape, with one extension for the effort. + * + * It marks no cache breakpoint. It used to send Anthropic's `cache_control` on + * the last block, on the theory that the gateway would forward it to a provider + * that reads it. Measured on 2026-09-04 against a prefix nobody had sent + * before, twice for `openai/gpt-5.6-luna` and twice for + * `anthropic/claude-haiku-4.5`: the second call read 12229 and 13630 cached + * tokens, the same to the token with the breakpoint and without it. This shape + * caches on whatever the gateway does, which is what `api-kind.ts` ranks it on. + */ +type CompletionsBody = Omit & { + /** The OpenRouter reasoning field. It is not part of the OpenAI type. */ + readonly reasoning?: { readonly effort: Effort }; + readonly messages: readonly WireMessage[]; +}; + +/** + * A message, as this shape takes one. + * + * A tool result is a message of its own here, with a role of its own, where + * both other shapes carry it as content inside a message. A call is a field on + * the assistant's message rather than a block in it, for the same reason: this + * shape was built before a message could hold anything but words. + */ +type WireMessage = + | { + readonly role: 'system' | 'user' | 'assistant'; + readonly content: readonly ContentBlock[]; + readonly tool_calls?: readonly CallBlock[]; + } + | { readonly role: 'tool'; readonly tool_call_id: string; readonly content: string }; + +type ContentBlock = + | { readonly type: 'text'; readonly text: string } + | { readonly type: 'image_url'; readonly image_url: { readonly url: string } }; + +/** A call, as this shape takes one. The arguments stay the text the model wrote. */ +interface CallBlock { + readonly id: string; + readonly type: 'function'; + readonly function: { readonly name: string; readonly arguments: string }; +} + +/** + * Reasoning is left out. Providers relayed through this shape report their + * thinking under two different field names and neither takes it back, so there + * is no block this shape could replay. + */ +const renderPart = (part: PromptPart): ContentBlock | undefined => { + switch (part.kind) { + case 'text': { + return { type: 'text', text: part.text }; + } + case 'image': { + return { type: 'image_url', image_url: { url: dataUri(part) } }; + } + /* None of these is content on this shape. A call is a field on the message + and a result is a message of its own, both built by `renderMessage`; + thinking has no form this shape takes back at all. */ + case 'reasoning': + case 'redacted': + case 'toolCall': + case 'toolResult': { + return undefined; + } + } +}; + +const callBlock = (part: PromptPart): CallBlock | undefined => + part.kind === 'toolCall' + ? { + id: part.callId, + type: 'function', + function: { name: part.name, arguments: part.arguments }, + } + : undefined; + +/** + * A result is its own message here, one per result, and it must follow the + * message that made the call. The turns arrive in order, so it does. + * + * There is no flag for a failed result, so the text says so instead. + */ +const resultMessage = (part: PromptPart): WireMessage | undefined => + part.kind === 'toolResult' + ? { + role: 'tool', + tool_call_id: part.callId, + content: resultText(part.body, part.failed), + } + : undefined; + +/** + * One turn, as the messages this shape takes: what was said, then every result + * it carried. A turn of nothing but results produces no message of its own, + * because a message with no content is refused. + */ +const renderMessage = (message: PromptMessage): readonly WireMessage[] => { + const content = message.parts.map(renderPart).filter(part => part !== undefined); + const calls = message.parts.map(callBlock).filter(block => block !== undefined); + const results = message.parts.map(resultMessage).filter(item => item !== undefined); + const said: readonly WireMessage[] = + content.length === 0 && calls.length === 0 + ? [] + : [{ role: message.role, content, ...(calls.length === 0 ? {} : { tool_calls: calls }) }]; + return [...said, ...results]; +}; + +/** A tool, as this shape takes one: a function, wrapped in an envelope. */ +const toolBlock = (tool: ToolDefinition): OpenAI.Chat.ChatCompletionTool => ({ + type: 'function', + function: { + name: tool.name, + description: tool.description, + parameters: { ...tool.parameters }, + }, +}); + +const toBody = ({ prompt, model, maxTokens, effort, tools }: ModelRequest): CompletionsBody => ({ + model, + max_tokens: maxTokens, + stream: true, + stream_options: { include_usage: true }, + ...(effort === undefined ? {} : { reasoning: { effort } }), + ...(tools === undefined || tools.length === 0 ? {} : { tools: tools.map(toolBlock) }), + messages: [ + ...prompt.system.map(part => ({ + role: 'system' as const, + content: [{ type: 'text' as const, text: part.text }], + })), + ...prompt.messages.flatMap(renderMessage), + ], +}); + +interface Counts { + prompt_tokens: TokenCount; + completion_tokens: TokenCount; + prompt_tokens_details?: { cached_tokens?: TokenCount | null } | null; +} + +const stopReasons: Readonly> = { + stop: 'end', + length: 'maxTokens', + content_filter: 'refusal', + tool_calls: 'tools', +}; + +const asStop = stopFrom(stopReasons); + +interface DeltaEvent { + choices: { delta: { content?: string | null } }[]; +} + +/** + * The relayed providers do not agree on a name for the thinking: OpenRouter + * sends `reasoning` and others send `reasoning_content`, so both are read. + */ +interface ReasoningEvent { + choices: { delta: { reasoning?: string | null; reasoning_content?: string | null } }[]; +} + +interface UsageEvent { + usage: Counts; +} + +/** The last content frame of a choice names why that choice ended. */ +interface StopEvent { + choices: { finish_reason: string }[]; +} + +/** + * A call, in pieces. The opening frame carries the identifier and the name, and + * every frame after it carries another fragment of the arguments. Both may sit + * on the opening frame, so `callStart` takes the fragment with them. + * + * There is no frame that closes a call on this shape. The next one opens the + * next call, and the end of the stream closes the last. + */ +interface CallEvent { + choices: { + delta: { + tool_calls: { + id?: string | null; + function?: { name?: string | null; arguments?: string | null } | null; + }[]; + }; + }[]; +} + +const isDelta = createIs(); +const isCall = createIs(); +const isReasoning = createIs(); +const isUsage = createIs(); +const isStop = createIs(); + +const readUsage = (usage: Counts): Partial => + readCached( + usage.prompt_tokens, + usage.completion_tokens, + usage.prompt_tokens_details?.cached_tokens ?? 0 + ); + +/** + * The empty-choices frame is filtered here rather than in the type. A tuple + * with a rest element expresses it, but typia then copies the rest on every + * check, which costs three times as much on the per-token path. + * + * A `content` of `""` is a placeholder a provider puts on the frame that + * carries the thinking, not a word, so only a non-empty one is a text delta. + * Reading the empty string as an answer would shadow the thinking beside it. + */ +const toDelta = (event: unknown): WirePart | undefined => { + const said = isDelta(event) ? (event.choices[0]?.delta.content ?? undefined) : undefined; + if (said !== undefined && said.length > 0) { + return { kind: 'delta', text: said }; + } + if (!isReasoning(event)) { + return toCall(event); + } + const thought = event.choices[0]?.delta; + const text = thought?.reasoning ?? thought?.reasoning_content ?? undefined; + return text === undefined ? toCall(event) : { kind: 'reasoning', text }; +}; + +/** The pieces of a call, kept apart from the per-token path above. */ +const toCall = (event: unknown): WirePart | undefined => { + const call = isCall(event) ? event.choices[0]?.delta.tool_calls[0] : undefined; + if (call === undefined) { + return undefined; + } + const text = call.function?.arguments ?? undefined; + if (call.id === undefined || call.id === null) { + return text === undefined ? undefined : { kind: 'callArguments', text }; + } + return { + kind: 'callStart', + id: call.id, + name: call.function?.name ?? '', + ...(text === undefined || text.length === 0 ? {} : { text }), + }; +}; + +const toUsage = (event: unknown): Partial | undefined => + isUsage(event) ? readUsage(event.usage) : undefined; + +const toStop = (event: unknown): StopReason | undefined => + isStop(event) ? asStop(event.choices[0]?.finish_reason) : undefined; + +const completionsWire: Wire = { + path: '/api/gateway/v1/chat/completions', + toBody, + toDelta, + toUsage, + toStop, +}; + +export { completionsWire }; diff --git a/packages/harness-sdk/src/plugins/gateway/wire/image.test.ts b/packages/harness-sdk/src/plugins/gateway/wire/image.test.ts new file mode 100644 index 0000000000..d3efcca11f --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/image.test.ts @@ -0,0 +1,91 @@ +import { assert } from 'typia'; +import { expect, it } from 'vitest'; +import type { ModelRequest } from '../../../core/model.js'; +import type { Prompt } from '../../../core/prompt.js'; +import { completionsWire } from './completions.js'; +import { messagesWire } from './messages.js'; +import { responsesWire } from './responses.js'; + +const pixel = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAE'; + +const promptWith = (media: string): Prompt => ({ + system: [{ text: 'sys', cache: true }], + messages: [ + { + role: 'user', + parts: [ + { kind: 'text', text: 'what is this' }, + { kind: 'image', media, data: pixel }, + ], + cache: true, + }, + ], +}); + +const request = (media: string): ModelRequest => ({ + prompt: promptWith(media), + model: 'claude-opus-5', + maxTokens: 64, + cacheKey: 'ses_1', +}); + +it('sends the image as base64 the provider can read, without encoding it again', () => { + const body = messagesWire.toBody(request('image/png')); + + expect(body).toMatchObject({ + messages: [ + { + content: [ + { type: 'text', text: 'what is this' }, + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: pixel } }, + ], + }, + ], + }); +}); + +it('marks the breakpoint on the last block of the message, not on every one', () => { + const body = assert<{ messages: { content: { cache_control?: unknown }[] }[] }>( + messagesWire.toBody(request('image/png')) + ); + + /* A breakpoint on the text block would cut the prefix before the image, so + the image would be paid for on every single request of the session. */ + const marks = body.messages[0]?.content.map(block => block.cache_control !== undefined); + expect(marks).toEqual([false, true]); +}); + +it('refuses a media type the shape cannot carry rather than sending it', () => { + expect(() => messagesWire.toBody(request('image/heic'))).toThrow(); +}); + +it('sends the image as a data URI on both OpenAI shapes', () => { + const uri = `data:image/png;base64,${pixel}`; + + expect(responsesWire.toBody(request('image/png'))).toMatchObject({ + input: [{ content: [{ type: 'input_text' }, { type: 'input_image', image_url: uri }] }], + }); + expect(completionsWire.toBody(request('image/png'))).toMatchObject({ + messages: [ + { role: 'system' }, + { content: [{ type: 'text' }, { type: 'image_url', image_url: { url: uri } }] }, + ], + }); +}); + +it('marks no breakpoint on the completions shape, which caches without one', () => { + /* Measured live on 2026-09-04 against a prefix nobody had sent before, twice + for `openai/gpt-5.6-luna` and twice for `anthropic/claude-haiku-4.5`: the + second call read 12229 and 13630 cached tokens, the same to the token with + an Anthropic `cache_control` on the last block and without one. The field + bought nothing on this shape, so the shape stopped sending it. Re-run the + measurement before putting it back. */ + const body = assert<{ messages: { content: { cache_control?: unknown }[] }[] }>( + completionsWire.toBody(request('image/png')) + ); + + const marked = body.messages.flatMap(message => + message.content.filter(block => block.cache_control !== undefined) + ); + expect(marked).toEqual([]); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/wire/messages.ts b/packages/harness-sdk/src/plugins/gateway/wire/messages.ts new file mode 100644 index 0000000000..9a651a4563 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/messages.ts @@ -0,0 +1,300 @@ +import type Anthropic from '@anthropic-ai/sdk'; +import { createAssert, createIs } from 'typia'; +import type { ModelRequest, ModelUsage, StopReason } from '../../../core/model.js'; +import type { PromptMessage, PromptPart } from '../../../core/prompt.js'; +import type { ToolDefinition } from '../../../core/tool.js'; +import { isLast } from './parts.js'; +import { stopFrom, type Wire, type WirePart } from './wire.js'; +import { type Counts, set, type TokenCount } from './usage.js'; + +/** The Anthropic types are the contract. `cache_control` marks a breakpoint. */ +type ContentBlock = + | Anthropic.TextBlockParam + | Anthropic.ImageBlockParam + | Anthropic.ThinkingBlockParam + | Anthropic.RedactedThinkingBlockParam + | Anthropic.ToolUseBlockParam + | Anthropic.ToolResultBlockParam; +type MessagesBody = Anthropic.MessageCreateParams; +type MediaType = Anthropic.Base64ImageSource['media_type']; + +const ephemeral = { type: 'ephemeral' } as const; + +/** + * Anthropic names the four image types it takes. The stored media type is a + * plain string, so it is checked here rather than assumed: an image this shape + * cannot carry must say so, not be sent and refused. + */ +const assertMedia = createAssert(); + +/** The system prompt takes text only, so it has a builder of its own. */ +const textBlock = (text: string, cache: boolean): Anthropic.TextBlockParam => + cache ? { type: 'text', text, cache_control: ephemeral } : { type: 'text', text }; + +const imageBlock = (media: string, data: string, cache: boolean): ContentBlock => { + const source = { type: 'base64' as const, media_type: assertMedia(media), data }; + return cache ? { type: 'image', source, cache_control: ephemeral } : { type: 'image', source }; +}; + +/** + * A thinking block goes back exactly as it came, so it carries no breakpoint of + * its own: a `cache_control` this package added would be a change to the block. + */ +const thinkingBlock = (text: string, signature: string): Anthropic.ThinkingBlockParam => ({ + type: 'thinking', + thinking: text, + signature, +}); + +/** + * The arguments as this shape wants them: an object, not the text the model + * wrote. Text this cannot parse produced a call nothing could run, and its + * result already says so; an empty object keeps that exchange replayable, where + * throwing would make the session unusable forever over one bad call. + */ +const argumentsIn = (text: string): Record => { + try { + const parsed: unknown = JSON.parse(text); + return isArguments(parsed) ? parsed : {}; + } catch { + return {}; + } +}; + +const isArguments = createIs>(); + +const renderPart = (part: PromptPart, cache: boolean): ContentBlock | undefined => { + switch (part.kind) { + case 'toolCall': { + return { + type: 'tool_use', + id: part.callId, + name: part.name, + input: argumentsIn(part.arguments), + }; + } + case 'toolResult': { + return { + type: 'tool_result', + tool_use_id: part.callId, + content: part.body, + ...(part.failed ? { is_error: true } : {}), + }; + } + case 'text': { + return textBlock(part.text, cache); + } + case 'image': { + return imageBlock(part.media, part.data, cache); + } + case 'reasoning': { + /* Without a signature the provider refuses the block, so it is left out + rather than sent and rejected. */ + return part.signature === undefined ? undefined : thinkingBlock(part.text, part.signature); + } + case 'redacted': { + /* Already encrypted by the provider, so it needs no signature and must + go back byte for byte. */ + return { type: 'redacted_thinking', data: part.data }; + } + } +}; + +const renderMessage = ( + message: PromptMessage +): { role: 'user' | 'assistant'; content: ContentBlock[] } => ({ + role: message.role, + content: message.parts + .map((part, index) => renderPart(part, isLast(message, index))) + .filter(block => block !== undefined), +}); + +/** + * A tool, as this shape takes one. The schema goes across untouched: nothing + * here looks inside it, and a shape that rewrote it would change the prefix. + */ +const toolBlock = (tool: ToolDefinition): Anthropic.Tool => ({ + name: tool.name, + description: tool.description, + /* Rebuilt only because this SDK types the list as mutable. Every field goes + across as it was given, which is what the prefix depends on. */ + input_schema: { + ...tool.parameters, + type: 'object', + required: tool.parameters.required === undefined ? [] : [...tool.parameters.required], + }, +}); + +const toBody = ({ prompt, model, maxTokens, effort, tools }: ModelRequest): MessagesBody => ({ + model, + max_tokens: maxTokens, + stream: true, + ...(effort === undefined ? {} : { output_config: { effort } }), + /* Left out rather than sent empty: a shape that is offered no tool must not + be told there is an empty list of them, and an empty list would sit in the + prefix of a session that has none. */ + ...(tools === undefined || tools.length === 0 ? {} : { tools: tools.map(toolBlock) }), + system: prompt.system.map(part => textBlock(part.text, part.cache)), + messages: prompt.messages.map(renderMessage), +}); + +/** + * The shapes are matched structurally, not by a `type` discriminator: gateways + * relay a dozen models and only agree on where the numbers sit, not on how the + * frames are named. Extra fields are allowed; typia's `is` ignores them. + */ +interface WireUsage { + input_tokens?: TokenCount | null; + output_tokens?: TokenCount | null; + cache_read_input_tokens?: TokenCount | null; + cache_creation_input_tokens?: TokenCount | null; +} + +/** + * Why this shape says the model stopped. `pause_turn` is not mapped: nothing + * here has produced one, and naming it would claim a meaning nothing has + * tested. + * + * The keys are strings and not `Anthropic.StopReason`, because the SDK's union + * is behind its own documentation: at 0.104.1 it does not carry + * `model_context_window_exceeded`, which the provider documents and sends. + * Binding to the union would reject a name the provider actually uses. The two + * OpenAI shapes have no such gap; keeping all three the same way is worth more + * than one of them typechecking against a vendor list. + */ +const stopReasons: Readonly> = { + end_turn: 'end', + stop_sequence: 'end', + max_tokens: 'maxTokens', + /* A second wall, and the provider's own guidance is to treat it as + truncated: the answer stopped because the model's window filled, not + because `max_tokens` did. Both leave half a sentence, which is what + `maxTokens` names here. */ + model_context_window_exceeded: 'maxTokens', + refusal: 'refusal', + tool_use: 'tools', +}; + +const asStop = stopFrom(stopReasons); + +interface DeltaEvent { + delta: { text: string }; +} + +/** A thinking block streams under its own field, so the two never collide. */ +interface ThinkingEvent { + delta: { thinking: string }; +} + +/** + * The signature closes a thinking block. It arrives on its own event, with no + * thinking on it, so it is read on its own and never mixed into the text. + */ +interface SignatureEvent { + delta: { signature: string }; +} + +/** + * Thinking the provider encrypted. Unlike a thinking block it arrives whole, at + * the start of the block, and there is nothing to accumulate. + */ +interface RedactedEvent { + content_block: { type: 'redacted_thinking'; data: string }; +} + +/** The block that opens a call. The arguments follow it, a fragment at a time. */ +interface CallStartEvent { + content_block: { type: 'tool_use'; id: string; name: string }; +} + +/** One fragment of the arguments, as text. This shape streams them as JSON. */ +interface CallArgumentsEvent { + delta: { partial_json: string }; +} + +/** The end of any block. It closes a call when one is open, and nothing when not. */ +interface BlockEndEvent { + type: 'content_block_stop'; +} + +interface UsageEvent { + usage: WireUsage; +} + +/** `message_delta` carries the stop reason beside the output count. */ +interface StopEvent { + delta: { stop_reason: string }; +} + +/** `message_start` carries the input counts and `message_delta` the output count. */ +interface StartEvent { + message: UsageEvent; +} + +/** A stream event is an edge, so it is validated before the package believes it. */ +const isDelta = createIs(); +const isThinking = createIs(); +const isSignature = createIs(); +const isRedacted = createIs(); +const isCallStart = createIs(); +const isCallArguments = createIs(); +const isBlockEnd = createIs(); +const isUsage = createIs(); +const isStop = createIs(); +const isStart = createIs(); + +const readUsage = (usage: WireUsage): Partial => { + const counts: Counts = {}; + set(counts, 'inputTokens', usage.input_tokens); + set(counts, 'outputTokens', usage.output_tokens); + set(counts, 'cacheReadTokens', usage.cache_read_input_tokens); + set(counts, 'cacheWriteTokens', usage.cache_creation_input_tokens); + return counts; +}; + +const toDelta = (event: unknown): WirePart | undefined => { + if (isDelta(event)) { + return { kind: 'delta', text: event.delta.text }; + } + if (isThinking(event)) { + return { kind: 'reasoning', text: event.delta.thinking }; + } + if (isSignature(event)) { + return { kind: 'reasoning', text: '', signature: event.delta.signature }; + } + if (isRedacted(event)) { + return { kind: 'redacted', data: event.content_block.data }; + } + return toCall(event); +}; + +/** The three frames of a tool call, kept apart from the per-token path above. */ +const toCall = (event: unknown): WirePart | undefined => { + if (isCallStart(event)) { + return { kind: 'callStart', id: event.content_block.id, name: event.content_block.name }; + } + if (isCallArguments(event)) { + return { kind: 'callArguments', text: event.delta.partial_json }; + } + return isBlockEnd(event) ? { kind: 'callEnd' } : undefined; +}; + +const toUsage = (event: unknown): Partial | undefined => { + if (isStart(event)) { + return readUsage(event.message.usage); + } + return isUsage(event) ? readUsage(event.usage) : undefined; +}; + +const toStop = (event: unknown): StopReason | undefined => + isStop(event) ? asStop(event.delta.stop_reason) : undefined; + +const messagesWire: Wire = { + path: '/api/gateway/v1/messages', + toBody, + toDelta, + toUsage, + toStop, +}; + +export { messagesWire }; diff --git a/packages/harness-sdk/src/plugins/gateway/wire/parts.ts b/packages/harness-sdk/src/plugins/gateway/wire/parts.ts new file mode 100644 index 0000000000..65c3f429b9 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/parts.ts @@ -0,0 +1,34 @@ +import type { PromptMessage, PromptPart } from '../../../core/prompt.js'; + +/** + * What every shape needs from a message's parts. + * + * The cache breakpoint belongs to the message, not to a part, and a provider + * reads it on a block. So it goes on the last block of the message: everything + * before it is then inside the cached prefix. + */ +const isLast = (message: PromptMessage, index: number): boolean => + message.cache && index === message.parts.length - 1; + +/** + * An image as a data URI, which is how both OpenAI shapes take one. + * + * The bytes are already base64, because that is how a part is stored. Nothing + * is decoded and encoded again on the way out. + */ +const dataUri = (part: Extract): string => + `data:${part.media};base64,${part.data}`; + +/** + * How a failed result reads to a model on a shape that has no flag for one. + * + * The Anthropic shape marks a failed result on the block itself. Neither OpenAI + * shape has anywhere to put it, and a failure the model cannot tell from an + * answer is a failure it will build on, so it is said in the text instead. + */ +const failedText = (body: string): string => `The tool call failed. ${body}`; + +/** What a result reads as on a shape with no flag: the text says which it is. */ +const resultText = (body: string, failed: boolean): string => (failed ? failedText(body) : body); + +export { dataUri, isLast, resultText }; diff --git a/packages/harness-sdk/src/plugins/gateway/wire/render-fixture.ts b/packages/harness-sdk/src/plugins/gateway/wire/render-fixture.ts new file mode 100644 index 0000000000..c5ed20e5be --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/render-fixture.ts @@ -0,0 +1,20 @@ +import type { Prompt, PromptPart } from '../../../core/prompt.js'; +import type { Wire } from './wire.js'; + +/** + * What the wire tests share: a prompt of one assistant message, and the body a + * shape makes of it. It lives outside a `*.test.ts` file so more than one test + * file can use it, and it is excluded from `dist/` with the other test doubles. + */ + +/** What one shape puts on the wire for a prompt, so a test can read the blocks. */ +const bodyOf = (wire: Wire, prompt: Prompt): unknown => + wire.toBody({ prompt, model: 'm', maxTokens: 8 }); + +/** One assistant message of the given parts, with a system prompt in front. */ +const promptOf = (parts: readonly PromptPart[]): Prompt => ({ + system: [{ text: 'sys', cache: true }], + messages: [{ role: 'assistant', parts, cache: false }], +}); + +export { bodyOf, promptOf }; diff --git a/packages/harness-sdk/src/plugins/gateway/wire/replay.test.ts b/packages/harness-sdk/src/plugins/gateway/wire/replay.test.ts new file mode 100644 index 0000000000..1f4d5608fb --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/replay.test.ts @@ -0,0 +1,274 @@ +import { expect, it } from 'vitest'; +import type { PromptPart } from '../../../core/prompt.js'; +import { completionsWire } from './completions.js'; +import { messagesWire } from './messages.js'; +import { bodyOf, promptOf } from './render-fixture.js'; +import { responsesWire } from './responses.js'; + +/** + * How each shape puts a turn's thinking on the wire, and reads it back off. + * + * The three do not agree on where thinking lives, what seals it, or whether it + * can be handed back at all, so each is checked on its own terms. What the + * session does with the parts is `core/reasoning.test.ts`. + */ + +it('renders the thinking block the way the provider issued it', () => { + const body = bodyOf( + messagesWire, + promptOf([ + { kind: 'reasoning', text: 'first second', signature: 'sig_abc' }, + { kind: 'text', text: 'the answer' }, + ]) + ); + + expect(body).toMatchObject({ + messages: [ + { + role: 'assistant', + content: [ + { type: 'thinking', thinking: 'first second', signature: 'sig_abc' }, + { type: 'text', text: 'the answer' }, + ], + }, + ], + }); +}); + +it('keeps an encrypted block between the thinking blocks around it', () => { + /* The provider will not take the sequence rearranged. This is the whole + point of holding the thinking as an ordered list rather than as words + beside a list of encrypted blocks: the bytes on the wire come out in the + order the model produced them. */ + const body = bodyOf( + messagesWire, + promptOf([ + { kind: 'reasoning', text: 'before', signature: 'sig_one' }, + { kind: 'redacted', data: 'ENCRYPTED' }, + { kind: 'reasoning', text: 'after', signature: 'sig_two' }, + { kind: 'text', text: 'the answer' }, + ]) + ); + + expect(body).toMatchObject({ + messages: [ + { + content: [ + { type: 'thinking', thinking: 'before', signature: 'sig_one' }, + { type: 'redacted_thinking', data: 'ENCRYPTED' }, + { type: 'thinking', thinking: 'after', signature: 'sig_two' }, + { type: 'text', text: 'the answer' }, + ], + }, + ], + }); +}); + +it('leaves out a thinking block that has no signature', () => { + /* The provider refuses a block whose signature is missing, so a shape that + cannot prove the thinking is the model's own says nothing rather than + sending it and being refused. */ + const body = bodyOf( + messagesWire, + promptOf([ + { kind: 'reasoning', text: 'unsigned' }, + { kind: 'text', text: 'the answer' }, + ]) + ); + + expect(body).toMatchObject({ + messages: [{ content: [{ type: 'text', text: 'the answer' }] }], + }); +}); + +it('leaves the reasoning out of the chat shape, which cannot replay it', () => { + /* Providers relayed through this shape report their thinking under two + different field names and neither takes it back. This shape carries the + system prompt as its first message. */ + expect( + bodyOf( + completionsWire, + promptOf([ + { kind: 'reasoning', text: 'first second', signature: 'sig_abc' }, + { kind: 'text', text: 'the answer' }, + ]) + ) + ).toMatchObject({ + messages: [ + { role: 'system' }, + { role: 'assistant', content: [{ type: 'text', text: 'the answer' }] }, + ], + }); +}); + +it('replays the thinking of the responses shape as its own item', () => { + /* This shape does not carry thinking inside a message. It is an item beside + the message, holding the provider's own encrypted copy, and it goes first + because that is the order the model produced it in. */ + const seal = JSON.stringify({ id: 'rs_1', encrypted_content: 'ENCRYPTED' }); + const body = bodyOf( + responsesWire, + promptOf([ + { kind: 'reasoning', text: 'first second', signature: seal }, + { kind: 'text', text: 'the answer' }, + ]) + ); + + expect(body).toMatchObject({ + include: ['reasoning.encrypted_content'], + input: [ + { + type: 'reasoning', + id: 'rs_1', + encrypted_content: 'ENCRYPTED', + /* The summary stays empty. The provider sealed the item as it issued + it, and writing our own words into it would change what it sealed. */ + summary: [], + }, + { role: 'assistant', content: [{ type: 'input_text', text: 'the answer' }] }, + ], + }); +}); + +it('leaves out a responses reasoning item the provider never sealed', () => { + const body = bodyOf( + responsesWire, + promptOf([ + { kind: 'reasoning', text: 'unsealed' }, + { kind: 'text', text: 'the answer' }, + ]) + ); + + expect(body).toMatchObject({ + input: [{ role: 'assistant', content: [{ type: 'input_text', text: 'the answer' }] }], + }); +}); + +it('closes a responses reasoning block on the finished item', () => { + expect( + responsesWire.toDelta({ + type: 'response.output_item.done', + item: { type: 'reasoning', id: 'rs_1', encrypted_content: 'ENCRYPTED' }, + }) + ).toEqual({ + kind: 'reasoning', + text: '', + signature: JSON.stringify({ id: 'rs_1', encrypted_content: 'ENCRYPTED' }), + }); + + /* A finished message item is not a reasoning item. */ + expect( + responsesWire.toDelta({ + type: 'response.output_item.done', + item: { type: 'message', id: 'msg_1' }, + }) + ).toBeUndefined(); +}); + +it('tells the thinking of each shape apart from its answer', () => { + expect(messagesWire.toDelta({ delta: { thinking: 'hmm' } })).toEqual({ + kind: 'reasoning', + text: 'hmm', + }); + expect(messagesWire.toDelta({ delta: { text: 'said' } })).toEqual({ + kind: 'delta', + text: 'said', + }); + + /* The signature closes the block and arrives with no thinking on it. */ + expect(messagesWire.toDelta({ delta: { signature: 'sig_abc' } })).toEqual({ + kind: 'reasoning', + text: '', + signature: 'sig_abc', + }); + + expect( + responsesWire.toDelta({ type: 'response.reasoning_summary_text.delta', delta: 'hmm' }) + ).toEqual({ kind: 'reasoning', text: 'hmm' }); + + /* Two providers relayed through the same shape name the field differently. */ + expect(completionsWire.toDelta({ choices: [{ delta: { reasoning: 'hmm' } }] })).toEqual({ + kind: 'reasoning', + text: 'hmm', + }); + expect(completionsWire.toDelta({ choices: [{ delta: { reasoning_content: 'hmm' } }] })).toEqual({ + kind: 'reasoning', + text: 'hmm', + }); + expect(completionsWire.toDelta({ choices: [{ delta: { content: 'said' } }] })).toEqual({ + kind: 'delta', + text: 'said', + }); +}); + +it('reads the thinking off a frame whose content is present but empty', () => { + /* A provider puts `content: ""` on the frames that carry the thinking: the + empty string is a placeholder, not a word. Reading it as the answer would + shadow the thinking on the same frame. */ + expect( + completionsWire.toDelta({ choices: [{ delta: { content: '', reasoning: 'hmm' } }] }) + ).toEqual({ kind: 'reasoning', text: 'hmm' }); + expect( + completionsWire.toDelta({ choices: [{ delta: { content: '', reasoning_content: 'hmm' } }] }) + ).toEqual({ kind: 'reasoning', text: 'hmm' }); + + /* A frame whose content is empty and that carries nothing else says nothing. */ + expect(completionsWire.toDelta({ choices: [{ delta: { content: '' } }] })).toBeUndefined(); +}); + +it('renders an encrypted block as the provider named it', () => { + const body = bodyOf( + messagesWire, + promptOf([ + { kind: 'redacted', data: 'ENCRYPTED' }, + { kind: 'text', text: 'said' }, + ]) + ); + + expect(body).toMatchObject({ + messages: [ + { + content: [ + { type: 'redacted_thinking', data: 'ENCRYPTED' }, + { type: 'text', text: 'said' }, + ], + }, + ], + }); +}); + +it('reads an encrypted block off the stream', () => { + /* It arrives whole, at the start of the block, so there is nothing to + accumulate and nothing that could be split across two events. */ + expect( + messagesWire.toDelta({ + type: 'content_block_start', + content_block: { type: 'redacted_thinking', data: 'ENCRYPTED' }, + }) + ).toEqual({ kind: 'redacted', data: 'ENCRYPTED' }); + + /* An ordinary block start must not be read as one. */ + expect( + messagesWire.toDelta({ + type: 'content_block_start', + content_block: { type: 'thinking', thinking: '', signature: '' }, + }) + ).toBeUndefined(); +}); + +it('leaves an encrypted block out of the two shapes that cannot carry it', () => { + const parts: readonly PromptPart[] = [ + { kind: 'redacted', data: 'ENCRYPTED' }, + { kind: 'text', text: 'said' }, + ]; + + expect(bodyOf(responsesWire, promptOf(parts))).toMatchObject({ + input: [{ content: [{ type: 'input_text', text: 'said' }] }], + }); + expect(bodyOf(completionsWire, promptOf(parts))).toMatchObject({ + messages: [ + { role: 'system' }, + { role: 'assistant', content: [{ type: 'text', text: 'said' }] }, + ], + }); +}); diff --git a/packages/harness-sdk/src/plugins/gateway/wire/responses.ts b/packages/harness-sdk/src/plugins/gateway/wire/responses.ts new file mode 100644 index 0000000000..a4e8777b88 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/responses.ts @@ -0,0 +1,300 @@ +import type OpenAI from 'openai'; +import { createAssert, createIs } from 'typia'; +import type { ModelRequest, ModelUsage, StopReason } from '../../../core/model.js'; +import type { PromptMessage, PromptPart } from '../../../core/prompt.js'; +import type { ToolDefinition } from '../../../core/tool.js'; +import { dataUri, resultText } from './parts.js'; +import { stopFrom, type Wire, type WirePart } from './wire.js'; +import { readCached, type TokenCount } from './usage.js'; + +/** + * The OpenAI Responses shape. It has no cache breakpoint. It caches on + * `prompt_cache_key`, so the caller passes the session identifier and every + * request of one session lands on the same cache entry. + */ +type ResponsesBody = OpenAI.Responses.ResponseCreateParams; + +/** + * This shape has no cache breakpoint, so a part carries no mark. An image goes + * as a data URI, which is the only way this shape takes bytes. + * + * Thinking is not message content here. It is an item beside the message, so it + * is rendered by `itemsOf` rather than by this. + */ +const renderPart = (part: PromptPart): OpenAI.Responses.ResponseInputContent | undefined => { + switch (part.kind) { + case 'text': { + return { type: 'input_text', text: part.text }; + } + case 'image': { + return { type: 'input_image', image_url: dataUri(part), detail: 'auto' }; + } + /* A block the provider encrypted belongs to the Anthropic shape: this one + encrypts the whole reasoning item instead, and the two are not the same + bytes. A call and its result are items beside the message, not content in + it. All four are rendered by `itemsOf`, or by nothing at all. */ + case 'redacted': + case 'reasoning': + case 'toolCall': + case 'toolResult': { + return undefined; + } + } +}; + +/** + * A call, or its result, as the item this shape takes. The arguments go across + * as the text the model wrote, which is what this shape asks for, so a call + * that could not be run replays exactly as it was made. There is no flag for a + * failed result here, so the text says so instead. + */ +const toolItem = (part: PromptPart): OpenAI.Responses.ResponseInputItem | undefined => { + if (part.kind === 'toolCall') { + return { + type: 'function_call', + call_id: part.callId, + name: part.name, + arguments: part.arguments, + }; + } + return part.kind === 'toolResult' + ? { + type: 'function_call_output', + call_id: part.callId, + output: resultText(part.body, part.failed), + } + : undefined; +}; + +/** + * What this shape hides inside a signature: the item's identifier and the + * provider's encrypted copy of the reasoning. + * + * A signature is opaque to the package and means only what the shape that made + * it says it means. This one is not a signature at all; it is the two fields + * this shape needs to hand a reasoning item back. + */ +interface ReasoningSeal { + readonly id: string; + readonly encrypted_content: string; +} + +const assertSeal = createAssert(); + +/** + * The reasoning item to send back, or nothing when there is none to send. + * + * A malformed seal throws, which the caller turns into a failed call. It cannot + * be repaired and sending the item without it would be refused anyway. + */ +const reasoningItem = (part: PromptPart): OpenAI.Responses.ResponseReasoningItem | undefined => { + if (part.kind !== 'reasoning' || part.signature === undefined) { + return undefined; + } + const seal = assertSeal(JSON.parse(part.signature)); + return { + type: 'reasoning', + id: seal.id, + encrypted_content: seal.encrypted_content, + /* Empty because that is how the item arrived. The encrypted copy is the + authoritative one, and a summary this package wrote instead of the + provider is a change to a block the provider signed. */ + summary: [], + }; +}; + +/** + * One message, as the items this shape takes. + * + * The reasoning goes first, as its own item, which is the order the model + * produced it in and the order this shape reports it in. A message with nothing + * but reasoning produces no message item at all: an empty one is refused. + */ +const itemsOf = (message: PromptMessage): OpenAI.Responses.ResponseInputItem[] => { + const thinking = message.parts.map(reasoningItem).filter(item => item !== undefined); + const content = message.parts.map(renderPart).filter(part => part !== undefined); + const tools = message.parts.map(toolItem).filter(item => item !== undefined); + const said = content.length === 0 ? [] : [{ role: message.role, content }]; + return [...thinking, ...said, ...tools]; +}; + +/** + * A tool, as this shape takes one. `strict` is false because the schema comes + * from the caller unchanged: strict mode adds rules a hand-written schema will + * not always meet, and a rejected schema is a session that cannot start. + */ +const toolItemFor = (tool: ToolDefinition): OpenAI.Responses.FunctionTool => ({ + type: 'function', + name: tool.name, + description: tool.description, + parameters: { ...tool.parameters }, + strict: false, +}); + +const toBody = ({ + prompt, + model, + maxTokens, + cacheKey, + effort, + tools, +}: ModelRequest): ResponsesBody => ({ + model, + max_output_tokens: maxTokens, + stream: true, + ...(effort === undefined ? {} : { reasoning: { effort } }), + /* Without this the provider keeps the reasoning and hands back only an + identifier, which is no use to a package that stores the session itself. */ + include: ['reasoning.encrypted_content'], + store: false, + instructions: prompt.system.map(part => part.text).join('\n'), + ...(cacheKey === undefined ? {} : { prompt_cache_key: cacheKey }), + ...(tools === undefined || tools.length === 0 ? {} : { tools: tools.map(toolItemFor) }), + input: prompt.messages.flatMap(itemsOf), +}); + +interface Counts { + input_tokens: TokenCount; + output_tokens: TokenCount; + input_tokens_details?: { cached_tokens?: TokenCount | null } | null; +} + +/** + * Why this shape says the model stopped. A finished response says so in its + * status; an unfinished one names the wall it hit. + */ +const incompleteReasons: Readonly> = { + max_output_tokens: 'maxTokens', + content_filter: 'refusal', +}; + +const asIncomplete = stopFrom(incompleteReasons); + +const asStop = (status: string | null | undefined, reason: string | null | undefined): StopReason => + status === 'completed' ? 'end' : (asIncomplete(reason) ?? 'unknown'); + +/** This shape names its frames, so the two events are matched on `type`. */ +interface DeltaEvent { + type: 'response.output_text.delta'; + delta: string; +} + +/** + * The frame that carries the counts, which is either frame that ends an answer: + * one stopped at the wall reports them exactly as a finished one does. Reading + * only the finished one counts a truncated answer as zero, and the session then + * believes its window is empty. + */ +interface CompletedEvent { + type: 'response.completed' | 'response.incomplete'; + response: { usage: Counts }; +} + +/** The frame that says the answer ended, and names the wall when it hit one. */ +interface EndEvent { + type: 'response.completed' | 'response.incomplete' | 'response.failed'; + response: { status?: string | null; incomplete_details?: { reason?: string | null } | null }; +} +/** + * This shape streams the thinking under a name of its own, and under two of + * them: `reasoning_summary_text` when the provider returns a summary, and + * `reasoning` when it returns the thinking itself. The kilo gateway relays + * Anthropic through this shape and sends the second. + */ +interface ReasoningEvent { + type: 'response.reasoning_summary_text.delta' | 'response.reasoning.delta'; + delta: string; +} + +/** + * The finished reasoning item, which is where the encrypted copy arrives. The + * summary streamed in pieces before it; this closes the block, the way a + * signature does on the Anthropic shape. + */ +interface ReasoningDoneEvent { + type: 'response.output_item.done'; + item: { type: 'reasoning'; id: string; encrypted_content: string }; +} + +/** The item that opens a call. `call_id` names the call; `id` names the item. */ +interface CallStartEvent { + type: 'response.output_item.added'; + item: { type: 'function_call'; call_id: string; name: string }; +} + +/** One fragment of the arguments, as text. */ +interface CallArgumentsEvent { + type: 'response.function_call_arguments.delta'; + delta: string; +} + +/** The finished call item. The arguments already streamed, so this only closes. */ +interface CallEndEvent { + type: 'response.output_item.done'; + item: { type: 'function_call' }; +} + +const isDelta = createIs(); +const isCallStart = createIs(); +const isCallArguments = createIs(); +const isCallEnd = createIs(); +const isCompleted = createIs(); +const isEnd = createIs(); +const isReasoning = createIs(); +const isReasoningDone = createIs(); + +const readUsage = (usage: Counts): Partial => + readCached( + usage.input_tokens, + usage.output_tokens, + usage.input_tokens_details?.cached_tokens ?? 0 + ); + +const toDelta = (event: unknown): WirePart | undefined => { + if (isDelta(event)) { + return { kind: 'delta', text: event.delta }; + } + if (isReasoning(event)) { + return { kind: 'reasoning', text: event.delta }; + } + if (!isReasoningDone(event)) { + return toCall(event); + } + const seal: ReasoningSeal = { + id: event.item.id, + encrypted_content: event.item.encrypted_content, + }; + return { kind: 'reasoning', text: '', signature: JSON.stringify(seal) }; +}; + +/** The three frames of a tool call, kept apart from the per-token path above. */ +const toCall = (event: unknown): WirePart | undefined => { + if (isCallStart(event)) { + return { kind: 'callStart', id: event.item.call_id, name: event.item.name }; + } + if (isCallArguments(event)) { + return { kind: 'callArguments', text: event.delta }; + } + return isCallEnd(event) ? { kind: 'callEnd' } : undefined; +}; + +const toUsage = (event: unknown): Partial | undefined => + isCompleted(event) ? readUsage(event.response.usage) : undefined; + +const toStop = (event: unknown): StopReason | undefined => + isEnd(event) + ? asStop( + event.response.status ?? (event.type === 'response.completed' ? 'completed' : undefined), + event.response.incomplete_details?.reason + ) + : undefined; + +const responsesWire: Wire = { + path: '/api/gateway/v1/responses', + toBody, + toDelta, + toUsage, + toStop, +}; + +export { responsesWire }; diff --git a/packages/harness-sdk/src/plugins/gateway/wire/usage.ts b/packages/harness-sdk/src/plugins/gateway/wire/usage.ts new file mode 100644 index 0000000000..0bcfcfc807 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/usage.ts @@ -0,0 +1,34 @@ +import type { tags } from 'typia'; +import type { ModelUsage } from '../../../core/model.js'; + +/** + * A token count as a provider reports it. `uint32` is what makes the check + * reject `NaN` and `Infinity`: `JSON.parse` turns an overflowing literal into + * `Infinity`, and a bare `number` accepts it, which then poisons every later + * sum with `NaN`. + */ +type TokenCount = number & tags.Type<'uint32'>; + +type Counts = { -readonly [K in keyof ModelUsage]?: number }; + +/** Writes a count only when the reply carried one. A missing count is not a zero. */ +const set = (target: Counts, key: keyof ModelUsage, value: number | null | undefined): void => { + const count = value ?? undefined; + if (count !== undefined) { + target[key] = count; + } +}; + +/** + * Both OpenAI shapes report the cached tokens inside the input total, so the + * cached count is subtracted out. They differ only in what the three fields + * are called. + */ +const readCached = (input: number, output: number, cached: number): Partial => ({ + inputTokens: input - cached, + outputTokens: output, + cacheReadTokens: cached, +}); + +export type { Counts, TokenCount }; +export { readCached, set }; diff --git a/packages/harness-sdk/src/plugins/gateway/wire/wire.ts b/packages/harness-sdk/src/plugins/gateway/wire/wire.ts new file mode 100644 index 0000000000..35114710fb --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wire/wire.ts @@ -0,0 +1,98 @@ +import { createIs } from 'typia'; +import type { ModelRequest, ModelUsage, StopReason } from '../../../core/model.js'; + +/** + * What one streamed event says, when it says anything. + * + * A signature arrives on its own event, after the thinking and with no text, + * because that is how a provider streams it. + * + * `redacted` is thinking the provider encrypted rather than showed. It has no + * text at all, and it is a kind of its own so that nothing can render its bytes + * as words by mistake. + * + * The first three are a `ModelEvent` as they stand, so a part reaches the caller + * unchanged and no conversion sits on the per-token path. + * + * The last three are not, and cannot be. A tool call arrives in pieces on every + * shape — the name first, then the arguments a fragment at a time — and no + * shape sends the same pieces as another. So each shape reports the pieces it + * has and the gateway collects them into one `toolCall`, which is the only form + * anything above the transport ever sees. See `openCall` in the gateway. + * + * `callEnd` closes whatever call is open. Two shapes send an event that means + * exactly that; the third closes a call by starting the next one or by ending + * the stream, so it sends none and the gateway closes it either way. + */ +type WirePart = + | { readonly kind: 'delta'; readonly text: string } + | { readonly kind: 'reasoning'; readonly text: string; readonly signature?: string } + | { readonly kind: 'redacted'; readonly data: string } + | { + readonly kind: 'callStart'; + readonly id: string; + readonly name: string; + /** + * Arguments the opening frame already carried. One shape puts the name + * and the first fragment on the same frame, and a reader that returns one + * part per frame has nowhere else to put it. + */ + readonly text?: string; + } + | { readonly kind: 'callArguments'; readonly text: string } + | { readonly kind: 'callEnd' }; + +/** + * One gateway shape. A wire maps a request onto a body and maps the reply back. + * `toBody` throws when a shape cannot carry what it was given; the caller wraps + * it. + * + * A stream event is an edge, so every reader below validates what it finds. + * One event carries text, or reasoning, or token counts, or a stop reason, or + * nothing at all. + */ +interface Wire { + readonly path: string; + readonly toBody: (request: ModelRequest) => unknown; + readonly toDelta: (event: unknown) => WirePart | undefined; + readonly toUsage: (event: unknown) => Partial | undefined; + /** Absent until the event that says why the model stopped. */ + readonly toStop: (event: unknown) => StopReason | undefined; +} + +/** + * Reads a stop reason a shape reports by name. A name the table does not hold + * is `unknown` rather than nothing: the model did stop, and this package + * simply has no word for why. No name at all is nothing, because that frame + * was not the one that said. + */ +const stopFrom = + (reasons: Readonly>) => + (named: string | null | undefined): StopReason | undefined => + named === null || named === undefined ? undefined : (reasons[named] ?? 'unknown'); + +/** + * A failure the provider reported after the answer started. + * + * All three shapes mark one the same way, with an `error` object on the frame, + * so it is read once here rather than per shape. Anthropic's streaming + * reference says so outright: the API may send an error in the event stream, + * such as an `overloaded_error` that would have been a 529 had the call not + * been streamed. Letting the frame pass ends the stream on `done` and stores a + * fragment as a whole answer. + * + * The responses shape is the exception: it carries the failure one level down, + * as `response.error` on a `response.failed` frame, so both places are read. + * + * Only an `error` object counts. That shape reports `error: null` inside + * `response` on every call that worked, and null is not an object, so a reply + * that succeeded does not match. + */ +type FailureEvent = + | { error: { message?: string; type?: string } } + | { response: { error: { message?: string; code?: string } } }; + +const isFailure = createIs(); + +export type { Wire, WirePart }; +export { isFailure, stopFrom }; diff --git a/packages/harness-sdk/src/plugins/gateway/wires.ts b/packages/harness-sdk/src/plugins/gateway/wires.ts new file mode 100644 index 0000000000..722b7d4c90 --- /dev/null +++ b/packages/harness-sdk/src/plugins/gateway/wires.ts @@ -0,0 +1,35 @@ +import { Effect } from 'effect'; +import { pickKind } from './api-kind.js'; +import { type ApiKind, CatalogError, type ModelCatalogService } from '../../core/catalog.js'; +import { ModelError } from '../../core/model.js'; +import { completionsWire } from './wire/completions.js'; +import { messagesWire } from './wire/messages.js'; +import { responsesWire } from './wire/responses.js'; +import type { Wire } from './wire/wire.js'; + +/** Keyed by `ApiKind`, so picking a kind always finds a wire. */ +const wires: Readonly> = { + messages: messagesWire, + responses: responsesWire, + chat_completions: completionsWire, +}; + +/** A catalog that cannot answer is a model the package must not guess at. */ +const asModelError = (error: CatalogError | ModelError): ModelError => + error instanceof CatalogError + ? new ModelError({ reason: 'unsupported', cause: error.cause }) + : error; + +/** Asks the catalog what the model speaks and returns the best wire for it. */ +const wireFor = (catalog: ModelCatalogService, model: string): Effect.Effect => + catalog.facts(model).pipe( + Effect.mapError(asModelError), + Effect.flatMap(facts => { + const kind = pickKind(facts.apiKinds); + return kind === undefined + ? Effect.fail(new ModelError({ reason: 'unsupported', cause: model })) + : Effect.succeed(wires[kind]); + }) + ); + +export { wireFor, wires }; diff --git a/packages/harness-sdk/src/plugins/kilo.test.ts b/packages/harness-sdk/src/plugins/kilo.test.ts new file mode 100644 index 0000000000..7fed67d0c7 --- /dev/null +++ b/packages/harness-sdk/src/plugins/kilo.test.ts @@ -0,0 +1,93 @@ +import { Effect, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import type { ModelFacts } from '../core/catalog.js'; +import { openSession } from '../core/run.js'; +import { fakeFetch, type Reply, sse } from './gateway/fake.js'; +import type { TokenSourceService } from '../core/token.js'; +import { layerKilo } from './kilo.js'; + +/** + * The composed layer. What is proved here is that one call gives a session + * everything it asks for, and that the request that comes out is the one the + * hand-wired layers made. How each plugin behaves is that plugin's own test. + */ + +const answer: Reply = { + ok: true, + status: 200, + body: '', + chunks: sse( + { type: 'message_start', message: { usage: { input_tokens: 5 } } }, + { type: 'content_block_delta', delta: { text: 'hello' } }, + { type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 1 } } + ), +}; + +const ask = (token: string | TokenSourceService, facts?: ModelFacts) => { + const { fetch, calls } = fakeFetch([answer]); + const layers = layerKilo({ + baseUrl: 'https://gateway.test', + org: { kind: 'organization', id: 'org_1' }, + fetch, + token, + ...(facts === undefined ? {} : { fallback: facts }), + }); + const said = Effect.scoped( + Effect.flatMap(openSession({ system: 'sys', model: 'm', maxTokens: 8 }), session => + Stream.runFold(session.ask('hi'), '', (held, event) => + event.kind === 'delta' ? held + event.text : held + ) + ) + ); + return Effect.map(Effect.provide(said, layers), text => ({ text, calls })); +}; + +it('gives a session every plugin it asks for, in one call', async () => { + const { text } = await Effect.runPromise(ask('tok_1')); + + expect(text).toBe('hello'); +}); + +it('sends the token and the organization the caller named', async () => { + const { calls } = await Effect.runPromise(ask('tok_2')); + + expect(calls[0]?.request.headers).toMatchObject({ + authorization: 'Bearer tok_2', + 'x-kilocode-organizationid': 'org_1', + }); +}); + +it('sends a model it knows nothing about to the best shape there is', async () => { + /* No catalog at all. The gateway relays every model through all three + shapes, so assuming all three and picking the best one is what a caller + who names nothing wants. */ + const { calls } = await Effect.runPromise(ask('tok_3')); + + expect(calls[0]?.url).toBe('https://gateway.test/api/gateway/v1/messages'); +}); + +it('sends the model to the shape its catalog names', async () => { + const { calls } = await Effect.runPromise(ask('tok_4', { apiKinds: ['chat_completions'] })); + + expect(calls[0]?.url).toBe('https://gateway.test/api/gateway/v1/chat/completions'); +}); + +it('asks a token source for the credential rather than holding a string', async () => { + /* The credential is the one plugin a long-lived caller has to replace, and + the kilo token expires. Rewriting `layerKilo` to replace it means + rebuilding the shared catalog by hand, which is the trap `layerKilo` + closes, so the option takes a source too. */ + let asked = 0; + const minting: TokenSourceService = { + get: () => + Effect.sync(() => { + asked += 1; + return `tok_${String(asked)}`; + }), + }; + + const { calls } = await Effect.runPromise(ask(minting)); + + expect(asked).toBe(1); + expect(calls[0]?.request.headers['authorization']).toBe('Bearer tok_1'); +}); diff --git a/packages/harness-sdk/src/plugins/kilo.ts b/packages/harness-sdk/src/plugins/kilo.ts new file mode 100644 index 0000000000..c27d28379b --- /dev/null +++ b/packages/harness-sdk/src/plugins/kilo.ts @@ -0,0 +1,100 @@ +import { Layer } from 'effect'; +import { + type EntropyError, + type EntropySource, + type ModelCatalog, + type ModelClient, + type ModelFacts, + type PromptAssembler, + TokenSource, + type TokenSourceService, +} from '../core/index.js'; +import { layerTableCatalog } from './catalog/table.js'; +import { layerWebCrypto } from './entropy/web-crypto.js'; +import { layerKiloGateway, type KiloGatewayConfig } from './gateway/index.js'; +import { layerAssembler } from './prompt/default.js'; +import { layerBackoff } from './retry/backoff.js'; +import { layerStaticToken } from './token/static.js'; + +/** + * What a session needs, in one call. + * + * The pieces are still plugins and every one of them can be wired by hand. + * This is the wiring almost every caller writes, so the package writes it: the + * five layers, in the one order that works, with the catalog shared rather + * than nested. + * + * `token` takes a source as well as a string, because the credential is the + * one plugin every long-lived caller has to replace, and rewriting this + * function to replace it means rebuilding the shared catalog by hand. A caller + * who needs another plugin — a catalog that asks the gateway, entropy from + * somewhere other than the global `crypto` — composes the layers themselves. + */ +interface KiloSetup extends KiloGatewayConfig { + /** + * One token for the life of the process, or a source that is asked per call. + * + * A string is right for a short run and for a token that outlives it. A + * long-lived session wants the source: the kilo token expires, and a session + * that outlives it starts failing with 401 while holding a string it still + * believes in. Read `TokenSource` before writing one — a source that reads + * its state while building the effect hands the same stale credential to + * every retry. + */ + readonly token: string | TokenSourceService; + /** What each model can do. A model the table does not name uses `fallback`. */ + readonly models?: Readonly>; + /** + * What to assume about a model the table does not name. By default, that it + * speaks all three shapes, which the gateway's relayed models do — the best + * one a model actually speaks is picked from this list, so naming all three + * costs nothing and naming none would refuse every model. + * + * It says nothing about a context window, so a session wired this way never + * compacts. A caller who wants compaction names the window here, or writes + * the model into `models`. + */ + readonly fallback?: ModelFacts; + /** How many times a failed call is tried again. */ + readonly retries?: number; +} + +/** + * Everything `openSession` asks for, apart from the scope the caller opens. + * + * The error is the web-crypto source's: a runtime with no global `crypto` + * fails when the layer is built, which is where a caller can still choose a + * different source. + */ +type KiloLayer = Layer.Layer< + PromptAssembler | EntropySource | ModelCatalog | ModelClient, + EntropyError +>; + +/** + * The catalog is built once and given to both the session and the gateway. + * Building it twice typechecks and answers the same, but the gateway then + * holds a different instance from the one the session reads, which is a trap + * worth closing here rather than documenting. + */ +/** Best first, which is how `wireFor` reads the list. */ +const everyShape: ModelFacts = { apiKinds: ['messages', 'responses', 'chat_completions'] }; + +const layerKilo = (setup: KiloSetup): KiloLayer => { + const catalog = layerTableCatalog(setup.models ?? {}, setup.fallback ?? everyShape); + const token = + typeof setup.token === 'string' + ? layerStaticToken(setup.token) + : Layer.succeed(TokenSource, setup.token); + return Layer.mergeAll( + layerAssembler, + layerWebCrypto, + catalog, + layerKiloGateway(setup).pipe( + Layer.provide(Layer.mergeAll(catalog, token, layerBackoff(setup.retries))) + ) + ); +}; + +export type { KiloSetup }; +export { layerKilo }; diff --git a/packages/harness-sdk/src/plugins/model/fake.ts b/packages/harness-sdk/src/plugins/model/fake.ts new file mode 100644 index 0000000000..0e0fcaee4b --- /dev/null +++ b/packages/harness-sdk/src/plugins/model/fake.ts @@ -0,0 +1,92 @@ +import { Layer, Stream } from 'effect'; +import { + ModelClient, + type ModelError, + type ModelEvent, + type ModelRequest, + type ModelUsage, + type StopReason, + zeroUsage, +} from '../../core/model.js'; +import type { ToolCall } from '../../core/tool.js'; + +/** + * One scripted answer. `fail` ends the stream after the deltas it lists. + * + * `reasoning` streams before the deltas, which is the order a model produces + * them in. + */ +interface FakeReply { + readonly deltas: readonly string[]; + readonly reasoning?: readonly string[]; + readonly usage?: Partial; + readonly fail?: ModelError; + /** Closes the thinking, the way a provider does, on its own event. */ + readonly signature?: string; + /** Why the model stopped. A scripted answer finished unless it says so. */ + readonly stop?: StopReason; + /** Thinking the provider encrypted, streamed whole, ahead of the deltas. */ + readonly redacted?: readonly string[]; + /** + * Tools the model asks for, whole, after the words. A reply that lists any + * must say `stop: 'tools'` too, the way a shape reports one. + */ + readonly calls?: readonly ToolCall[]; + /** Never reaches `done`, so a test can interrupt the stream part way. */ + readonly stall?: boolean; + /** + * The events to stream, in this order, instead of the fields above. It is + * how a test scripts an order the fields cannot express, such as thinking + * that is interrupted by a block the provider encrypted. + */ + readonly events?: readonly ModelEvent[]; +} + +/** + * A model that answers from a script and records what it was asked. It is how + * this package tests a session without a network and without spending credit. + * + * It is not exported: `dist/` carries no test double. A consumer who wants one + * writes two functions against `ModelClientService`, which is the whole of the + * plugin point. Ship this instead the day somebody asks for it. + */ +const fakeModel = ( + replies: readonly FakeReply[] +): { readonly calls: ModelRequest[]; readonly layer: Layer.Layer } => { + const calls: ModelRequest[] = []; + const nextReply = (request: ModelRequest): FakeReply => { + calls.push(request); + return replies[Math.min(calls.length - 1, replies.length - 1)] ?? { deltas: [] }; + }; + + const stream = (request: ModelRequest): Stream.Stream => { + const reply = nextReply(request); + const deltas = Stream.fromIterable( + reply.events ?? [ + ...(reply.redacted ?? []).map((data): ModelEvent => ({ kind: 'redacted', data })), + ...(reply.reasoning ?? []).map((text): ModelEvent => ({ kind: 'reasoning', text })), + ...(reply.signature === undefined + ? [] + : [{ kind: 'reasoning', text: '', signature: reply.signature } as ModelEvent]), + ...reply.deltas.map((text): ModelEvent => ({ kind: 'delta', text })), + ...(reply.calls ?? []).map((call): ModelEvent => ({ kind: 'toolCall', call })), + ] + ); + const done = Stream.succeed({ + kind: 'done', + usage: { ...zeroUsage, ...reply.usage }, + stop: reply.stop ?? 'end', + }); + if (reply.stall === true) { + return Stream.concat(deltas, Stream.never); + } + return reply.fail === undefined + ? Stream.concat(deltas, done) + : Stream.concat(deltas, Stream.fail(reply.fail)); + }; + + return { calls, layer: Layer.succeed(ModelClient, { stream }) }; +}; + +export type { FakeReply }; +export { fakeModel }; diff --git a/packages/harness-sdk/src/plugins/prompt/default.ts b/packages/harness-sdk/src/plugins/prompt/default.ts new file mode 100644 index 0000000000..e3589ed06a --- /dev/null +++ b/packages/harness-sdk/src/plugins/prompt/default.ts @@ -0,0 +1,66 @@ +import { Layer } from 'effect'; +import { + PromptAssembler, + type Prompt, + type PromptInput, + type PromptPart, +} from '../../core/prompt.js'; +import type { TurnPart } from '../../core/turn.js'; + +/** + * Maps one turn part onto what the transport sends. + * + * Reasoning goes back out with the rest. The provider drops what the model + * cannot read and does not bill for it, and a block removed by hand can fail + * the request on ordering or on its signature, so the package hands back what + * it was given and lets the provider decide. + */ +const renderPart = (part: TurnPart): readonly PromptPart[] => { + switch (part.kind) { + /* A summary is text to the model. It is a kind of its own only so the + session can find where the prompt starts. */ + case 'summary': + case 'text': { + return [{ kind: 'text', text: part.body }]; + } + case 'image': { + return [{ kind: 'image', media: part.media, data: part.body }]; + } + case 'redacted': { + return [{ kind: 'redacted', data: part.body }]; + } + case 'reasoning': { + return [ + { + kind: 'reasoning', + text: part.body, + ...(part.signature === undefined ? {} : { signature: part.signature }), + }, + ]; + } + case 'toolCall': { + return [{ kind: 'toolCall', callId: part.callId, name: part.name, arguments: part.body }]; + } + case 'toolResult': { + return [{ kind: 'toolResult', callId: part.callId, body: part.body, failed: part.failed }]; + } + } +}; + +/** + * The core assembler. It sets two breakpoints: one after the system prompt, + * which every request of every session reads, and one on the last turn, which + * the next request of this session reads. + */ +const assemble = ({ system, turns }: PromptInput): Prompt => ({ + system: [{ text: system, cache: true }], + messages: turns.map((turn, index) => ({ + role: turn.role, + parts: turn.parts.flatMap(renderPart), + cache: index === turns.length - 1, + })), +}); + +const layerAssembler: Layer.Layer = Layer.succeed(PromptAssembler, { assemble }); + +export { assemble, layerAssembler }; diff --git a/packages/harness-sdk/src/plugins/retry/backoff.ts b/packages/harness-sdk/src/plugins/retry/backoff.ts new file mode 100644 index 0000000000..6654cec2be --- /dev/null +++ b/packages/harness-sdk/src/plugins/retry/backoff.ts @@ -0,0 +1,32 @@ +import { Layer, Schedule } from 'effect'; +import { RetryPolicy } from '../../core/retry.js'; +import type { ModelError } from '../../core/model.js'; + +/** A status the gateway or the network may recover from on its own. */ +const retryStatuses = new Set([408, 409, 425, 429, 500, 502, 503, 504]); + +const isRetryable = (error: ModelError): boolean => + error.reason === 'transport' || + (error.reason === 'status' && retryStatuses.has(error.status ?? 0)); + +/** + * The core policy: exponential backoff with jitter, up to `retries` further + * attempts. Jitter matters because a gateway outage makes every session retry + * on the same beat, and an unjittered fleet retries as one. + */ +const backoff = (retries: number): Schedule.Schedule => + Schedule.exponential('200 millis').pipe( + Schedule.jittered, + Schedule.whileInput(isRetryable), + Schedule.intersect(Schedule.recurs(retries)) + ); + +const layerBackoff = (retries = 3): Layer.Layer => + Layer.succeed(RetryPolicy, { schedule: backoff(retries) }); + +/** Tries once and gives up. Useful when the caller owns its own retry loop. */ +const layerNoRetry: Layer.Layer = Layer.succeed(RetryPolicy, { + schedule: Schedule.stop, +}); + +export { backoff, layerBackoff, layerNoRetry }; diff --git a/packages/harness-sdk/src/plugins/store/connection.test.ts b/packages/harness-sdk/src/plugins/store/connection.test.ts new file mode 100644 index 0000000000..dfc3fad819 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/connection.test.ts @@ -0,0 +1,47 @@ +import { DatabaseSync } from 'node:sqlite'; +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { SessionStore } from '../../core/storage.js'; +import { textOf } from '../../core/turn.js'; +import { layerNodeStore } from './node.js'; + +/** + * A session and its subagent share one connection by design, and both write to + * it while the other is mid-write. `driver.ts` says what SQLite does to two + * transactions on one connection; this is the proof that neither loses its rows. + */ + +const session = { id: 'ses_1', system: 'sys', model: 'claude-opus-5' }; + +const turnFor = (id: string) => ({ + sessionId: id, + turns: [ + { + id: `trn_${id}`, + sessionId: id, + role: 'user' as const, + parts: [{ id: `prt_${id}`, kind: 'text' as const, body: `said by ${id}` }], + }, + ], + prompted: 0, +}); + +it('writes two sessions at once over one connection without losing either', async () => { + const loaded = await Effect.runPromise( + Effect.provide( + Effect.flatMap(SessionStore, store => + Effect.gen(function* () { + yield* store.create(session); + yield* store.create({ ...session, id: 'ses_2' }); + yield* Effect.all([store.append(turnFor('ses_1')), store.append(turnFor('ses_2'))], { + concurrency: 'unbounded', + }); + return yield* Effect.all([store.load('ses_1'), store.load('ses_2')]); + }) + ), + layerNodeStore(new DatabaseSync(':memory:')) + ) + ); + + expect(loaded.map(turns => turns.map(textOf))).toEqual([['said by ses_1'], ['said by ses_2']]); +}); diff --git a/packages/harness-sdk/src/plugins/store/driver.ts b/packages/harness-sdk/src/plugins/store/driver.ts new file mode 100644 index 0000000000..96830acdb8 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/driver.ts @@ -0,0 +1,68 @@ +/** + * The seam every platform adapter fills: run this SQL with these parameters and + * give back the rows. + * + * That is the whole of it, which is why `node.ts` and `expo.ts` are about twenty + * lines each and share every query in this folder. + */ + +/** What this store binds to a statement. It holds text and numbers, nothing else. */ +type SqlValue = string | number | null; + +/** + * Rows come back by position, not by name, which is the shape drizzle maps onto + * the selected columns. + */ +type SqlDriver = ( + sql: string, + params: readonly SqlValue[], + method: 'run' | 'all' | 'values' | 'get' +) => Promise<{ rows: unknown[] }>; + +/** + * A driver and the line every write on it stands in. + * + * Every adapter is async, so each `await` inside a transaction lets another + * caller's statement in — and a session and its subagent share one connection by + * design. SQLite cannot start a transaction inside a transaction: the second + * `BEGIN` throws, its `ROLLBACK` takes the first writer's rows with it, and the + * first then commits nothing. A plain write that lands inside somebody else's + * transaction is undone with it the same way. So the writes queue. + * + * Reads stay out of the line. A session holds itself while it writes, so no read + * asks for the rows being written, and a load has no reason to wait behind them. + */ +interface Connection { + readonly driver: SqlDriver; + readonly write: (work: () => Promise) => Promise; +} + +const connectionOf = (driver: SqlDriver): Connection => { + let tail: Promise = Promise.resolve(); + return { + driver, + write: (work: () => Promise): Promise => { + const next = tail.then(work, work); + /* The line goes on after a write that failed, and the failure belongs to + the caller who asked for it rather than to the next one waiting. */ + tail = Promise.allSettled([next]); + return next; + }, + }; +}; + +/** Runs the work as one unit, so a process that dies part way leaves nothing half done. */ +const transact = (connection: Connection, run: () => Promise): Promise => + connection.write(async () => { + await connection.driver('BEGIN', [], 'run'); + try { + await run(); + await connection.driver('COMMIT', [], 'run'); + } catch (error) { + await connection.driver('ROLLBACK', [], 'run'); + throw error; + } + }); + +export type { Connection, SqlDriver, SqlValue }; +export { connectionOf, transact }; diff --git a/packages/harness-sdk/src/plugins/store/expo.test.ts b/packages/harness-sdk/src/plugins/store/expo.test.ts new file mode 100644 index 0000000000..3662f7ec01 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/expo.test.ts @@ -0,0 +1,111 @@ +import { DatabaseSync } from 'node:sqlite'; +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { SessionStore, type SessionStoreService } from '../../core/storage.js'; +import { expoDriver, type ExpoDatabase } from './expo.js'; +import type { SqlValue } from './driver.js'; +import { layerSqliteStore } from './sqlite.js'; +import { textOf } from '../../core/turn.js'; + +/** + * Expo's SQLite is a native module, so it cannot run here. What can run is the + * adapter: a database of the same shape, backed by Node's SQLite, proves that + * the raw rows land in the right columns and that every statement is finalized. + * + * The shapes cannot drift apart unnoticed. `layerExpoStore` takes Expo's own + * `SQLiteDatabase`, so the compiler checks a real database against the same + * interface this double implements. + */ +const standIn = (): { readonly database: ExpoDatabase; readonly open: () => number } => { + const sqlite = new DatabaseSync(':memory:'); + const live = { count: 0 }; + const database: ExpoDatabase = { + prepareSync: (source: string) => { + const statement = sqlite.prepare(source); + live.count += 1; + return { + executeSync: (params: SqlValue[]) => statement.run(...params), + executeForRawResultSync: (params: SqlValue[]) => ({ + /* Expo answers a raw query by position, so the double does too. */ + getAllSync: () => statement.all(...params).map(Object.values), + }), + finalizeSync: () => { + live.count -= 1; + }, + }; + }, + }; + return { database, open: () => live.count }; +}; + +const session = { id: 'ses_1', system: 'sys', model: 'claude-opus-5', effort: 'high' } as const; + +const use = ( + database: ExpoDatabase, + run: (store: SessionStoreService) => Effect.Effect +): Promise => + Effect.runPromise( + Effect.provide(Effect.flatMap(SessionStore, run), layerSqliteStore(expoDriver(database))) + ); + +it('carries a session and its turns through the Expo shape', async () => { + const { database } = standIn(); + + const read = await use(database, store => + Effect.gen(function* () { + yield* store.create(session); + yield* store.append({ + sessionId: 'ses_1', + turns: [ + { + id: 'trn_1', + sessionId: 'ses_1', + role: 'user', + parts: [{ id: `prt_${String('hello')}`, kind: 'text', body: 'hello' }], + }, + ], + prompted: 0, + }); + yield* store.append({ + sessionId: 'ses_1', + turns: [ + { + id: 'trn_2', + sessionId: 'ses_1', + role: 'assistant', + parts: [{ id: `prt_${String('hi')}`, kind: 'text', body: 'hi' }], + }, + ], + prompted: 41, + }); + return { options: yield* store.read('ses_1'), turns: yield* store.load('ses_1') }; + }) + ); + + expect(read.options).toMatchObject({ _tag: 'Some', value: { ...session, prompted: 41 } }); + expect(read.turns.map(textOf)).toEqual(['hello', 'hi']); +}); + +it('finalizes every statement it prepares, including the one that failed', async () => { + const { database, open } = standIn(); + + const failed = await use(database, store => + Effect.flip( + store.append({ + sessionId: 'ses_missing', + turns: [ + { + id: 'trn_1', + sessionId: 'ses_missing', + role: 'user', + parts: [{ id: `prt_${String('hello')}`, kind: 'text', body: 'hello' }], + }, + ], + prompted: 0, + }) + ) + ); + + expect(failed).toMatchObject({ operation: 'append' }); + expect(open()).toBe(0); +}); diff --git a/packages/harness-sdk/src/plugins/store/expo.ts b/packages/harness-sdk/src/plugins/store/expo.ts new file mode 100644 index 0000000000..d3284f1898 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/expo.ts @@ -0,0 +1,76 @@ +import type { Layer } from 'effect'; +import type { SessionStore, StoreError } from '../../core/storage.js'; +import type { SqlDriver, SqlValue } from './driver.js'; +import { layerSqliteStore } from './sqlite.js'; + +/** + * The store on Expo's SQLite, which is how a React Native app reaches one. + * + * It names no package. The database arrives already open, exactly as on Node, + * and what this file needs of one is the two methods below — so an Expo + * database satisfies it structurally, and the plugin depends on nothing. + */ + +/** + * The part of an Expo database this adapter uses, and the whole of what + * `layerExpoStore` asks for. + * + * Naming `SQLiteDatabase` here instead would buy nothing and cost a dependency: + * the caller passes their own database, so their compiler checks the real type + * against this one at the call. It also lets a test supply one without a native + * module, which is how `expo.test.ts` runs at all. + */ +interface ExpoStatement { + readonly executeSync: (params: SqlValue[]) => unknown; + readonly executeForRawResultSync: (params: SqlValue[]) => { + readonly getAllSync: () => unknown[][]; + }; + readonly finalizeSync: () => void; +} + +interface ExpoDatabase { + readonly prepareSync: (source: string) => ExpoStatement; +} + +/** + * A statement holds a native handle, so it is finalized whether the query + * answered or threw. A lost one leaks the handle for the life of the app. + */ +const query = (database: ExpoDatabase, sql: string, use: (statement: ExpoStatement) => A): A => { + const statement = database.prepareSync(sql); + try { + return use(statement); + } finally { + statement.finalizeSync(); + } +}; + +/** + * Adapts a database to the driver seam. + * + * Rows come back by position, from Expo's raw result, which is the shape the + * store maps onto the selected columns. A `get` that finds nothing answers with + * no row: the store reads at most one row through `all` and a `limit`, so the + * shape drizzle wants back from a missing `get` is never asked for. + */ +const expoDriver = + (database: ExpoDatabase): SqlDriver => + (sql, params, method) => + Promise.resolve( + query(database, sql, statement => { + const bound: SqlValue[] = [...params]; + if (method === 'run') { + statement.executeSync(bound); + return { rows: [] }; + } + const rows = statement.executeForRawResultSync(bound).getAllSync(); + return { rows: method === 'get' ? (rows[0] ?? []) : rows }; + }) + ); + +/** Opens the store on a database the caller already opened, and still closes. */ +const layerExpoStore = (database: ExpoDatabase): Layer.Layer => + layerSqliteStore(expoDriver(database)); + +export type { ExpoDatabase }; +export { expoDriver, layerExpoStore }; diff --git a/packages/harness-sdk/src/plugins/store/migrate.ts b/packages/harness-sdk/src/plugins/store/migrate.ts new file mode 100644 index 0000000000..485ca699e6 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/migrate.ts @@ -0,0 +1,59 @@ +import { assert } from 'typia'; +import { transact, type Connection, type SqlDriver } from './driver.js'; +import { migrations } from './migrations.js'; + +/** + * Applying the migrations the bundle carries. + * + * The SQL is inlined rather than read from disk: React Native has no filesystem + * to read it from, and Drizzle's answer there is a bundler plugin, which a + * package must not force on the people who install it. `pnpm check:migrations` + * is what keeps the inlined copy honest. + */ + +/** + * How many migrations this database has had, held in SQLite's own + * `user_version`. It costs no table of our own and no query on the read path. + */ +const versionOf = async (driver: SqlDriver): Promise => { + const { rows } = await driver('PRAGMA user_version', [], 'get'); + return assert(rows)[0] ?? 0; +}; + +/** + * Runs the statements in order. A migration that reorders is a migration that + * fails, so these cannot be a `Promise.all`. It recurses rather than loops + * because `no-await-in-loop` reads a sequential loop as a missed chance to run + * in parallel, which here is the whole point. + */ +const runAll = async (driver: SqlDriver, statements: readonly string[]): Promise => { + const [first, ...rest] = statements; + if (first === undefined) { + return; + } + await driver(first, [], 'run'); + await runAll(driver, rest); +}; + +/** + * Applies every migration the database has not seen, then records how far it + * got. The version is written into the statement rather than bound, because + * SQLite allows no parameter in a pragma. The value is this array's length, + * never anything a caller supplies. + */ +const applyPending = async (driver: SqlDriver, applied: number): Promise => { + await runAll(driver, migrations.slice(applied).flat()); + await driver(`PRAGMA user_version = ${String(migrations.length)}`, [], 'run'); +}; + +/** Migrates in one unit, so the version and the schema always agree. */ +const migrate = async (connection: Connection): Promise => { + const { driver } = connection; + const applied = await versionOf(driver); + if (applied >= migrations.length) { + return; + } + await transact(connection, () => applyPending(driver, applied)); +}; + +export { migrate }; diff --git a/packages/harness-sdk/src/plugins/store/migrations.ts b/packages/harness-sdk/src/plugins/store/migrations.ts new file mode 100644 index 0000000000..9f17161edb --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/migrations.ts @@ -0,0 +1,35 @@ +/* Generated by `pnpm migrations`. Edit the schema and run it again. */ + +/** + * Every migration, oldest first, each one a list of statements to run in + * order. The version a database has applied is its index plus one, held in + * SQLite's own `user_version`, so the store needs no table of its own to know + * where it stands. + */ +const migrations: readonly (readonly string[])[] = [ + /* 0000_blushing_vance_astro */ + [ + 'CREATE TABLE `sessions` (\n\t`id` text PRIMARY KEY NOT NULL,\n\t`system` text NOT NULL,\n\t`model` text NOT NULL,\n\t`effort` text,\n\t`max_tokens` integer\n);', + 'CREATE TABLE `turns` (\n\t`id` text PRIMARY KEY NOT NULL,\n\t`session_id` text NOT NULL,\n\t`role` text NOT NULL,\n\t`content` text NOT NULL,\n\tFOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE no action\n);', + 'CREATE INDEX `turns_session_id_id` ON `turns` (`session_id`,`id`);', + ], + /* 0001_tranquil_reptil */ + [ + 'CREATE TABLE `parts` (\n\t`id` text PRIMARY KEY NOT NULL,\n\t`turn_id` text NOT NULL,\n\t`session_id` text NOT NULL,\n\t`kind` text NOT NULL,\n\t`body` text NOT NULL,\n\t`media` text,\n\tFOREIGN KEY (`turn_id`) REFERENCES `turns`(`id`) ON UPDATE no action ON DELETE no action,\n\tFOREIGN KEY (`session_id`) REFERENCES `sessions`(`id`) ON UPDATE no action ON DELETE no action\n);', + 'CREATE INDEX `parts_session_id_id` ON `parts` (`session_id`,`id`);', + 'ALTER TABLE `turns` DROP COLUMN `content`;', + ], + /* 0002_bent_susan_delgado */ + ['ALTER TABLE `parts` ADD `signature` text;'], + /* 0003_flaky_gideon */ + ['ALTER TABLE `sessions` ADD `prompted` integer;'], + /* 0004_keen_boomer */ + [ + 'ALTER TABLE `parts` ADD `call_id` text;', + 'ALTER TABLE `parts` ADD `name` text;', + 'ALTER TABLE `parts` ADD `failed` integer;', + 'ALTER TABLE `sessions` ADD `tools` text;', + ], +]; + +export { migrations }; diff --git a/packages/harness-sdk/src/plugins/store/node.ts b/packages/harness-sdk/src/plugins/store/node.ts new file mode 100644 index 0000000000..fffc0916ab --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/node.ts @@ -0,0 +1,54 @@ +import type { DatabaseSync } from 'node:sqlite'; +import type { Layer } from 'effect'; +import type { SessionStore, StoreError } from '../../core/storage.js'; +import type { SqlDriver, SqlValue } from './driver.js'; +import { layerSqliteStore } from './sqlite.js'; + +/** + * The store on Node's own SQLite. This file is one of the two in the package + * that name a platform, which is why it is a plugin and why nothing else + * imports it. + * + * `node:sqlite` needs Node 22.13, which is where it stopped asking for + * `--experimental-sqlite`. On 22.5 to 22.12 the import fails without the flag, + * and before 22.5 the module does not exist. + */ + +/** + * Rows come back as arrays rather than objects, because that is what the seam + * is defined in: the reader maps them onto columns by position. + * + * SQLite fills the object in the order the columns were selected, so reading + * the values back off it recovers that order. Two columns of one name would + * collapse into one key, which no query here can produce: every read selects + * from a single table and joins nothing. + */ +const rowsOf = (database: DatabaseSync, sql: string, params: readonly SqlValue[]): unknown[][] => + database + .prepare(sql) + .all(...params) + .map(Object.values); + +/** + * Adapts a database to the driver seam. + * + * A `get` that finds nothing answers with no row. The store reads at most one + * row, through `all` and a `limit`, so the shape drizzle wants back from a + * missing `get` is never asked for. + */ +const nodeDriver = + (database: DatabaseSync): SqlDriver => + (sql, params, method) => { + if (method === 'run') { + database.prepare(sql).run(...params); + return Promise.resolve({ rows: [] }); + } + const rows = rowsOf(database, sql, params); + return Promise.resolve({ rows: method === 'get' ? (rows[0] ?? []) : rows }); + }; + +/** Opens the store on a database the caller already opened, and still closes. */ +const layerNodeStore = (database: DatabaseSync): Layer.Layer => + layerSqliteStore(nodeDriver(database)); + +export { layerNodeStore }; diff --git a/packages/harness-sdk/src/plugins/store/rows.ts b/packages/harness-sdk/src/plugins/store/rows.ts new file mode 100644 index 0000000000..41b9b8f957 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/rows.ts @@ -0,0 +1,130 @@ +import { createAssert } from 'typia'; +import type { Effort } from '../../core/model.js'; +import type { StoredSession } from '../../core/storage.js'; +import type { TurnPart, TurnRole } from '../../core/turn.js'; + +/** + * What comes back off the disk, and what it means. + * + * A drizzle type states what the schema declares, not what the file on disk + * holds: a database written by an older build, or by another program, still + * arrives as `unknown`. So every row is asserted here before the package + * believes any of it. + */ + +/** The rows this store reads back, stated so they can be validated at the edge. */ +interface SessionRow { + readonly id: string; + readonly system: string; + readonly model: string; + readonly effort: Effort | null; + readonly maxTokens: number | null; + readonly prompted: number | null; + /** A JSON array of tool names, as `toolsIn` writes it. */ + readonly tools: string | null; +} + +interface TurnRow { + readonly id: string; + readonly sessionId: string; + readonly role: TurnRole; +} + +interface PartRow { + readonly id: string; + readonly turnId: string; + readonly kind: TurnPart['kind']; + readonly body: string; + readonly media: string | null; + readonly signature: string | null; + readonly callId: string | null; + readonly name: string | null; + readonly failed: boolean | null; +} + +const assertSessions = createAssert(); +const assertTurns = createAssert(); +const assertParts = createAssert(); + +/** + * The tool names a session offers. A column that holds anything but a list of + * strings is a row this package did not write, so it is refused rather than + * repaired: a session opened with the wrong tools sends the wrong prefix. + */ +const assertNames = createAssert(); + +/** + * Both halves of a tool call name the call they belong to. Neither is any use + * without it: a call with no identifier cannot be answered, and a result with + * none answers nothing, and every shape refuses the pair. + */ +const asToolPart = (row: PartRow, kind: 'toolCall' | 'toolResult'): TurnPart => { + if (row.callId === null) { + throw new Error(`the ${kind} part ${row.id} names no call`); + } + if (kind === 'toolResult') { + return { id: row.id, kind, body: row.body, callId: row.callId, failed: row.failed === true }; + } + if (row.name === null) { + throw new Error(`the toolCall part ${row.id} names no tool`); + } + return { id: row.id, kind, body: row.body, callId: row.callId, name: row.name }; +}; + +/** + * A row is one part, and only an image names a media type. A row that claims to + * be an image without one is a row this package did not write, so it is refused + * rather than repaired. + */ +const asPart = (row: PartRow): TurnPart => { + if (row.kind === 'reasoning') { + return { + id: row.id, + kind: 'reasoning', + body: row.body, + ...(row.signature === null ? {} : { signature: row.signature }), + }; + } + if (row.kind === 'toolCall' || row.kind === 'toolResult') { + return asToolPart(row, row.kind); + } + if (row.kind !== 'image') { + return { id: row.id, kind: row.kind, body: row.body }; + } + if (row.media === null) { + throw new Error(`the image part ${row.id} names no media type`); + } + return { id: row.id, kind: 'image', body: row.body, media: row.media }; +}; + +/** Groups the parts by turn in one pass, so joining them back on costs no scan. */ +const byTurn = (rows: readonly PartRow[]): Map => { + const held = new Map(); + for (const row of rows) { + const already = held.get(row.turnId); + const part = asPart(row); + if (already === undefined) { + held.set(row.turnId, [part]); + } else { + already.push(part); + } + } + return held; +}; + +/** A column with no value is a value the caller never named, which is absent here. */ +const asStoredSession = (row: SessionRow): StoredSession => ({ + id: row.id, + system: row.system, + model: row.model, + ...(row.effort === null ? {} : { effort: row.effort }), + ...(row.maxTokens === null ? {} : { maxTokens: row.maxTokens }), + ...(row.prompted === null ? {} : { prompted: row.prompted }), + ...(row.tools === null ? {} : { tools: assertNames(JSON.parse(row.tools)) }), +}); + +/** The tool names as one column. A session that names none leaves it empty. */ +const toolsIn = (names: readonly string[] | undefined): string | undefined => + names === undefined || names.length === 0 ? undefined : JSON.stringify(names); + +export { assertParts, assertSessions, assertTurns, asStoredSession, byTurn, toolsIn }; diff --git a/packages/harness-sdk/src/plugins/store/schema.ts b/packages/harness-sdk/src/plugins/store/schema.ts new file mode 100644 index 0000000000..828d1797c5 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/schema.ts @@ -0,0 +1,106 @@ +import { index, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'; + +/** + * What a store holds. Two tables, no nesting and no join on the read path. + * + * The schema is written here and the SQL is generated from it, so a migration + * and a query can never disagree about a column. What the schema cannot say is + * whether the database on disk matches it. That is checked at the edge with a + * validator, not assumed from these types. + */ + +/** + * What `SessionOptions` freezes, stored so a session can be continued. Without + * these a resumed session would take whatever the caller passed the second + * time, and a system prompt that differs by one byte drops the whole prefix. + * + * `prompted` is the one column here that changes: it is the provider's own + * count of the last request's input, and it is what decides whether the next + * question compacts first. It is stored because nothing estimates a token + * count, so a session reopened without it would not know how full it is. + * + * There is no created column. The identifier is a ULID, so it already carries + * the time and sorts by it. + */ +const sessions = sqliteTable('sessions', { + id: text('id').primaryKey(), + system: text('system').notNull(), + model: text('model').notNull(), + effort: text('effort', { enum: ['low', 'medium', 'high', 'xhigh', 'max'] }), + maxTokens: integer('max_tokens'), + prompted: integer('prompted'), + /** + * The tools the session offers, as a JSON array of names in the order the + * model sees them. It is one column and not a table because nothing ever + * queries it: it is read whole with the session and written once, at open. + * + * The names and not the definitions, because a definition lives in code. See + * AGENTS.md, "A session names its tools; the registry defines them". + */ + tools: text('tools'), +}); + +/** + * One row per turn. A turn holds no content of its own: its content is its + * parts, which is what makes an image or a piece of reasoning storable beside + * text without a column per kind. + * + * The index covers the pair, not the session alone: every read asks for one + * session's turns in identifier order, and the pair answers that straight from + * the index without a sort. + */ +const turns = sqliteTable( + 'turns', + { + id: text('id').primaryKey(), + sessionId: text('session_id') + .notNull() + .references(() => sessions.id), + role: text('role', { enum: ['user', 'assistant'] }).notNull(), + }, + table => [index('turns_session_id_id').on(table.sessionId, table.id)] +); + +/** + * One row per piece of a turn, ordered by identifier like everything else. + * + * `session_id` repeats what `turn_id` could reach, and it is there for the + * reader: loading a session is then two indexed scans over two tables with no + * join at all, rather than a join whose cost grows with the conversation. + * + * `body` is the only payload column, because every kind has exactly one payload + * — the text, the reasoning, the base64 of the image, the arguments of a call, + * or what a tool gave back. `media` names the media type and is empty for + * everything but an image. `signature` is what the provider issued with a + * thinking block and reads back to know the thinking is its own; it is empty for + * everything but reasoning, and for reasoning from a shape that issues none. + * + * The last three belong to tools. `call_id` is what the provider called the + * call, and it is on both halves: a result names the call it answers, and every + * shape refuses a call whose result is missing. `name` is the tool, on the call + * only. `failed` says the tool did not do what it was asked, on the result only. + */ +const parts = sqliteTable( + 'parts', + { + id: text('id').primaryKey(), + turnId: text('turn_id') + .notNull() + .references(() => turns.id), + sessionId: text('session_id') + .notNull() + .references(() => sessions.id), + kind: text('kind', { + enum: ['text', 'summary', 'reasoning', 'redacted', 'image', 'toolCall', 'toolResult'], + }).notNull(), + body: text('body').notNull(), + media: text('media'), + signature: text('signature'), + callId: text('call_id'), + name: text('name'), + failed: integer('failed', { mode: 'boolean' }), + }, + table => [index('parts_session_id_id').on(table.sessionId, table.id)] +); + +export { parts, sessions, turns }; diff --git a/packages/harness-sdk/src/plugins/store/sqlite.test.ts b/packages/harness-sdk/src/plugins/store/sqlite.test.ts new file mode 100644 index 0000000000..f1fca721a6 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/sqlite.test.ts @@ -0,0 +1,296 @@ +import { DatabaseSync } from 'node:sqlite'; +import { Effect, Layer, Option } from 'effect'; +import { expect, it } from 'vitest'; +import { SessionStore, type SessionStoreService } from '../../core/storage.js'; +import { migrations } from './migrations.js'; +import { layerNodeStore } from './node.js'; +import { textOf } from '../../core/turn.js'; + +const database = (): DatabaseSync => new DatabaseSync(':memory:'); + +const use = ( + db: DatabaseSync, + run: (store: SessionStoreService) => Effect.Effect +): Promise => + Effect.runPromise(Effect.provide(Effect.flatMap(SessionStore, run), layerNodeStore(db))); + +const session = { id: 'ses_1', system: 'sys', model: 'claude-opus-5' }; + +/** + * A database as an earlier build of this package left it: the migrations up to + * that version applied, and `user_version` saying so. + * + * It is always one behind, whatever the newest migration is, so this test + * covers the migration being added rather than one that was added once. + */ +const oneVersionBehind = (): { readonly db: DatabaseSync; readonly version: number } => { + const db = database(); + const version = migrations.length - 1; + for (const statement of migrations.slice(0, version).flat()) { + db.prepare(statement).run(); + } + db.prepare(`PRAGMA user_version = ${String(version)}`).run(); + return { db, version }; +}; + +/** + * The migration that matters is the one applied to a database that already + * holds something. `check:migrations` proves the SQL matches the schema; only + * this proves the SQL can be applied to a conversation somebody already had. + */ +it('migrates a database that already holds a conversation, and keeps it', async () => { + const { db, version } = oneVersionBehind(); + db.prepare('INSERT INTO sessions (id, system, model) VALUES (?, ?, ?)').run( + 'ses_1', + 'sys', + 'claude-opus-5' + ); + db.prepare('INSERT INTO turns (id, session_id, role) VALUES (?, ?, ?)').run( + 'trn_1', + 'ses_1', + 'user' + ); + db.prepare('INSERT INTO parts (id, turn_id, session_id, kind, body) VALUES (?, ?, ?, ?, ?)').run( + 'prt_1', + 'trn_1', + 'ses_1', + 'text', + 'hello' + ); + + const read = await use(db, store => + Effect.all({ options: store.read('ses_1'), turns: store.load('ses_1') }) + ); + + expect(version).toBeLessThan(migrations.length); + expect(db.prepare('PRAGMA user_version').all()).toEqual([{ user_version: migrations.length }]); + expect(read.turns.map(textOf)).toEqual(['hello']); + /* Whatever the newest migration added, an older row has no value for it, and + absent is what the session was doing before the column existed. */ + expect(Option.getOrThrow(read.options)).toEqual({ + id: 'ses_1', + system: 'sys', + model: 'claude-opus-5', + }); +}); + +it('writes the columns the migration added to a session written before it', async () => { + const { db } = oneVersionBehind(); + db.prepare('INSERT INTO sessions (id, system, model) VALUES (?, ?, ?)').run( + 'ses_1', + 'sys', + 'claude-opus-5' + ); + + const read = await use(db, store => + Effect.zipRight( + store.append({ + sessionId: 'ses_1', + turns: [ + { + id: 'trn_1', + sessionId: 'ses_1', + role: 'user', + parts: [{ id: 'prt_1', kind: 'text', body: 'hello' }], + }, + ], + prompted: 12, + }), + store.read('ses_1') + ) + ); + + expect(Option.getOrThrow(read)).toMatchObject({ prompted: 12 }); +}); + +it('leaves an already migrated database alone when it is opened again', async () => { + const db = database(); + await use(db, () => Effect.void); + await use(db, () => Effect.void); + + /* Every migration, applied once. A second open that re-ran them would throw + on the first CREATE TABLE. */ + const versions = db.prepare('PRAGMA user_version').all(); + expect(versions).toEqual([{ user_version: migrations.length }]); + expect(migrations.length).toBeGreaterThan(1); +}); + +it('reads the turns back in the order they were appended', async () => { + const db = database(); + const loaded = await use(db, store => + Effect.gen(function* () { + yield* store.create(session); + for (const [index, role] of (['user', 'assistant', 'user'] as const).entries()) { + yield* store.append({ + sessionId: session.id, + turns: [ + { + id: `trn_${String(index)}`, + sessionId: session.id, + role, + parts: [ + { + id: `prt_${String(`message ${String(index)}`)}`, + kind: 'text', + body: `message ${String(index)}`, + }, + ], + }, + ], + prompted: 0, + }); + } + return yield* store.load(session.id); + }) + ); + + expect(loaded.map(textOf)).toEqual(['message 0', 'message 1', 'message 2']); +}); + +it('reads the parts of a turn back in the order they were written', async () => { + /* The provider refuses a turn whose thinking blocks come back rearranged, so + a store that reordered them would undo the ordering the session keeps. The + read sorts on the part identifier, which is a ULID and so rises with the + order the parts were made in. */ + const written = [ + { id: 'prt_0', kind: 'reasoning', body: 'before', signature: 'sig_one' }, + { id: 'prt_1', kind: 'redacted', body: 'ENCRYPTED' }, + { id: 'prt_2', kind: 'reasoning', body: 'after', signature: 'sig_two' }, + { id: 'prt_3', kind: 'text', body: 'said' }, + ] as const; + const db = database(); + const loaded = await use(db, store => + Effect.gen(function* () { + yield* store.create(session); + yield* store.append({ + sessionId: session.id, + turns: [{ id: 'trn_0', sessionId: session.id, role: 'assistant', parts: written }], + prompted: 0, + }); + return yield* store.load(session.id); + }) + ); + + expect(loaded[0]?.parts).toEqual(written); +}); + +it('gives back the options a session was opened with, absent ones included', async () => { + const db = database(); + const read = await use(db, store => + Effect.gen(function* () { + yield* store.create({ ...session, effort: 'high' }); + return yield* store.read(session.id); + }) + ); + + expect(Option.getOrThrow(read)).toEqual({ + id: 'ses_1', + system: 'sys', + model: 'claude-opus-5', + effort: 'high', + }); +}); + +it('answers with nothing for a session it has never heard of', async () => { + const read = await use(database(), store => store.read('ses_missing')); + + expect(Option.isNone(read)).toBe(true); +}); + +it('refuses a turn whose session was never created', async () => { + const failed = await use(database(), store => + store + .append({ + sessionId: 'ses_missing', + turns: [ + { + id: 'trn_1', + sessionId: 'ses_missing', + role: 'user', + parts: [{ id: `prt_${String('hello')}`, kind: 'text', body: 'hello' }], + }, + ], + prompted: 0, + }) + .pipe(Effect.flip) + ); + + expect(failed).toMatchObject({ operation: 'append' }); +}); + +it('refuses a row the schema cannot explain rather than handing it back', async () => { + const db = database(); + await use(db, store => + Effect.zipRight( + store.create(session), + store.append({ + sessionId: session.id, + turns: [{ id: 'trn_1', sessionId: session.id, role: 'user', parts: [] }], + prompted: 0, + }) + ) + ); + db.prepare('INSERT INTO parts (id, turn_id, session_id, kind, body) VALUES (?, ?, ?, ?, ?)').run( + 'prt_1', + 'trn_1', + session.id, + 'banana', + 'hello' + ); + + const failed = await use(db, store => Effect.flip(store.load(session.id))); + + /* The cause is named, so a validator that has silently stopped running + cannot pass this test by failing for some other reason. */ + expect(failed).toMatchObject({ operation: 'load' }); + expect(String(failed.cause)).toContain('invalid type on $input[0].kind'); +}); + +it('keeps two sessions apart', async () => { + const db = database(); + const loaded = await use(db, store => + Effect.gen(function* () { + yield* store.create(session); + yield* store.create({ ...session, id: 'ses_2' }); + yield* store.append({ + sessionId: 'ses_1', + turns: [ + { + id: 'trn_1', + sessionId: 'ses_1', + role: 'user', + parts: [{ id: `prt_${String('first')}`, kind: 'text', body: 'first' }], + }, + ], + prompted: 0, + }); + yield* store.append({ + sessionId: 'ses_2', + turns: [ + { + id: 'trn_2', + sessionId: 'ses_2', + role: 'user', + parts: [{ id: `prt_${String('second')}`, kind: 'text', body: 'second' }], + }, + ], + prompted: 0, + }); + return yield* store.load('ses_2'); + }) + ); + + expect(loaded.map(textOf)).toEqual(['second']); +}); + +/** The layer builds the store once, so a driver that cannot migrate fails there. */ +it('fails when the layer is built, not when a question is asked', async () => { + const db = database(); + db.exec('CREATE TABLE sessions (wrong text)'); + + const failed = await Effect.runPromise( + Effect.flip(Effect.scoped(Layer.build(layerNodeStore(db)))) + ); + + expect(failed).toMatchObject({ operation: 'create' }); +}); diff --git a/packages/harness-sdk/src/plugins/store/sqlite.ts b/packages/harness-sdk/src/plugins/store/sqlite.ts new file mode 100644 index 0000000000..6cad9dd689 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/sqlite.ts @@ -0,0 +1,169 @@ +import { asc, eq } from 'drizzle-orm'; +import { drizzle } from 'drizzle-orm/sqlite-proxy'; +import { Effect, Layer, Option } from 'effect'; +import { + SessionStore, + StoreError, + type SessionStoreService, + type StoredExchange, +} from '../../core/storage.js'; +import type { Turn, TurnPart } from '../../core/turn.js'; +import { connectionOf, transact, type Connection, type SqlDriver } from './driver.js'; +import { migrate } from './migrate.js'; +import { + assertParts, + assertSessions, + assertTurns, + asStoredSession, + byTurn, + toolsIn, +} from './rows.js'; +import { parts, sessions, turns } from './schema.js'; + +/** + * The SQLite store, written once for every platform. It holds every query. + * + * The seam is `driver.ts`: one function, so `node:sqlite` and Expo each need an + * adapter of about twenty lines and share all of this. What a row means is + * `rows.ts`, and how the tables come to exist is `migrate.ts`. + */ + +const failing = (operation: StoreError['operation']) => (cause: unknown) => + new StoreError({ operation, cause }); + +const attempt = ( + operation: StoreError['operation'], + run: () => Promise +): Effect.Effect => Effect.tryPromise({ try: run, catch: failing(operation) }); + +type Db = ReturnType; + +/** What only one kind of part carries. Everything else leaves the column empty. */ +const columnsOf = (part: TurnPart) => { + switch (part.kind) { + case 'image': { + return { media: part.media }; + } + case 'reasoning': { + return part.signature === undefined ? {} : { signature: part.signature }; + } + case 'toolCall': { + return { callId: part.callId, name: part.name }; + } + case 'toolResult': { + return { callId: part.callId, failed: part.failed }; + } + case 'text': + case 'summary': + case 'redacted': { + return {}; + } + } +}; + +/** One part, as its row. Only the kind that has a column fills it. */ +const partRow = (turn: Turn, part: TurnPart) => ({ + id: part.id, + turnId: turn.id, + sessionId: turn.sessionId, + kind: part.kind, + body: part.body, + ...columnsOf(part), +}); + +/** + * Writes the turns, their parts, and the session's new count as one unit. A + * turn whose parts went missing would read back as an empty message and quietly + * shorten the prompt, a question written without its answer would go back out + * with every later request, and a count written apart from either would say the + * session holds something it does not. + */ +const insertExchange = (connection: Connection, db: Db, exchange: StoredExchange): Promise => + transact(connection, async () => { + const written = exchange.turns; + if (written.length > 0) { + await db + .insert(turns) + .values(written.map(turn => ({ id: turn.id, sessionId: turn.sessionId, role: turn.role }))); + const rows = written.flatMap(turn => turn.parts.map(part => partRow(turn, part))); + if (rows.length > 0) { + await db.insert(parts).values(rows); + } + } + await db + .update(sessions) + .set({ prompted: exchange.prompted }) + .where(eq(sessions.id, exchange.sessionId)); + }); + +/** + * Two indexed scans and no join. Both tables carry the session, so each read is + * a range over one index and the parts are matched up in memory, in one pass. + */ +const selectTurns = async (db: Db, sessionId: string): Promise => { + const [turnRows, partRows] = await Promise.all([ + db.select().from(turns).where(eq(turns.sessionId, sessionId)).orderBy(asc(turns.id)), + db.select().from(parts).where(eq(parts.sessionId, sessionId)).orderBy(asc(parts.id)), + ]); + const held = byTurn(assertParts(partRows)); + return assertTurns(turnRows).map( + (turn): Turn => ({ + id: turn.id, + sessionId: turn.sessionId, + role: turn.role, + parts: held.get(turn.id) ?? [], + }) + ); +}; + +/** + * Builds the store on a driver. Every write lands at once, so `flush` has + * nothing to do: batching would trade a lost turn for a saving this package has + * not measured a need for. + */ +const storeOn = (connection: Connection): SessionStoreService => { + const db = drizzle(connection.driver); + + return { + create: session => + attempt('create', () => + connection.write(async () => { + await db.insert(sessions).values({ ...session, tools: toolsIn(session.tools) }); + }) + ), + + read: sessionId => + attempt('read', async () => { + const rows = await db.select().from(sessions).where(eq(sessions.id, sessionId)).limit(1); + return Option.map(Option.fromNullable(assertSessions(rows)[0]), asStoredSession); + }), + + append: exchange => attempt('append', () => insertExchange(connection, db, exchange)), + + load: sessionId => attempt('load', () => selectTurns(db, sessionId)), + + flush: () => Effect.void, + }; +}; + +/** + * Opens the store: migrates, then hands back the plugin. Migrating here rather + * than on first use means a database that cannot be migrated fails when the + * layer is built, not on the first question somebody asks. + */ +const layerSqliteStore = (driver: SqlDriver): Layer.Layer => + Layer.suspend(() => { + const connection = connectionOf(driver); + return Layer.effect( + SessionStore, + Effect.map( + attempt('create', async () => { + await driver('PRAGMA foreign_keys = ON', [], 'run'); + await migrate(connection); + }), + () => storeOn(connection) + ) + ); + }); + +export { layerSqliteStore }; diff --git a/packages/harness-sdk/src/plugins/store/tools.test.ts b/packages/harness-sdk/src/plugins/store/tools.test.ts new file mode 100644 index 0000000000..c70897d923 --- /dev/null +++ b/packages/harness-sdk/src/plugins/store/tools.test.ts @@ -0,0 +1,101 @@ +import { DatabaseSync } from 'node:sqlite'; +import { Effect, Option } from 'effect'; +import { expect, it } from 'vitest'; +import { SessionStore, type SessionStoreService } from '../../core/storage.js'; +import { layerNodeStore } from './node.js'; + +/** + * What the store does with the two halves of a tool call. + * + * They are the one pair it must never split. Every shape refuses a call whose + * result is missing, so a session whose store lost one half can never be + * continued at all, and the only symptom is the next question failing. + */ + +const database = (): DatabaseSync => new DatabaseSync(':memory:'); + +const use = ( + db: DatabaseSync, + run: (store: SessionStoreService) => Effect.Effect +): Promise => + Effect.runPromise(Effect.provide(Effect.flatMap(SessionStore, run), layerNodeStore(db))); + +const session = { id: 'ses_1', system: 'sys', model: 'claude-opus-5' }; + +const call = { + id: 'prt_1', + kind: 'toolCall', + body: '{"city":"Oslo"}', + callId: 'tc_1', + name: 'weather', +} as const; + +const answered = { + id: 'prt_2', + kind: 'toolResult', + body: 'it rains', + callId: 'tc_1', + failed: false, +} as const; + +const refused = { + id: 'prt_3', + kind: 'toolResult', + body: 'no such city', + callId: 'tc_2', + failed: true, +} as const; + +it('writes a call and its result, and reads both back whole', async () => { + const db = database(); + const loaded = await use(db, store => + Effect.gen(function* () { + yield* store.create({ ...session, tools: ['weather', 'question'] }); + yield* store.append({ + sessionId: session.id, + turns: [ + { id: 'trn_1', sessionId: session.id, role: 'assistant', parts: [call] }, + { id: 'trn_2', sessionId: session.id, role: 'user', parts: [answered, refused] }, + ], + prompted: 12, + }); + return { turns: yield* store.load(session.id), stored: yield* store.read(session.id) }; + }) + ); + + expect(loaded.turns[0]?.parts).toEqual([call]); + /* Whether a result failed is what tells the model to try something else, so + it is a column and not something inferred from the text. */ + expect(loaded.turns[1]?.parts).toEqual([answered, refused]); + /* The order of the tools is part of the prefix, so it comes back as written. */ + expect(Option.getOrThrow(loaded.stored).tools).toEqual(['weather', 'question']); +}); + +it('gives back no tools for a session that named none', async () => { + const read = await use(database(), store => + Effect.zipRight(store.create(session), store.read(session.id)) + ); + + expect(Option.getOrThrow(read).tools).toBeUndefined(); +}); + +it('refuses a call that names no call rather than handing it back', async () => { + const db = database(); + const failed = await use(db, store => + Effect.gen(function* () { + yield* store.create(session); + yield* store.append({ + sessionId: session.id, + turns: [{ id: 'trn_1', sessionId: session.id, role: 'assistant', parts: [] }], + prompted: 0, + }); + db.prepare( + 'INSERT INTO parts (id, turn_id, session_id, kind, body) VALUES (?, ?, ?, ?, ?)' + ).run('prt_1', 'trn_1', session.id, 'toolCall', '{}'); + return yield* Effect.flip(store.load(session.id)); + }) + ); + + expect(failed).toMatchObject({ operation: 'load' }); + expect(String(failed.cause)).toContain('names no call'); +}); diff --git a/packages/harness-sdk/src/plugins/token/static.ts b/packages/harness-sdk/src/plugins/token/static.ts new file mode 100644 index 0000000000..c65f8235e0 --- /dev/null +++ b/packages/harness-sdk/src/plugins/token/static.ts @@ -0,0 +1,12 @@ +import { Effect, Layer } from 'effect'; +import { TokenSource } from '../../core/token.js'; + +/** + * One token for the life of the process. Correct for a short run and for a + * token that outlives it; wrong for a long-lived session, which wants a plugin + * that can refresh. + */ +const layerStaticToken = (token: string): Layer.Layer => + Layer.succeed(TokenSource, { get: () => Effect.succeed(token) }); + +export { layerStaticToken }; diff --git a/packages/harness-sdk/src/plugins/tools/index.ts b/packages/harness-sdk/src/plugins/tools/index.ts new file mode 100644 index 0000000000..263dfa6e6c --- /dev/null +++ b/packages/harness-sdk/src/plugins/tools/index.ts @@ -0,0 +1,4 @@ +export * from './question.js'; +export * from './subagent.js'; +export * from './time.js'; +export * from './todo.js'; diff --git a/packages/harness-sdk/src/plugins/tools/question.test.ts b/packages/harness-sdk/src/plugins/tools/question.test.ts new file mode 100644 index 0000000000..7a0ad1df76 --- /dev/null +++ b/packages/harness-sdk/src/plugins/tools/question.test.ts @@ -0,0 +1,190 @@ +import { Duration, Effect } from 'effect'; +import { assert } from 'typia'; +import { expect, it } from 'vitest'; +import { type ToolCall, ToolFailure } from '../../core/tool.js'; +import { type Answer, type Asker, type Question, questionTool } from './question.js'; + +/** + * The one tool the package ships. What the model sends is the model's, and + * every one of these is a shape it will send sooner or later; what comes back + * is what the model then has to read, and it has to read it without a second + * round asking what an answer meant. + */ + +const call = (args: unknown): ToolCall => ({ + id: 'tc_1', + name: 'question', + arguments: JSON.stringify(args), +}); + +/** An asker that answers from a script and records what it was asked. */ +const asking = ( + answers: readonly Answer[] | ToolFailure +): { readonly asked: Question[][]; readonly ask: Asker } => { + const asked: Question[][] = []; + const ask: Asker = questions => { + asked.push([...questions]); + return answers instanceof ToolFailure ? Effect.fail(answers) : Effect.succeed(answers); + }; + return { asked, ask }; +}; + +const run = (tool: ReturnType, one: ToolCall) => + Effect.runPromise(Effect.either(tool.run(one))); + +const said = async ( + answers: readonly Answer[], + questions: readonly Partial[] +): Promise => { + const { ask } = asking(answers); + const got = await run(questionTool(ask), call({ questions })); + return got._tag === 'Right' ? got.right : `failed: ${String(got.left.cause)}`; +}; + +it('asks everything in one call and answers in the order the model asked', async () => { + const { asked, ask } = asking([ + { id: 'where', text: 'eu-west-1' }, + { id: 'db', chosen: ['postgres'] }, + ]); + + const got = await run( + questionTool(ask), + call({ + questions: [ + { + id: 'db', + prompt: 'Which database?', + choices: [{ value: 'postgres', label: 'Postgres' }], + }, + { id: 'where', prompt: 'Which region?' }, + ], + }) + ); + + expect(asked).toHaveLength(1); + /* The model's order, not the caller's. A model reading its own questions back + out of order has to work out which answer went with which. */ + expect(got._tag === 'Right' && got.right).toBe( + 'Which database? [db]\npostgres\n\nWhich region? [where]\neu-west-1' + ); +}); + +it('reports every choice when several were picked', async () => { + const answered = await said( + [{ id: 'r', chosen: ['eu-west-1', 'us-east-1'] }], + [{ id: 'r', prompt: 'Which regions?', multiple: true }] + ); + + expect(answered).toBe('Which regions? [r]\neu-west-1, us-east-1'); +}); + +it('says a question went unanswered rather than leaving a blank', async () => { + /* Three ways to answer nothing, and the model must not have to guess that a + blank line meant anything. */ + const answered = await said( + [{ id: 'b' }, { id: 'c', text: '' }], + [ + { id: 'a', prompt: 'Skipped outright?' }, + { id: 'b', prompt: 'Answered with nothing?' }, + { id: 'c', prompt: 'Answered with an empty string?' }, + ] + ); + + expect(answered.split('\n\n')).toEqual([ + 'Skipped outright? [a]\n(not answered)', + 'Answered with nothing? [b]\n(not answered)', + 'Answered with an empty string? [c]\n(not answered)', + ]); +}); + +it('drops an answer to a question nobody asked', async () => { + const answered = await said( + [ + { id: 'other', text: 'i answered something else' }, + { id: 'a', text: 'yes' }, + ], + [{ id: 'a', prompt: 'Go ahead?' }] + ); + + /* Otherwise the caller decides what the model believes it asked. */ + expect(answered).toBe('Go ahead? [a]\nyes'); +}); + +it('hands the model back what was wrong with its arguments', async () => { + const { asked, ask } = asking([]); + + const got = await run(questionTool(ask), call({ questions: [{ prompt: 'no id' }] })); + + expect(got._tag === 'Left' && String(got.left.cause)).toContain('questions[0].id'); + /* And nobody was disturbed over a call the model got wrong. */ + expect(asked).toEqual([]); +}); + +it('hands the model back arguments that are not JSON at all', async () => { + const { ask } = asking([]); + + const got = await run(questionTool(ask), { id: 'tc_1', name: 'question', arguments: '{oops' }); + + expect(got._tag).toBe('Left'); +}); + +it('fails the call rather than the session when the asking itself fails', async () => { + const { ask } = asking(new ToolFailure({ cause: 'nobody is at the terminal' })); + + const got = await run(questionTool(ask), call({ questions: [{ id: 'a', prompt: 'Go ahead?' }] })); + + /* A failed result, which the model can act on. Anything else would end a + session because somebody closed a window. */ + expect(got._tag === 'Left' && String(got.left.cause)).toContain('nobody is at the terminal'); +}); + +it('never asks over itself, so two rounds reach one person one at a time', async () => { + const seen: string[] = []; + const slow: Asker = questions => + Effect.sync(() => void seen.push('in')) + .pipe(Effect.flatMap(() => Effect.sleep('30 millis'))) + .pipe(Effect.tap(() => Effect.sync(() => void seen.push('out')))) + .pipe(Effect.as(questions.map(question => ({ id: question.id, text: 'yes' })))); + const tool = questionTool(slow); + const one = call({ questions: [{ id: 'a', prompt: 'a?' }] }); + + /* Two callers at once, which is a parent and its subagent over one terminal. + The permit is the tool's, so the session they run in is beside the point. */ + await Promise.all([run(tool, one), run(tool, one)]); + + expect(seen).toStrictEqual(['in', 'out', 'in', 'out']); +}); + +it('takes the name and the deadline a harness gives it', () => { + const { ask } = asking([]); + + const tool = questionTool(ask, { name: 'ask_user', inlineFor: Duration.seconds(5) }); + + expect({ name: tool.definition.name, inlineFor: tool.inlineFor }).toEqual({ + name: 'ask_user', + inlineFor: Duration.seconds(5), + }); +}); + +it('describes the questions it takes, so the model can write one', () => { + const { ask } = asking([]); + + const { parameters } = questionTool(ask).definition; + + /* The model writes this shape from the schema alone. A property the schema + does not name is a property the model never sends. */ + const { items } = assert<{ + readonly items: { + readonly properties: Readonly>; + readonly required: readonly string[]; + }; + }>(parameters.properties['questions']); + expect(Object.keys(items.properties)).toEqual([ + 'id', + 'prompt', + 'choices', + 'multiple', + 'optional', + ]); + expect(items.required).toEqual(['id', 'prompt']); +}); diff --git a/packages/harness-sdk/src/plugins/tools/question.ts b/packages/harness-sdk/src/plugins/tools/question.ts new file mode 100644 index 0000000000..1e58ec7844 --- /dev/null +++ b/packages/harness-sdk/src/plugins/tools/question.ts @@ -0,0 +1,217 @@ +import { type Duration, Effect } from 'effect'; +import { createAssert } from 'typia'; +import { type Tool, ToolFailure, type ToolCall, type JsonSchema } from '../../core/tool.js'; + +/** + * The one tool no harness can do without, and the one no harness can write for + * itself: asking the person a question. + * + * Everything about the question is the model's: how many, what each says, what + * may be picked, whether one answer or several, whether it may be skipped. + * Everything about the asking is the caller's: a terminal prompt, a dialog, a + * form on a web page, a message to somebody on call. This holds the middle — + * the shape of a question, the shape of an answer, and the words the model gets + * back — so the two never have to agree on anything else. + * + * A question outlives a request more often than not. Nobody answers in the + * moment they are asked, and a model that had to wait would hold a request open + * on a person making coffee. So the tool is backgrounded like any other: the + * model is told the question is out, carries on with what does not depend on + * it, and the session starts a round of its own when the answer arrives. + */ + +/** One thing a person may pick. `value` comes back; `label` is what they read. */ +interface Choice { + readonly value: string; + readonly label: string; + /** Why somebody would pick this one. Shown under the label where there is room. */ + readonly description?: string; +} + +/** One question. No `choices` means the answer is whatever the person types. */ +interface Question { + /** The model's name for this question. The answer carries it back. */ + readonly id: string; + readonly prompt: string; + readonly choices?: readonly Choice[]; + /** Several choices at once rather than one. Ignored where there are none. */ + readonly multiple?: boolean; + /** The person may answer nothing. By default an answer is expected. */ + readonly optional?: boolean; +} + +/** + * What came back for one question. + * + * A caller fills in `chosen` for what was picked and `text` for what was typed. + * Neither, and the question went unanswered, which is a fact the model needs + * rather than an error: a person who skips a question has told you something. + */ +interface Answer { + readonly id: string; + readonly chosen?: readonly string[]; + readonly text?: string; +} + +/** + * How this harness asks. The caller writes one of these and nothing else. + * + * It may take as long as it likes and may fail: neither ends the session. A + * failure reaches the model as a failed result, which is the only party that + * can decide whether to ask again, ask differently, or carry on without. Fail + * with a `ToolFailure` to choose the words the model reads; anything else is + * wrapped and reaches it as whatever it prints as. + */ +type Asker = (questions: readonly Question[]) => Effect.Effect; + +/** What the model sends. Anything else is a failed result and a chance to retry. */ +interface Asked { + readonly questions: readonly Question[]; +} + +const assertAsked = createAssert(); + +const choiceSchema = { + type: 'object', + properties: { + value: { type: 'string', description: 'What comes back when this one is picked.' }, + label: { type: 'string', description: 'The words the person reads.' }, + description: { type: 'string', description: 'Why somebody would pick this one.' }, + }, + required: ['value', 'label'], +} as const; + +const questionSchema = { + type: 'object', + properties: { + id: { + type: 'string', + description: 'Your name for this question. The answer comes back under it.', + }, + prompt: { type: 'string', description: 'The question, as the person will read it.' }, + choices: { + type: 'array', + items: choiceSchema, + description: 'What may be picked. Leave it out to let the person write an answer.', + }, + multiple: { + type: 'boolean', + description: 'Whether several choices may be picked at once. Ignored without choices.', + }, + optional: { type: 'boolean', description: 'Whether the question may be left unanswered.' }, + }, + required: ['id', 'prompt'], +} as const; + +const parameters: JsonSchema = { + type: 'object', + properties: { + questions: { + type: 'array', + minItems: 1, + items: questionSchema, + description: 'Every question at once. Asking together is cheaper than asking in turn.', + }, + }, + required: ['questions'], + additionalProperties: false, +}; + +const description = + 'Asks the person one or more questions and returns their answers. Ask ' + + 'everything you need in one call rather than one question at a time. A ' + + 'question with choices is answered by picking from them; a question without ' + + "is answered in the person's own words. Waiting is the default, because you " + + 'asked to find something out; set wait to false and carry on if there is ' + + 'useful work the answer does not block.'; + +/** What the model sent, or a failed result saying what was wrong with it. */ +const asked = (call: ToolCall): Effect.Effect => + Effect.try({ + try: () => assertAsked(JSON.parse(call.arguments)), + catch: cause => new ToolFailure({ cause }), + }); + +/** One answer, in words. What was picked, or what was typed, or nothing. */ +const wordsOf = (answer: Answer | undefined): string => { + if (answer === undefined) { + return '(not answered)'; + } + const chosen = answer.chosen ?? []; + if (chosen.length > 0) { + return chosen.join(', '); + } + return answer.text === undefined || answer.text === '' ? '(not answered)' : answer.text; +}; + +/** + * The answers as the model reads them, in the order it asked. + * + * It walks the questions rather than the answers, so a caller who answers two + * of three questions is reported as answering two of three, and an answer for a + * question nobody asked is dropped rather than shown as one the model wrote. + */ +const wordsFor = (questions: readonly Question[], answers: readonly Answer[]): string => + questions + .map(question => { + const answer = answers.find(one => one.id === question.id); + return `${question.prompt} [${question.id}]\n${wordsOf(answer)}`; + }) + .join('\n\n'); + +/** What the caller may change about the tool. Everything else is the model's. */ +interface QuestionOptions { + /** + * How long the model waits before carrying on without the answer. It falls + * back to the session's own deadline, which is what most callers want: the + * person is asked either way, and only the waiting is cut short. + */ + readonly inlineFor?: Duration.DurationInput; + /** + * Whether the model waits for an answer, as it is told by default. True, + * because a model asks in order to find something out. A harness whose + * people answer slowly, or whose model always has other work, says false. + */ + readonly wait?: boolean; + /** The name the model calls it by, for a harness that already has one. */ + readonly name?: string; +} + +/** + * The tool, given a way to ask. + * + * **It holds one permit, so the asker is never called again before the last + * call answers.** That is what lets a caller write one that owns the terminal, + * or one dialog, without a lock of its own. + * + * The permit is here rather than in the session because the session is not what + * it protects: the asker is, and one asker is one terminal and one person. A + * session knows nothing about either. So one `questionTool(ask)` is one person + * asked one thing at a time, however many sessions call it — and two of them + * over one asker is two permits and two dialogs, which is a reason to build one. + */ +const questionTool = (ask: Asker, options?: QuestionOptions): Tool => { + const permit = Effect.unsafeMakeSemaphore(1); + return { + definition: { name: options?.name ?? 'question', description, parameters }, + /* The model asked because it cannot go on without the answer, so waiting is + what it wants. It is still only the default: a model with work the answer + does not block says so on the call, and the deadline still moves it on. */ + wait: options?.wait ?? true, + ...(options?.inlineFor === undefined ? {} : { inlineFor: options.inlineFor }), + run: (call: ToolCall) => + permit.withPermits(1)( + Effect.flatMap(asked(call), ({ questions }) => + ask(questions).pipe( + Effect.mapError(cause => + cause instanceof ToolFailure ? cause : new ToolFailure({ cause }) + ), + Effect.map(answers => wordsFor(questions, answers)) + ) + ) + ), + }; +}; + +export type { Answer, Asker, Choice, Question, QuestionOptions }; +export { questionTool }; diff --git a/packages/harness-sdk/src/plugins/tools/subagent.test.ts b/packages/harness-sdk/src/plugins/tools/subagent.test.ts new file mode 100644 index 0000000000..90cd2796b1 --- /dev/null +++ b/packages/harness-sdk/src/plugins/tools/subagent.test.ts @@ -0,0 +1,226 @@ +import { Effect, Layer, Stream } from 'effect'; +import { expect, it } from 'vitest'; +import { ModelError, type ModelUsage } from '../../core/model.js'; +import { recordingStore, runWith } from '../../core/session-fixture.js'; +import { ToolRegistry } from '../../core/tool.js'; +import { layerTableCatalog } from '../catalog/table.js'; +import { layerSeededEntropy } from '../entropy/seeded.js'; +import { fakeModel, type FakeReply } from '../model/fake.js'; +import { layerAssembler } from '../prompt/default.js'; +import { + type SubagentContext, + type SubagentOptions, + type SubagentReport, + subagentTool, +} from './subagent.js'; + +/** + * A tool that is a session of its own. + * + * What is under test is what crosses between the two. One answer goes up to the + * parent; the subagent's own steps stay in its own transcript, which is what + * makes a subagent worth having. Nothing else crosses on its own: the counts go + * to whoever asked for them, and a subagent that fails is a failed result and + * not a failed session. + */ + +/** + * The model asks to wait, which the tool does not do by default. + * + * Handing a task over is how a model carries on, so `subagentTool` says + * `wait: false` and every call to it is backgrounded at once. What crosses + * between parent and subagent is the same either way, and reading it where the + * call was made is what keeps each test below about its own subject. The + * default has a test of its own at the end. + */ +const call = { + id: 'tc_1', + name: 'subagent', + arguments: '{"task":"count the files","wait":true}', +}; + +const options = { + system: 'sys', + model: 'claude-opus-5', + maxTokens: 1024, + tools: ['subagent'], +}; + +/** A tool only the subagent has, so a step of its own is a step to look for. */ +const look = Layer.succeed(ToolRegistry, { + tools: [ + { + definition: { + name: 'look', + description: 'Counts the files.', + parameters: { type: 'object', properties: {} }, + }, + run: () => Effect.succeed('nine files'), + }, + ], +}); + +/** The layers a subagent runs under, with a model of its own to script. */ +const under = (replies: readonly FakeReply[]) => { + const model = fakeModel(replies); + return { + model, + layers: Layer.mergeAll( + layerAssembler, + layerTableCatalog({}, { apiKinds: ['messages'] }), + layerSeededEntropy(2), + model.layer, + look + ), + }; +}; + +const answered = (events: readonly { readonly kind: string }[]) => + events.filter(event => event.kind === 'toolResult'); + +/** The parent's registry, holding one subagent over the layers given. */ +const offering = (layers: Layer.Layer, extra: Partial = {}) => + Layer.succeed(ToolRegistry, { + tools: [ + subagentTool({ system: 'You count things.', model: 'claude-haiku-4-5', ...extra }, layers), + ], + }); + +/** + * One parent round in which the subagent takes two rounds of its own: it calls + * its own tool, reads what the tool said, and only then answers. + */ +const twoRounds = () => { + const reports: SubagentReport[] = []; + const sub = under([ + { deltas: ['looking'], calls: [{ id: 'sc_1', name: 'look', arguments: '{}' }], stop: 'tools' }, + { deltas: ['there are nine files'] }, + ]); + const store = recordingStore(); + const ran = runWith({ + options, + store: store.layer, + tools: offering(sub.layers, { + tools: ['look'], + onFinished: report => Effect.sync(() => void reports.push(report)), + }), + replies: [{ deltas: [], calls: [call], stop: 'tools' }, { deltas: ['nine, it says'] }], + use: session => + Effect.gen(function* () { + const events = [...(yield* Stream.runCollect(session.ask('how many files?')))]; + return { id: session.id, events, history: yield* session.history }; + }), + }); + return { reports, sub, store, ran }; +}; + +const textIn = (turns: readonly { readonly parts: readonly { readonly body: string }[] }[]) => + turns.map(turn => turn.parts.map(part => part.body).join('')).join('|'); + +it('hands the parent one answer, and keeps the subagent’s steps to itself', async () => { + const { sub, ran } = twoRounds(); + const { value } = await ran; + + /* One string reached the parent: what the subagent finally said. */ + expect(answered(value.events)).toMatchObject([ + { result: { callId: 'tc_1', body: 'there are nine files', failed: false } }, + ]); + /* And nothing it did on the way is in the parent's transcript. */ + expect(textIn(value.history)).not.toContain('looking'); + expect(textIn(value.history)).toContain('there are nine files'); + expect(sub.model.calls).toHaveLength(2); +}); + +it('writes the subagent to the parent’s store, under a session of its own', async () => { + const { reports, store, ran } = twoRounds(); + const { value } = await ran; + + /* A session reads the store from the context it runs in, and a tool runs in + the parent's. So both wrote to one database — under two session + identifiers, which is what a store keyed by session is for. */ + expect(store.seen.join('|')).toContain('assistant:looking'); + expect(reports[0]?.sessionId).not.toBe(value.id); +}); + +it('counts the subagent’s tokens against the subagent, and hands them over', async () => { + const reports: SubagentReport[] = []; + const sub = under([{ deltas: ['nine'], usage: { inputTokens: 40, outputTokens: 9 } }]); + + const { value } = await runWith({ + options, + tools: offering(sub.layers, { + onFinished: report => Effect.sync(() => void reports.push(report)), + }), + replies: [ + { deltas: [], calls: [call], stop: 'tools', usage: { inputTokens: 11, outputTokens: 2 } }, + { deltas: ['nine, it says'], usage: { inputTokens: 12, outputTokens: 3 } }, + ], + use: session => Effect.zipRight(Stream.runDrain(session.ask('how many?')), session.usage), + }); + + const parent: ModelUsage = value; + /* The parent paid for its own two calls and nothing else. */ + expect(parent.inputTokens).toBe(23); + expect(parent.outputTokens).toBe(5); + /* The subagent's counts are its own, and reach the caller that asked. */ + expect(reports).toMatchObject([{ said: 'nine', usage: { inputTokens: 40, outputTokens: 9 } }]); + expect(reports[0]?.sessionId).not.toBe(''); +}); + +it('hands the model a failed result when the subagent fails', async () => { + const sub = under([ + { deltas: [], fail: new ModelError({ reason: 'transport', cause: 'no route' }) }, + ]); + + const { value } = await runWith({ + options, + tools: offering(sub.layers), + replies: [{ deltas: [], calls: [call], stop: 'tools' }, { deltas: ['I will do it myself'] }], + use: session => Stream.runCollect(session.ask('how many files?')), + }); + + const [result] = answered([...value]); + expect(result).toMatchObject({ result: { callId: 'tc_1', failed: true } }); + /* The parent read what went wrong and carried on, which is the whole point of + a failed result rather than a failed session. */ + expect(String(Reflect.get(result ?? {}, 'result'))).not.toBe(''); +}); + +it('refuses a task it cannot read, without opening a session', async () => { + const sub = under([{ deltas: ['never asked'] }]); + + const { value } = await runWith({ + options, + tools: offering(sub.layers), + replies: [ + { deltas: [], calls: [{ ...call, arguments: '{"job":"count","wait":true}' }], stop: 'tools' }, + { deltas: ['I will do it myself'] }, + ], + use: session => Stream.runCollect(session.ask('how many files?')), + }); + + expect(answered([...value])).toMatchObject([{ result: { failed: true } }]); + /* Nothing was asked of the subagent's model, so nothing was spent on it. */ + expect(sub.model.calls).toHaveLength(0); +}); + +it('hands the task over and carries on, when the model says nothing', async () => { + const sub = under([{ deltas: ['there are nine files'] }]); + + const { value } = await runWith({ + options, + tools: offering(sub.layers), + replies: [ + { deltas: [], calls: [{ ...call, arguments: '{"task":"count the files"}' }], stop: 'tools' }, + { deltas: ['I have asked, and I will say when it answers'] }, + ], + use: session => Stream.runCollect(session.ask('how many files?')), + }); + + /* The whole point of handing a task over is to carry on. The answer still + comes back, in a round of its own, like any backgrounded call. */ + const [result] = answered([...value]); + expect(String(Reflect.get(Reflect.get(result ?? {}, 'result') ?? {}, 'body'))).toContain( + 'still running' + ); +}); diff --git a/packages/harness-sdk/src/plugins/tools/subagent.ts b/packages/harness-sdk/src/plugins/tools/subagent.ts new file mode 100644 index 0000000000..15f8d1d3e6 --- /dev/null +++ b/packages/harness-sdk/src/plugins/tools/subagent.ts @@ -0,0 +1,216 @@ +import { type Duration, Effect, type Layer, type Scope, Stream } from 'effect'; +import { createAssert } from 'typia'; +import type { Effort, ModelUsage } from '../../core/model.js'; +import { openSession } from '../../core/run.js'; +import { type JsonSchema, type Tool, ToolFailure, type ToolCall } from '../../core/tool.js'; +import type { SessionContext } from '../../core/wiring.js'; + +/** + * A tool that is a session of its own. + * + * The model asks for one thing in one sentence, and something else goes and + * does it: its own system prompt, its own model, its own tools, and a + * transcript the parent never sees. What comes back is one answer, so the + * parent pays for the answer rather than for every step that produced it. That + * is the whole point of a subagent — a long search, a file read three ways, a + * draft rewritten twice, all of it kept out of a conversation that has to stay + * cheap to replay. + * + * Nothing here is new machinery. A subagent is `openSession` called from inside + * a tool, which is why it takes the layers it should run under: a tool is + * handed no context, so whatever the session needs must be given to the tool + * when it is built. The layers may be the parent's own or another set entirely, + * which is how a subagent runs on a cheaper model than the one that called it. + * + * What reaches the parent is one string, and what it says on the way to its own + * tools is not part of it: a model narrates before it calls something, and + * handing that up would put back the noise a subagent exists to absorb. + * + * The store is the one thing that is shared, and it is shared on purpose. A + * session reads `SessionStore` from whatever context it runs in, and a tool runs + * inside the parent's, so a subagent writes to the same database — under a + * session of its own. One database, two transcripts, neither poisoning the + * other. Pass layers with a store of their own to separate even that. + * + * What reaches the parent's `usage` is nothing, because the counts belong to + * the session that spent them. `onFinished` hands them over for a caller that + * is adding up what a conversation cost. + * + * It is a tool like any other, so the session's deadline applies to it and a + * caller can send a running one to the background. A subagent is usually the + * longest call in a harness, so that matters more here than anywhere else. + */ + +/** What the subagent was asked to do. Anything else is a failed result. */ +interface Asked { + readonly task: string; +} + +const assertAsked = createAssert(); + +/** Everything the subagent's session is opened with, and who to tell about it. */ +interface SubagentOptions { + /** The subagent's own system prompt. It never sees the parent's. */ + readonly system: string; + readonly model: string; + /** The name the model calls it by. `subagent` unless a harness has its own. */ + readonly name?: string; + /** What the model reads about it. Say what it is for, in the harness's words. */ + readonly description?: string; + /** + * The tools the subagent may use, by name, out of the registry in its layers. + * A subagent offered the tool that starts it can start one of its own, and + * nothing here stops that: how deep is the harness's decision, not this + * package's. + */ + readonly tools?: readonly string[]; + readonly maxTokens?: number; + readonly effort?: Effort; + /** + * How long the parent's model waits before the call goes to the background. + * A subagent that reads or searches usually outlives a request, so this is + * worth setting where a harness knows. + */ + readonly inlineFor?: Duration.DurationInput; + /** + * Whether the model waits for the subagent, as it is told by default. False, + * because handing a task over is how a model carries on. A harness whose + * subagents are quick, or whose parent has nothing else to do, says true. + */ + readonly wait?: boolean; + /** + * Told what one subagent cost and where its transcript is, once it has + * answered. This is the only thing that crosses back: a caller adding up what + * a conversation spent needs the subagent's counts, and the parent session + * cannot see them. + */ + readonly onFinished?: (report: SubagentReport) => Effect.Effect; +} + +/** What one subagent did, for the caller that is counting. */ +interface SubagentReport { + /** The subagent's own session, which is where its turns are stored. */ + readonly sessionId: string; + readonly usage: ModelUsage; + readonly said: string; +} + +/** Everything a subagent session needs, less the scope the tool opens itself. */ +type SubagentContext = Exclude; + +const parameters: JsonSchema = { + type: 'object', + properties: { + task: { + type: 'string', + description: + 'What to do, in full. The subagent starts fresh and knows nothing ' + + 'about this conversation, so say everything it needs in this one ' + + 'sentence, and say what you want back.', + }, + }, + required: ['task'], + additionalProperties: false, +}; + +/** + * Three sentences, two of which `e2e/tool-matrix.ts` bought. + * + * The first version named two uses — work of several steps, work that produces + * more reading than you need — and two of eleven models read those as the only + * two. Asked one thing they could not know, they answered that no tool of + * theirs could look it up and asked the person for a source. That is the right + * move if a subagent is only for long work, so the uses became a list to lead + * with rather than a gate, and the first one is what those two needed: + * something you cannot answer yourself. It moved one of them. + * + * The one left doubted the subagent could reach anything it could not, so the + * second sentence says what a subagent is: a session started from instructions + * of its own. That is true of every subagent, including one given no tools at + * all, which is why it can be said here rather than by the harness. + */ +const description = + 'Hands one task to a subagent, which goes and does it in a conversation of ' + + 'its own and answers with the result. It starts from instructions of its own, ' + + 'so it can reach what this conversation cannot. Use it for anything you ' + + 'cannot answer from what you already know: something to look up, somewhere to ' + + 'search, work that takes several steps, or work that produces more reading ' + + 'than you need to keep. It remembers nothing between calls and cannot ask you ' + + 'anything, so give it everything at once and say what you want back.'; + +/** What the model sent, or a failed result saying what was wrong with it. */ +const asked = (call: ToolCall): Effect.Effect => + Effect.try({ + try: () => assertAsked(JSON.parse(call.arguments)), + catch: cause => new ToolFailure({ cause }), + }); + +/** The subagent's own session, run to the end of one answer. */ +const answering = ( + options: SubagentOptions, + task: string +): Effect.Effect => + Effect.gen(function* () { + const session = yield* openSession({ + system: options.system, + model: options.model, + ...(options.tools === undefined ? {} : { tools: options.tools }), + ...(options.maxTokens === undefined ? {} : { maxTokens: options.maxTokens }), + ...(options.effort === undefined ? {} : { effort: options.effort }), + }); + const said = yield* Stream.runFold(session.ask(task), '', (held: string, event) => { + if (event.kind === 'delta') { + return held + event.text; + } + /* What the subagent said on the way to a tool is not its answer. A model + narrates before it calls something — "let me look" — and handing that + up would put the noise a subagent exists to absorb back into the + parent's transcript. The answer is what it said after its last call. */ + return event.kind === 'done' && event.stop === 'tools' ? '' : held; + }); + return { sessionId: session.id, usage: yield* session.usage, said }; + }); + +/** Tells whoever is counting, and hands the model the answer. */ +const finish = (options: SubagentOptions, report: SubagentReport): Effect.Effect => + Effect.as(options.onFinished?.(report) ?? Effect.void, report.said); + +/** + * The tool, given what the subagent is and the layers it runs under. + * + * A failure inside the subagent is a failed result and not a failed session: + * the parent model reads what went wrong and decides whether to ask again, ask + * differently, or carry on without it. + */ +const subagentTool = ( + options: SubagentOptions, + /** + * A layer that fails is a failed result like any other: the harness that + * built it hears about it through the model, which is the only party that can + * decide what to do without one. + */ + layers: Layer.Layer +): Tool => ({ + definition: { + name: options.name ?? 'subagent', + description: options.description ?? description, + parameters, + }, + /* Handing a task over is how the model carries on, so it does not wait for + one by default. A model that has nothing to do until the task is done says + so on the call. */ + wait: options.wait ?? false, + ...(options.inlineFor === undefined ? {} : { inlineFor: options.inlineFor }), + run: (call: ToolCall) => + Effect.flatMap(asked(call), ({ task }) => + Effect.scoped(Effect.provide(answering(options, task), layers)).pipe( + Effect.mapError(cause => + cause instanceof ToolFailure ? cause : new ToolFailure({ cause }) + ), + Effect.flatMap(report => finish(options, report)) + ) + ), +}); + +export type { SubagentContext, SubagentOptions, SubagentReport }; +export { subagentTool }; diff --git a/packages/harness-sdk/src/plugins/tools/time.test.ts b/packages/harness-sdk/src/plugins/tools/time.test.ts new file mode 100644 index 0000000000..428d070bb2 --- /dev/null +++ b/packages/harness-sdk/src/plugins/tools/time.test.ts @@ -0,0 +1,68 @@ +import { Effect, TestClock, TestContext } from 'effect'; +import { expect, it } from 'vitest'; +import type { ToolCall } from '../../core/tool.js'; +import { timeTool } from './time.js'; + +/** + * The clock is pinned, so these assert the answer rather than assert around a + * number that moves while they run. That is the whole reason the tool reads + * `Clock` instead of calling `Date.now` itself. + */ + +const call: ToolCall = { id: 'tc_1', name: 'time', arguments: '{}' }; + +/** A Thursday, deliberately: the weekday has to come from the date. */ +const at = Date.parse('2026-09-03T17:04:09.512Z'); + +const asked = (tool: ReturnType, one: ToolCall = call): Promise => + Effect.runPromise( + Effect.provide( + Effect.flatMap(TestClock.setTime(at), () => tool.run(one)), + TestContext.TestContext + ) + ); + +it('answers with the time the clock says, to the second', async () => { + expect(await asked(timeTool())).toBe('2026-09-03T17:04:09Z (Thursday, UTC)'); +}); + +it('gives the local time too when the harness named a zone', async () => { + const said = await asked(timeTool({ zone: 'Europe/Amsterdam' })); + + /* Two hours ahead of UTC in September, which is the point of asking. */ + expect(said).toBe('2026-09-03T17:04:09Z (Thursday, UTC)\nEurope/Amsterdam: 2026-09-03 19:04:09'); +}); + +it('says midnight as hour zero, not hour twenty-four', async () => { + /* 22:30 UTC is 00:30 the next day in Amsterdam, which is the hour the two + ways of asking for a 24-hour clock disagree about. */ + const midnight = Date.parse('2026-09-03T22:30:00.000Z'); + const said = await Effect.runPromise( + Effect.provide( + Effect.flatMap(TestClock.setTime(midnight), () => + timeTool({ zone: 'Europe/Amsterdam' }).run(call) + ), + TestContext.TestContext + ) + ); + + expect(said).toContain('Europe/Amsterdam: 2026-09-04 00:30:00'); +}); + +it('answers a call that carries a field nobody reads', async () => { + const noisy: ToolCall = { ...call, arguments: '{"zone":"Mars/Olympus"}' }; + + /* A model that sends more than the schema asks for is answered rather than + failed: nothing here reads the arguments, so there is nothing to be wrong. */ + expect(await asked(timeTool(), noisy)).toBe('2026-09-03T17:04:09Z (Thursday, UTC)'); +}); + +it('is waited for, because the answer is the reason it was called', () => { + const tool = timeTool(); + + /* No `wait` of its own: the deadline decides, and a tool with no deadline + advertises true. A model told to carry on without the time would have to + ask again to use it. */ + expect(tool.wait).toBeUndefined(); + expect(tool.inlineFor).toBeUndefined(); +}); diff --git a/packages/harness-sdk/src/plugins/tools/time.ts b/packages/harness-sdk/src/plugins/tools/time.ts new file mode 100644 index 0000000000..777c7f42c5 --- /dev/null +++ b/packages/harness-sdk/src/plugins/tools/time.ts @@ -0,0 +1,114 @@ +import { Clock, Effect } from 'effect'; +import type { JsonSchema, Tool, ToolCall } from '../../core/tool.js'; + +/** + * What time it is. + * + * A model does not know. It knows roughly when it was trained, states that date + * with the same confidence it states everything else, and is wrong by however + * long it has been since. Every harness hits this: a model asked how old a file + * is, whether a deadline has passed, or what to put at the top of a changelog + * answers from a stale prior and nothing in the reply says so. + * + * The fix is one call, and it is the package's to own for the same reason + * `question` is: every harness needs it, and there is nothing about it a + * harness would write differently. + * + * It reads the clock through Effect's `Clock`, so a test pins it rather than + * asserting around a moving number. + */ + +/** + * Named rather than derived, because deriving the weekday needs `Intl` and this + * does not. A model asked what day it is should not depend on a runtime's + * locale data being complete. + */ +const weekdays = [ + 'Sunday', + 'Monday', + 'Tuesday', + 'Wednesday', + 'Thursday', + 'Friday', + 'Saturday', +] as const; + +/** No arguments. There is nothing about the current time for a model to choose. */ +const parameters: JsonSchema = { + type: 'object', + properties: {}, + additionalProperties: false, +}; + +const description = + 'Answers with the current date and time. Call it whenever the answer depends ' + + 'on what the date is now — how old something is, whether a deadline has ' + + 'passed, what to write as today — rather than working from the date you were ' + + 'trained on, which is in the past and which you cannot tell has moved.'; + +/** What the caller may change. Everything else is the clock's. */ +interface TimeOptions { + /** The name the model calls it by, for a harness that already has one. */ + readonly name?: string; + /** + * An IANA zone — `Europe/Amsterdam` — to give the local time in as well. + * + * The harness's and never the model's: a model naming its own zone guesses, + * and a guess that is not a zone is a failed call for no gain. UTC is always + * given, so a harness that leaves this out loses nothing a model can reason + * with, and a runtime without complete `Intl` data must leave it out. + */ + readonly zone?: string; +} + +/** ISO 8601 to the second. The milliseconds are noise in an answer. */ +const utcOf = (at: Date): string => `${at.toISOString().slice(0, 19)}Z`; + +/** + * The same layout as the UTC line, assembled from named parts. + * + * The parts are read out by name rather than taking a locale's own formatting, + * so the answer does not change shape with the runtime's locale data. The + * shorter version of this was `toLocaleString('sv-SE')`, which reads as ISO + * only because Swedish convention happens to be — and which falls back to + * another format, silently and with no error, on a runtime whose ICU data does + * not carry Swedish. + * + * `h23` and not `hour12: false`: the two differ at midnight, where some ICU + * versions answer hour 24. + */ +const localOf = (at: Date, zone: string): string => { + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: zone, + hourCycle: 'h23', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }).formatToParts(at); + const of = (type: string): string => parts.find(part => part.type === type)?.value ?? ''; + return `${of('year')}-${of('month')}-${of('day')} ${of('hour')}:${of('minute')}:${of('second')}`; +}; + +const wordsFor = (at: Date, zone: string | undefined): string => { + const now = `${utcOf(at)} (${weekdays[at.getUTCDay()] ?? ''}, UTC)`; + return zone === undefined ? now : `${now}\n${zone}: ${localOf(at, zone)}`; +}; + +/** + * The tool. + * + * `run` reads nothing off the call, so there is nothing to validate: the model + * sends `{}` and a model that sends more is answered anyway rather than being + * failed over a field nobody reads. + */ +const timeTool = (options?: TimeOptions): Tool => ({ + definition: { name: options?.name ?? 'time', description, parameters }, + run: (_call: ToolCall) => + Effect.map(Clock.currentTimeMillis, (at: number) => wordsFor(new Date(at), options?.zone)), +}); + +export type { TimeOptions }; +export { timeTool }; diff --git a/packages/harness-sdk/src/plugins/tools/todo.test.ts b/packages/harness-sdk/src/plugins/tools/todo.test.ts new file mode 100644 index 0000000000..5753cbdf43 --- /dev/null +++ b/packages/harness-sdk/src/plugins/tools/todo.test.ts @@ -0,0 +1,102 @@ +import { Effect } from 'effect'; +import { expect, it } from 'vitest'; +import { type ToolCall, ToolFailure } from '../../core/tool.js'; +import { type Todo, todoTool } from './todo.js'; + +/** + * What the model sends is the model's, and every shape here is one it will send + * sooner or later. What comes back is what it then has to read. + */ + +const call = (args: unknown): ToolCall => ({ + id: 'tc_1', + name: 'todo', + arguments: JSON.stringify(args), +}); + +const run = (tool: ReturnType, one: ToolCall) => + Effect.runPromise(Effect.either(tool.run(one))); + +const said = async (tool: ReturnType, todos: readonly unknown[]) => { + const got = await run(tool, call({ todos })); + return got._tag === 'Right' ? got.right : `FAILED: ${String(got.left.cause)}`; +}; + +it('reads the list back with a mark for each state', async () => { + const list = await said(todoTool(), [ + { text: 'Read the spec', state: 'done' }, + { text: 'Write the parser', state: 'doing' }, + { text: 'Ship it', state: 'pending' }, + ]); + + expect(list).toBe('[x] Read the spec\n[>] Write the parser\n[ ] Ship it'); +}); + +it('replaces the list rather than adding to it', async () => { + const tool = todoTool(); + await said(tool, [{ text: 'First', state: 'pending' }]); + + /* The second call does not mention the first step, so the first step is gone. + A model that meant to keep it sends it again; that is the whole contract. */ + expect(await said(tool, [{ text: 'Second', state: 'doing' }])).toBe('[>] Second'); +}); + +it('says so rather than answering nothing when the list is emptied', async () => { + const tool = todoTool(); + await said(tool, [{ text: 'First', state: 'pending' }]); + + expect(await said(tool, [])).toBe('The list is empty.'); +}); + +it('keeps one list per tool, not one per call', async () => { + const one = todoTool(); + const other = todoTool(); + await said(one, [{ text: 'Mine', state: 'doing' }]); + + /* Two tools built separately do not share a list. A harness that wants a + list per session builds a tool per session, which is what makes that work. */ + expect(await said(other, [])).toBe('The list is empty.'); + expect(await said(one, [{ text: 'Mine', state: 'done' }])).toBe('[x] Mine'); +}); + +it('tells the harness what changed, so it can draw it', async () => { + const drawn: (readonly Todo[])[] = []; + const tool = todoTool({ onChanged: todos => Effect.sync(() => void drawn.push(todos)) }); + + await said(tool, [{ text: 'Only', state: 'pending' }]); + + expect(drawn).toStrictEqual([[{ text: 'Only', state: 'pending' }]]); +}); + +it('tells the model when the harness could not draw the list', async () => { + const tool = todoTool({ onChanged: () => Effect.fail(new Error('no terminal')) }); + + const got = await run(tool, call({ todos: [{ text: 'Only', state: 'pending' }] })); + + /* The person cannot see what the model wrote down, which is a thing the model + needs to know rather than a thing to swallow. */ + expect(got._tag).toBe('Left'); + expect(got._tag === 'Left' && got.left).toBeInstanceOf(ToolFailure); +}); + +it('refuses a state it does not have, rather than storing it', async () => { + const tool = todoTool(); + + const got = await run(tool, call({ todos: [{ text: 'Only', state: 'nearly' }] })); + + expect(got._tag).toBe('Left'); + /* And the list is untouched, so a bad call does not lose what was there. */ + expect(await said(tool, [])).toBe('The list is empty.'); +}); + +it('refuses a step with no text', async () => { + const got = await run(todoTool(), call({ todos: [{ state: 'pending' }] })); + + expect(got._tag).toBe('Left'); +}); + +it('refuses arguments that are not a list at all', async () => { + const got = await run(todoTool(), { id: 'tc_1', name: 'todo', arguments: 'not json' }); + + expect(got._tag).toBe('Left'); +}); diff --git a/packages/harness-sdk/src/plugins/tools/todo.ts b/packages/harness-sdk/src/plugins/tools/todo.ts new file mode 100644 index 0000000000..fc3906f592 --- /dev/null +++ b/packages/harness-sdk/src/plugins/tools/todo.ts @@ -0,0 +1,145 @@ +import { Effect, Ref } from 'effect'; +import { createAssert } from 'typia'; +import { type JsonSchema, type Tool, ToolFailure, type ToolCall } from '../../core/tool.js'; + +/** + * The list a model keeps of what it is doing. + * + * A model given a task of several steps forgets one, does two at once, or says + * it is finished with a step still open. Writing the steps down and reading + * them back is the fix every harness reaches for, and every harness writes the + * same one, which is what makes it the package's. + * + * The model sends the whole list every time rather than a change to it. That is + * deliberate and it is the difference between a tool that works and one that + * does not: patching needs stable identifiers, models invent them, and a patch + * against an identifier that does not exist either fails the call or silently + * edits the wrong line. A whole list cannot be wrong about what it means. + * + * What comes back is the list as it now stands, so the model reads its own + * state rather than trusting what it thinks it sent. + */ + +/** Where one step has got to. */ +type TodoState = 'pending' | 'doing' | 'done'; + +interface Todo { + readonly text: string; + readonly state: TodoState; +} + +/** What the model sends. Anything else is a failed result and a chance to retry. */ +interface Asked { + readonly todos: readonly Todo[]; +} + +const assertAsked = createAssert(); + +const parameters: JsonSchema = { + type: 'object', + properties: { + todos: { + type: 'array', + description: + 'The whole list, in order, as it should now stand. Send every step ' + + 'every time, including the ones that have not changed: this replaces ' + + 'the list rather than adding to it.', + items: { + type: 'object', + properties: { + text: { + type: 'string', + description: 'The step, as one short imperative line.', + }, + state: { + type: 'string', + enum: ['pending', 'doing', 'done'], + description: + 'Where this step has got to. Exactly one step is `doing` at a ' + + 'time; mark it `done` before starting the next.', + }, + }, + required: ['text', 'state'], + additionalProperties: false, + }, + }, + }, + required: ['todos'], + additionalProperties: false, +}; + +const description = + 'Writes down what you are doing, and reads the list back. Use it for a task ' + + 'of several steps: put the steps down before you start, mark one `doing` ' + + 'while you are on it, and mark it `done` the moment it is finished rather ' + + 'than in a batch at the end. Send the whole list every time — it replaces ' + + 'what is there. A task of one step does not need it.'; + +const marks: Readonly> = { + pending: '[ ]', + doing: '[>]', + done: '[x]', +}; + +/** The list as the model reads it back. */ +const wordsFor = (todos: readonly Todo[]): string => + todos.length === 0 + ? 'The list is empty.' + : todos.map(todo => `${marks[todo.state]} ${todo.text}`).join('\n'); + +/** What the caller may change. Everything about the steps is the model's. */ +interface TodoOptions { + /** The name the model calls it by, for a harness that already has one. */ + readonly name?: string; + /** + * Told the list every time it changes, for a harness that draws it. + * + * It may fail, and a failure reaches the model as a failed result: a harness + * that could not draw the list has told the model something worth knowing, + * which is that the person cannot see what it wrote down. + */ + readonly onChanged?: (todos: readonly Todo[]) => Effect.Effect; +} + +/** + * The tool. + * + * **The list belongs to this tool, not to a session.** It is held in a `Ref` + * made when the tool is built, so a registry shared by a parent and its + * subagents shares one list between them. That is right for a harness whose + * subagent works on the parent's plan, and wrong for one whose subagents run + * unrelated errands; the second builds a tool per session. There is no third + * option the package could pick, because a tool is handed no session — see + * "A tool is handed no context" in AGENTS.md. + * + * It holds one permit, beside the list it protects. The model asks for several + * tools at once and this one replaces the whole list, so two overlapping calls + * would lose one of the two writes entirely rather than merging badly. + */ +const todoTool = (options?: TodoOptions): Tool => { + const held = Ref.unsafeMake([]); + const permit = Effect.unsafeMakeSemaphore(1); + return { + definition: { name: options?.name ?? 'todo', description, parameters }, + run: (call: ToolCall) => + permit.withPermits(1)( + Effect.try({ + try: () => assertAsked(JSON.parse(call.arguments)), + catch: cause => new ToolFailure({ cause }), + }).pipe( + Effect.tap(({ todos }) => Ref.set(held, todos)), + Effect.tap(({ todos }) => + (options?.onChanged?.(todos) ?? Effect.void).pipe( + Effect.mapError(cause => + cause instanceof ToolFailure ? cause : new ToolFailure({ cause }) + ) + ) + ), + Effect.flatMap(() => Effect.map(Ref.get(held), wordsFor)) + ) + ), + }; +}; + +export type { Todo, TodoOptions, TodoState }; +export { todoTool }; diff --git a/packages/harness-sdk/tsconfig.build.json b/packages/harness-sdk/tsconfig.build.json new file mode 100644 index 0000000000..d1678cf631 --- /dev/null +++ b/packages/harness-sdk/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "exclude": [ + "node_modules", + "dist", + "src/**/*.test.ts", + "src/plugins/**/fake.ts", + "src/plugins/**/test-gateway.ts", + "src/**/*-fixture.ts" + ] +} diff --git a/packages/harness-sdk/tsconfig.json b/packages/harness-sdk/tsconfig.json new file mode 100644 index 0000000000..b92c59cfe7 --- /dev/null +++ b/packages/harness-sdk/tsconfig.json @@ -0,0 +1,37 @@ +{ + "compilerOptions": { + "target": "es2023", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["esnext"], + "types": [], + + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "allowUnreachableCode": false, + "allowUnusedLabels": false, + "useDefineForClassFields": true, + "verbatimModuleSyntax": true, + "erasableSyntaxOnly": true, + "isolatedModules": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + + "rootDir": "./src", + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + + "plugins": [{ "transform": "typia/lib/transform" }] + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/harness-sdk/vitest.config.ts b/packages/harness-sdk/vitest.config.ts new file mode 100644 index 0000000000..01c207cc31 --- /dev/null +++ b/packages/harness-sdk/vitest.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vitest/config'; +import ttsc from '@ttsc/unplugin/vite'; + +// The tests import source, and typia's checks only exist after its transform runs. +// Without this plugin every `typia.*` call throws "no transform has been configured". +export default defineConfig({ + plugins: [ttsc()], + test: { + include: ['src/**/*.test.ts'], + // The timing tests are a separate gate: `pnpm test:perf`. + exclude: ['src/**/*.perf.test.ts'], + environment: 'node', + }, +}); diff --git a/packages/harness-sdk/vitest.perf.config.ts b/packages/harness-sdk/vitest.perf.config.ts new file mode 100644 index 0000000000..de4e5e8abb --- /dev/null +++ b/packages/harness-sdk/vitest.perf.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'vitest/config'; +import ttsc from '@ttsc/unplugin/vite'; + +// The timing gate. Its ceilings are about five times the recorded numbers, so +// it catches a regression in order of magnitude and not a busy machine. +export default defineConfig({ + plugins: [ttsc()], + test: { + include: ['src/**/*.perf.test.ts'], + environment: 'node', + // One file at a time: parallel workers compete for the CPU being measured. + fileParallelism: false, + testTimeout: 60_000, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 75b258dcbd..5810d0ef5d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1509,6 +1509,46 @@ importers: specifier: 'catalog:' version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@25.5.2)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@29.1.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) + packages/harness-sdk: + dependencies: + drizzle-orm: + specifier: 0.45.2 + version: 0.45.2(@cloudflare/workers-types@4.20260605.1)(@opentelemetry/api@1.9.1)(@types/pg@8.18.0)(@upstash/redis@1.38.0)(bun-types@1.3.14)(expo-sqlite@57.0.1(expo@57.0.15)(react-native@0.86.2(@types/react@19.2.14)(bufferutil@4.1.0)(react@19.2.6)(utf-8-validate@6.0.6))(react@19.2.6))(kysely@0.29.2)(pg@8.20.0) + effect: + specifier: 3.22.1 + version: 3.22.1 + eventsource-parser: + specifier: 3.0.8 + version: 3.0.8 + typia: + specifier: 14.0.4 + version: 14.0.4(ttsc@0.28.3) + devDependencies: + '@anthropic-ai/sdk': + specifier: 0.104.1 + version: 0.104.1(zod@4.4.3) + '@ttsc/unplugin': + specifier: 0.28.3 + version: 0.28.3(ttsc@0.28.3) + '@types/node': + specifier: 'catalog:' + version: 24.12.4 + drizzle-kit: + specifier: 'catalog:' + version: 0.31.10 + openai: + specifier: 6.49.0 + version: 6.49.0(@aws-sdk/credential-provider-node@3.972.21)(ws@8.21.0(bufferutil@4.1.0)(utf-8-validate@6.0.6))(zod@4.4.3) + ttsc: + specifier: 0.28.3 + version: 0.28.3 + typescript: + specifier: 7.0.2 + version: 7.0.2 + vitest: + specifier: 'catalog:' + version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@29.1.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) + packages/kilo-chat: dependencies: '@kilocode/event-service': @@ -10192,6 +10232,46 @@ packages: resolution: {integrity: sha512-w071DSzP94YfN6XiWhOxnLpYT3uqtxJBDYdh6Jdjzt+Ce6DNspJsPQgpC7rbts/B8tEkq0LHoYuIF/O5Jh5rPg==} engines: {node: '>=18'} + '@ttsc/darwin-arm64@0.28.3': + resolution: {integrity: sha512-ZdsiVD1OWfecgnBJkF11NX+C5iqLJ6XtEF4pr7nQW806hygSkK8DmMeWuRFerSqID6PQ4AzIhvpPeUKpMFsr0Q==} + cpu: [arm64] + os: [darwin] + + '@ttsc/darwin-x64@0.28.3': + resolution: {integrity: sha512-PubVbhvqJnk028bH9QTjas7u0PU2fMFwMezu96pkKC7zVTLekTVyZPOD1wKqN1/eaLvLagZygsGa98k+OlFQVQ==} + cpu: [x64] + os: [darwin] + + '@ttsc/linux-arm64@0.28.3': + resolution: {integrity: sha512-F4Kfr+xDOD+t+6MdiSi/u6ESsFdn76i/ErCg0nOmDFqG+l2Tm+9tJL66gw83f7/YBo/Ah8GxsyUBDW7/oFN61A==} + cpu: [arm64] + os: [linux] + + '@ttsc/linux-arm@0.28.3': + resolution: {integrity: sha512-46JdDKxBQPSQHeYQlFGPjnhBd96dH1JXhj+fODQGh0NAGcVoVNHr6WvWElL2YkyUbkjwkbdVKsMzprWwR0TeXQ==} + cpu: [arm] + os: [linux] + + '@ttsc/linux-x64@0.28.3': + resolution: {integrity: sha512-m5VLPOkPWZjrl11/BajN/rD4Np5knVK2Y9nHrt6j0LhIU8m6CAmfPyHGcd7Nw/W26SYvhh9SaHEQ799TjXZBoQ==} + cpu: [x64] + os: [linux] + + '@ttsc/unplugin@0.28.3': + resolution: {integrity: sha512-ctXXqshYGKts9Fpbxg1OPZ2ZhVwsBGFixNhX8CVuoFYWjIdmg0sgDVZQe97OvgI0SM3YngmBzQc8cNtM32ZW0Q==} + peerDependencies: + ttsc: ^0.28.3 + + '@ttsc/win32-arm64@0.28.3': + resolution: {integrity: sha512-hmXyPEr+JCoO2y4voD2zlhcoL+Ejk3+lp25nnuAN/zuusE959ahcDogFGXT3iEYhB6uVEffg3/wttbvTG7zb/Q==} + cpu: [arm64] + os: [win32] + + '@ttsc/win32-x64@0.28.3': + resolution: {integrity: sha512-SpjgxyckQDMXnMkUMHFGTQTvVunOhYQ3iotG+gQfd+hAysWa01Z4UA0SxEQhVDnBsn67NVZ/KaL4fXSRJ1WnsA==} + cpu: [x64] + os: [win32] + '@tybys/wasm-util@0.10.1': resolution: {integrity: sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==} @@ -10562,6 +10642,132 @@ packages: engines: {node: '>=16.20.0'} hasBin: true + '@typescript/typescript-aix-ppc64@7.0.2': + resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [aix] + + '@typescript/typescript-darwin-arm64@7.0.2': + resolution: {integrity: sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [darwin] + + '@typescript/typescript-darwin-x64@7.0.2': + resolution: {integrity: sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [darwin] + + '@typescript/typescript-freebsd-arm64@7.0.2': + resolution: {integrity: sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [freebsd] + + '@typescript/typescript-freebsd-x64@7.0.2': + resolution: {integrity: sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [freebsd] + + '@typescript/typescript-linux-arm64@7.0.2': + resolution: {integrity: sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [linux] + + '@typescript/typescript-linux-arm@7.0.2': + resolution: {integrity: sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==} + engines: {node: '>=16.20.0'} + cpu: [arm] + os: [linux] + + '@typescript/typescript-linux-loong64@7.0.2': + resolution: {integrity: sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==} + engines: {node: '>=16.20.0'} + cpu: [loong64] + os: [linux] + + '@typescript/typescript-linux-mips64el@7.0.2': + resolution: {integrity: sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==} + engines: {node: '>=16.20.0'} + cpu: [mips64el] + os: [linux] + + '@typescript/typescript-linux-ppc64@7.0.2': + resolution: {integrity: sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [linux] + + '@typescript/typescript-linux-riscv64@7.0.2': + resolution: {integrity: sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==} + engines: {node: '>=16.20.0'} + cpu: [riscv64] + os: [linux] + + '@typescript/typescript-linux-s390x@7.0.2': + resolution: {integrity: sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==} + engines: {node: '>=16.20.0'} + cpu: [s390x] + os: [linux] + + '@typescript/typescript-linux-x64@7.0.2': + resolution: {integrity: sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [linux] + + '@typescript/typescript-netbsd-arm64@7.0.2': + resolution: {integrity: sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [netbsd] + + '@typescript/typescript-netbsd-x64@7.0.2': + resolution: {integrity: sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [netbsd] + + '@typescript/typescript-openbsd-arm64@7.0.2': + resolution: {integrity: sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [openbsd] + + '@typescript/typescript-openbsd-x64@7.0.2': + resolution: {integrity: sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [openbsd] + + '@typescript/typescript-sunos-x64@7.0.2': + resolution: {integrity: sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [sunos] + + '@typescript/typescript-win32-arm64@7.0.2': + resolution: {integrity: sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [win32] + + '@typescript/typescript-win32-x64@7.0.2': + resolution: {integrity: sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [win32] + + '@typia/interface@14.0.4': + resolution: {integrity: sha512-5jLlBeEVpeH1p87sizstL16SWILulg+r9/TUvawAm9aZhgaVfrR8DX9oZekzvPTzFQm7RWLKMDdGz3fbMGRZ3Q==} + + '@typia/utils@14.0.4': + resolution: {integrity: sha512-43akDfhUdOUrJMzCoZiA0vTUvROaBuga45g0WvJSI/6egU0PIORbCVIXGia7RoN29io21VA1uZYuEykbCIHMJQ==} + '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} deprecated: Potential CWE-502 - Update to 1.3.1 or higher @@ -12700,6 +12906,9 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + effect@3.22.1: + resolution: {integrity: sha512-TNoXushmPOBAjJlthF5d2QwnX2xBPEtcNJr5XKNKbRLbDvBcOYkXlYDfvGfSA0zriwLFuCll5MDtNMAdZL17PQ==} + effect@4.0.0-beta.57: resolution: {integrity: sha512-rg32VgXnLKaPRs9tbRDaZ5jxmzNY7ojXt85gSHGUTwdlbWH5Ik+OCUY2q14TXliygPGoHwCAvNWS4bQJOqf00g==} @@ -13474,6 +13683,10 @@ packages: extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fast-check@3.23.2: + resolution: {integrity: sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==} + engines: {node: '>=8.0.0'} + fast-check@4.8.0: resolution: {integrity: sha512-GOJ158CUMnN6cSahsv4+ExARvIDuzzinFjkp0E9WtiBa5zcVeLozVkWaE4IzFcc+Y48Wp1EDlUZsXRyAztQcSg==} engines: {node: '>=12.17.0'} @@ -13866,9 +14079,6 @@ packages: get-tsconfig@4.13.7: resolution: {integrity: sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==} - get-tsconfig@4.14.0: - resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} - get-tsconfig@4.14.1: resolution: {integrity: sha512-Dz/6HxkrxgNehhxLVeyv8sad9UzF2xBVeaKBQNDfJ5XiSXmp2gTR0eO0RWiT2NCKS5aGP9jjkOMggTN90qU50A==} @@ -18506,6 +18716,11 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + ttsc@0.28.3: + resolution: {integrity: sha512-9+vVmqg08fMhX8mR7/ARYx/Qca+/uOOIQc7CAXVE5ccrXhL4YpXwVBWW3JPISjuAw3Dxwed03e8/rAZzE2uGrQ==} + engines: {node: '>=22.15.0'} + hasBin: true + tty-browserify@0.0.1: resolution: {integrity: sha512-C3TaO7K81YvjCgQH9Q1S3R3P3BtN3RIM8n+OvX4il1K1zgE8ZhI0op7kClgkxtutIE8hQrcrHBXvIheqKUUCxw==} @@ -18578,6 +18793,19 @@ packages: engines: {node: '>=14.17'} hasBin: true + typescript@7.0.2: + resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} + engines: {node: '>=16.20.0'} + hasBin: true + + typia@14.0.4: + resolution: {integrity: sha512-sUspRau04oH44UVU3YiRcXY5IvVYXq6rMkM73bKAD+Rn4S7MVv+xMRuv/seJXxf5bIPPs7VDMvT92HJJ+nFD3g==} + peerDependencies: + ttsc: '>=0.19.2' + peerDependenciesMeta: + ttsc: + optional: true + ua-parser-js@0.7.41: resolution: {integrity: sha512-O3oYyCMPYgNNHuO7Jjk3uacJWZF8loBgwrfd/5LE/HyZ3lUIOdniQ7DNXJcIgZbwioZxk0fLfI4EVnetdiX5jg==} hasBin: true @@ -21730,7 +21958,7 @@ snapshots: '@esbuild-kit/esm-loader@2.6.5': dependencies: '@esbuild-kit/core-utils': 3.3.2 - get-tsconfig: 4.14.0 + get-tsconfig: 4.14.1 '@esbuild/aix-ppc64@0.28.1': optional: true @@ -27837,6 +28065,32 @@ snapshots: '@ts-graphviz/ast': 2.0.7 '@ts-graphviz/common': 2.1.5 + '@ttsc/darwin-arm64@0.28.3': + optional: true + + '@ttsc/darwin-x64@0.28.3': + optional: true + + '@ttsc/linux-arm64@0.28.3': + optional: true + + '@ttsc/linux-arm@0.28.3': + optional: true + + '@ttsc/linux-x64@0.28.3': + optional: true + + '@ttsc/unplugin@0.28.3(ttsc@0.28.3)': + dependencies: + ttsc: 0.28.3 + unplugin: 2.3.11 + + '@ttsc/win32-arm64@0.28.3': + optional: true + + '@ttsc/win32-x64@0.28.3': + optional: true + '@tybys/wasm-util@0.10.1': dependencies: tslib: 2.8.1 @@ -28219,6 +28473,72 @@ snapshots: '@typescript/native-preview-win32-arm64': 7.0.0-dev.20260514.1 '@typescript/native-preview-win32-x64': 7.0.0-dev.20260514.1 + '@typescript/typescript-aix-ppc64@7.0.2': + optional: true + + '@typescript/typescript-darwin-arm64@7.0.2': + optional: true + + '@typescript/typescript-darwin-x64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-x64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm@7.0.2': + optional: true + + '@typescript/typescript-linux-loong64@7.0.2': + optional: true + + '@typescript/typescript-linux-mips64el@7.0.2': + optional: true + + '@typescript/typescript-linux-ppc64@7.0.2': + optional: true + + '@typescript/typescript-linux-riscv64@7.0.2': + optional: true + + '@typescript/typescript-linux-s390x@7.0.2': + optional: true + + '@typescript/typescript-linux-x64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-sunos-x64@7.0.2': + optional: true + + '@typescript/typescript-win32-arm64@7.0.2': + optional: true + + '@typescript/typescript-win32-x64@7.0.2': + optional: true + + '@typia/interface@14.0.4': {} + + '@typia/utils@14.0.4': + dependencies: + '@typia/interface': 14.0.4 + '@ungap/structured-clone@1.3.0': {} '@unrs/resolver-binding-android-arm-eabi@1.11.1': @@ -30497,6 +30817,11 @@ snapshots: ee-first@1.1.1: {} + effect@3.22.1: + dependencies: + '@standard-schema/spec': 1.1.0 + fast-check: 3.23.2 + effect@4.0.0-beta.57: dependencies: '@standard-schema/spec': 1.1.0 @@ -31665,6 +31990,10 @@ snapshots: extend@3.0.2: {} + fast-check@3.23.2: + dependencies: + pure-rand: 6.1.0 + fast-check@4.8.0: dependencies: pure-rand: 8.4.0 @@ -32113,10 +32442,6 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 - get-tsconfig@4.14.0: - dependencies: - resolve-pkg-maps: 1.0.0 - get-tsconfig@4.14.1: dependencies: resolve-pkg-maps: 1.0.0 @@ -38555,6 +38880,16 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + ttsc@0.28.3: + optionalDependencies: + '@ttsc/darwin-arm64': 0.28.3 + '@ttsc/darwin-x64': 0.28.3 + '@ttsc/linux-arm': 0.28.3 + '@ttsc/linux-arm64': 0.28.3 + '@ttsc/linux-x64': 0.28.3 + '@ttsc/win32-arm64': 0.28.3 + '@ttsc/win32-x64': 0.28.3 + tty-browserify@0.0.1: {} tw-animate-css@1.4.0: {} @@ -38607,6 +38942,37 @@ snapshots: typescript@6.0.3: {} + typescript@7.0.2: + optionalDependencies: + '@typescript/typescript-aix-ppc64': 7.0.2 + '@typescript/typescript-darwin-arm64': 7.0.2 + '@typescript/typescript-darwin-x64': 7.0.2 + '@typescript/typescript-freebsd-arm64': 7.0.2 + '@typescript/typescript-freebsd-x64': 7.0.2 + '@typescript/typescript-linux-arm': 7.0.2 + '@typescript/typescript-linux-arm64': 7.0.2 + '@typescript/typescript-linux-loong64': 7.0.2 + '@typescript/typescript-linux-mips64el': 7.0.2 + '@typescript/typescript-linux-ppc64': 7.0.2 + '@typescript/typescript-linux-riscv64': 7.0.2 + '@typescript/typescript-linux-s390x': 7.0.2 + '@typescript/typescript-linux-x64': 7.0.2 + '@typescript/typescript-netbsd-arm64': 7.0.2 + '@typescript/typescript-netbsd-x64': 7.0.2 + '@typescript/typescript-openbsd-arm64': 7.0.2 + '@typescript/typescript-openbsd-x64': 7.0.2 + '@typescript/typescript-sunos-x64': 7.0.2 + '@typescript/typescript-win32-arm64': 7.0.2 + '@typescript/typescript-win32-x64': 7.0.2 + + typia@14.0.4(ttsc@0.28.3): + dependencies: + '@typia/interface': 14.0.4 + '@typia/utils': 14.0.4 + randexp: 0.5.3 + optionalDependencies: + ttsc: 0.28.3 + ua-parser-js@0.7.41: {} uc.micro@2.1.0: {}