diff --git a/src/cost/faremeter.test.ts b/src/cost/faremeter.test.ts new file mode 100644 index 000000000..b29377a48 --- /dev/null +++ b/src/cost/faremeter.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; +import type { TokenUsage } from "@intx/types/runtime"; +import { createFaremeter } from "./faremeter.js"; + +const PRICES = { + inputPricePerToken: 1, + outputPricePerToken: 2, + cacheReadPricePerToken: 0.5, +}; + +describe("createFaremeter", () => { + test("bills uncached input at the input rate and cached reads at the cache rate", () => { + const faremeter = createFaremeter(PRICES); + // Normalized Responses-API usage: input excludes cached tokens (the + // adapter subtracts them), so the same token is never billed twice. + const usage: TokenUsage = { + input: 80, + output: 10, + cacheRead: 20, + cacheWrite: 0, + thinking: 0, + }; + + faremeter.addUsage(usage); + + expect(faremeter.getTotalCost()).toBe(80 * 1 + 10 * 2 + 20 * 0.5); + }); + + test("reports context occupancy as the full prompt size, not just uncached input", () => { + const faremeter = createFaremeter(PRICES); + faremeter.addUsage({ input: 200, output: 50, cacheRead: 800, cacheWrite: 0, thinking: 0 }); + + expect(faremeter.getInputTokens()).toBe(1000); + expect(faremeter.getTotalTokens()).toBe(1050); + }); + + test("accumulates cost across turns and counts thinking tokens as output volume", () => { + const faremeter = createFaremeter(PRICES); + faremeter.addUsage({ input: 100, output: 10, cacheRead: 0, cacheWrite: 0, thinking: 5 }); + faremeter.addUsage({ input: 40, output: 20, cacheRead: 60, cacheWrite: 0, thinking: 0 }); + + // Thinking tokens are tracked in the cumulative output count but ride the + // same unbilled slot today: totalCost prices usage.output only. + expect(faremeter.getTotalCost()).toBe(100 + 2 * 10 + (40 + 2 * 20 + 30)); + expect(faremeter.getOutputTokens()).toBe(35); + }); +}); diff --git a/src/provider/codex-responses-adapter.test.ts b/src/provider/codex-responses-adapter.test.ts index 35314c619..3850539ab 100644 --- a/src/provider/codex-responses-adapter.test.ts +++ b/src/provider/codex-responses-adapter.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import type { ConversationTurn, LastCycleSource } from "@intx/types/runtime"; +import type { ConversationTurn, LastCycleSource, TokenUsage } from "@intx/types/runtime"; import { PRODUCT_NAME } from "../branding.js"; import { createCodexResponsesAdapter, @@ -7,6 +7,7 @@ import { signatureForModel, tagSignature, } from "./codex-responses-adapter.js"; +import { contextTokensFromUsage } from "./context-window.js"; const source: LastCycleSource = { sourceId: "codex/test", @@ -89,48 +90,110 @@ describe("createCodexResponsesAdapter", () => { }); describe("createCodexResponsesAdapter usage parsing", () => { - test("maps a nonzero cache_creation_tokens count through to cacheWrite", () => { + // The Responses API reports `input_tokens` as the full prompt count with + // `cached_tokens` as a subset. Downstream consumers (context meter, + // compaction governor, faremeter) sum input + cacheRead + cacheWrite, so + // emitting the raw wire counts would double-count every cached token. + const completedUsage = ( + adapter: ReturnType, + sseData: string, + ): { usage: TokenUsage; source: LastCycleSource } => { + const event = adapter.parseResponse(sseData).find((e) => e.type === "inference.usage"); + if (event === undefined) throw new Error("stream carried no inference.usage event"); + return event.data as { usage: TokenUsage; source: LastCycleSource }; + }; + + test("subtracts cached_tokens from input_tokens so usage fields do not overlap", () => { const adapter = createCodexResponsesAdapter(source); const sseData = JSON.stringify({ type: "response.completed", response: { usage: { - input_tokens: 100, - input_tokens_details: { cached_tokens: 20, cache_creation_tokens: 15 }, + input_tokens: 1000, + input_tokens_details: { cached_tokens: 800 }, output_tokens: 50, output_tokens_details: { reasoning_tokens: 5 }, }, }, }); - const events = adapter.parseResponse(sseData); - const usageEvent = events.find((e) => e.type === "inference.usage"); + expect(completedUsage(adapter, sseData)).toEqual({ + usage: { input: 200, output: 50, cacheRead: 800, cacheWrite: 0, thinking: 5 }, + source, + }); + }); - expect(usageEvent?.data).toEqual({ - usage: { input: 100, output: 50, cacheRead: 20, cacheWrite: 15, thinking: 5 }, + test("keeps the context occupancy sum equal to the wire prompt token count", () => { + const adapter = createCodexResponsesAdapter(source); + const sseData = JSON.stringify({ + type: "response.completed", + response: { + usage: { + input_tokens: 1000, + input_tokens_details: { cached_tokens: 940 }, + output_tokens: 50, + }, + }, + }); + + const { usage } = completedUsage(adapter, sseData); + + expect(contextTokensFromUsage(usage)).toBe(1000); + }); + + test("reports input unchanged when the provider omits input_tokens_details", () => { + const adapter = createCodexResponsesAdapter(source); + const sseData = JSON.stringify({ + type: "response.completed", + response: { + usage: { + input_tokens: 100, + output_tokens: 50, + }, + }, + }); + + expect(completedUsage(adapter, sseData)).toEqual({ + usage: { input: 100, output: 50, cacheRead: 0, cacheWrite: 0, thinking: 0 }, + source, + }); + }); + + test("clamps input to zero when cached_tokens exceeds input_tokens", () => { + const adapter = createCodexResponsesAdapter(source); + const sseData = JSON.stringify({ + type: "response.completed", + response: { + usage: { + input_tokens: 10, + input_tokens_details: { cached_tokens: 25 }, + output_tokens: 50, + }, + }, + }); + + expect(completedUsage(adapter, sseData)).toEqual({ + usage: { input: 0, output: 50, cacheRead: 25, cacheWrite: 0, thinking: 0 }, source, }); }); - test("defaults cacheWrite to 0 when the provider does not report a cache-creation count", () => { + test("maps a nonzero cache_creation_tokens count through to cacheWrite", () => { const adapter = createCodexResponsesAdapter(source); const sseData = JSON.stringify({ type: "response.completed", response: { usage: { input_tokens: 100, - input_tokens_details: { cached_tokens: 20 }, + input_tokens_details: { cached_tokens: 20, cache_creation_tokens: 15 }, output_tokens: 50, output_tokens_details: { reasoning_tokens: 5 }, }, }, }); - const events = adapter.parseResponse(sseData); - const usageEvent = events.find((e) => e.type === "inference.usage"); - - expect(usageEvent?.data).toEqual({ - usage: { input: 100, output: 50, cacheRead: 20, cacheWrite: 0, thinking: 5 }, + expect(completedUsage(adapter, sseData)).toEqual({ + usage: { input: 80, output: 50, cacheRead: 20, cacheWrite: 15, thinking: 5 }, source, }); }); diff --git a/src/provider/codex-responses-adapter.ts b/src/provider/codex-responses-adapter.ts index 73339ef2a..01ee4a7bd 100644 --- a/src/provider/codex-responses-adapter.ts +++ b/src/provider/codex-responses-adapter.ts @@ -412,10 +412,18 @@ function usageFromResponse(response: Record): TokenUsage | unde const num = (v: unknown): number => (typeof v === "number" ? v : 0); const inputDetails = u["input_tokens_details"] as Record | undefined; const outputDetails = u["output_tokens_details"] as Record | undefined; + // Responses-API `input_tokens` counts the full prompt and `cached_tokens` + // is a subset of it. Downstream consumers (context meter, compaction + // governor, faremeter) treat the TokenUsage fields as non-overlapping and + // sum them, so the cached subset must be split out of input here — emitting + // the wire counts verbatim double-counts every cached token and inflates + // context occupancy up to ~2x on high cache-hit sessions. + const totalInputTokens = num(u["input_tokens"]); + const cachedTokens = num(inputDetails?.["cached_tokens"]); return { - input: num(u["input_tokens"]), + input: Math.max(0, totalInputTokens - cachedTokens), output: num(u["output_tokens"]), - cacheRead: num(inputDetails?.["cached_tokens"]), + cacheRead: cachedTokens, // OpenAI does not charge for writing to the prompt cache, so the public // Responses API usually omits a write count; read it defensively under // `cache_creation_tokens` in case a gateway/proxy in front of this diff --git a/tests/unit/codex-responses-adapter.test.ts b/tests/unit/codex-responses-adapter.test.ts index 2c642619d..20ee1921c 100644 --- a/tests/unit/codex-responses-adapter.test.ts +++ b/tests/unit/codex-responses-adapter.test.ts @@ -476,7 +476,7 @@ describe("codex-responses parseResponse", () => { ]); expect(out[0]).toMatchObject({ type: "inference.usage", - data: { usage: { input: 100, output: 20, cacheRead: 64, cacheWrite: 0, thinking: 8 } }, + data: { usage: { input: 36, output: 20, cacheRead: 64, cacheWrite: 0, thinking: 8 } }, }); }); diff --git a/tests/unit/codex-sse-fixtures.test.ts b/tests/unit/codex-sse-fixtures.test.ts index c6649a922..103288cbb 100644 --- a/tests/unit/codex-sse-fixtures.test.ts +++ b/tests/unit/codex-sse-fixtures.test.ts @@ -123,7 +123,7 @@ describe("codex-sse fixtures (golden parse)", () => { expect(out[9]).toMatchObject({ type: "inference.usage", data: { - usage: { input: 120, output: 40, cacheRead: 16, cacheWrite: 0, thinking: 12 }, + usage: { input: 104, output: 40, cacheRead: 16, cacheWrite: 0, thinking: 12 }, source: SOURCE, }, });