From a5cb219168a09fed9d722decc1bcef970af66e65 Mon Sep 17 00:00:00 2001 From: Node9 Date: Wed, 2 Sep 2026 22:01:16 +0300 Subject: [PATCH] fix(scan): Copilot cost was never zero, we just were not reading it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `scanCopilotHistory` hardcoded `totalCostUSD: 0` behind the comment "event logs carry no token/cost rollup". The comment is wrong. session.shutdown carries a modelMetrics rollup with inputTokens, outputTokens, cacheReadTokens and cacheWriteTokens, and costSync has been pricing it the whole time — $0.0426 of gpt-5-mini reaching the cloud while `node9 scan` displayed $0.00 for the same sessions. A $0 beside a real agent does not read as "no data". It reads as free, which is the opposite of the truth and exactly the conclusion a user would act on. No new arithmetic. parseCopilotSession already exists, is tested, and is what the upload path uses; scan simply never called it. Same shape as codexSessionCost: one function, several callers. Writing a second Copilot cost function would have recreated the divergence fixed one commit ago for Codex. Copilot's own `requests.cost` is 0 in real transcripts because GitHub bills a subscription, not tokens. The existing parser already prefers it only when > 0 and otherwise prices the tokens; a test now pins that, because trusting a vendor's billing field as a consumption figure is the same confusion the "API value" relabel exists to prevent. Windowed per row date. The rows carry their own `date`, so unlike Codex no session-level guard is needed. Measured, both binaries built and run back to back over --all: copilot $0.0000 -> $2.3526 <- the change gemini unchanged codex unchanged claude +$0.41 <- this session generating activity Mutation-tested three ways: restoring the hardcoded zero, removing the window filter, and trusting Copilot's cost field all turn the new tests red. Noticed while measuring: Gemini reports $11.43 over --all, so it computes correctly and its 30-day $0 is simply an empty window — not the same class of defect. Antigravity remains a genuine $0 with no token data anywhere; that one is a display problem, and it is next. --- src/__tests__/scan-copilot-cost.unit.test.ts | 100 +++++++++++++++++++ src/cli/commands/scan.ts | 20 ++++ 2 files changed, 120 insertions(+) create mode 100644 src/__tests__/scan-copilot-cost.unit.test.ts diff --git a/src/__tests__/scan-copilot-cost.unit.test.ts b/src/__tests__/scan-copilot-cost.unit.test.ts new file mode 100644 index 0000000..eb2cc1c --- /dev/null +++ b/src/__tests__/scan-copilot-cost.unit.test.ts @@ -0,0 +1,100 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { scanCopilotHistory } from '../cli/commands/scan'; +import { _resetPricingCache } from '../pricing/litellm'; + +// `scanCopilotHistory` hardcoded `totalCostUSD: 0` with the comment "event +// logs carry no token/cost rollup". That was wrong: session.shutdown carries a +// modelMetrics rollup, and costSync had been pricing it all along — the upload +// path reported $0.0426 of gpt-5-mini while `node9 scan` showed $0.00. +// +// A $0 beside a real agent does not read as "no data", it reads as "free". + +let home: string; +const DAY = 86_400_000; +const iso = (msAgo: number): string => new Date(Date.now() - msAgo).toISOString(); + +function writeSession(id: string, startedMsAgo: number, tokens: number): void { + const d = path.join(home, '.copilot', 'session-state', id); + fs.mkdirSync(d, { recursive: true }); + const lines = [ + { + type: 'session.start', + timestamp: iso(startedMsAgo), + data: { sessionId: id, startTime: iso(startedMsAgo), context: { cwd: '/p' } }, + }, + { + type: 'session.shutdown', + timestamp: iso(startedMsAgo), + data: { + modelMetrics: { + 'gpt-5-mini': { + usage: { + inputTokens: tokens, + outputTokens: tokens, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + // Copilot's own cost is 0 because GitHub bills a subscription, not + // tokens. Trusting it would report real consumption as free — the + // same confusion the "API value" relabel exists to prevent. + requests: { cost: 0 }, + }, + }, + }, + }, + ]; + fs.writeFileSync(path.join(d, 'events.jsonl'), lines.map((l) => JSON.stringify(l)).join('\n')); +} + +beforeEach(() => { + _resetPricingCache(); + home = fs.mkdtempSync(path.join(os.tmpdir(), 'n9-copilot-')); + vi.spyOn(os, 'homedir').mockReturnValue(home); +}); + +afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(home, { recursive: true, force: true }); +}); + +describe('scanCopilotHistory — cost', () => { + it('prices a session from its token rollup instead of reporting zero', () => { + writeSession('s1', 5 * DAY, 1_000_000); + expect(scanCopilotHistory(null).totalCostUSD).toBeGreaterThan(0); + }); + + it('does not take Copilot at its word that the session cost nothing', () => { + // requests.cost is 0 in the fixture, as it is in real transcripts. The + // tokens are what we measure. + writeSession('s1', 5 * DAY, 1_000_000); + const cost = scanCopilotHistory(null).totalCostUSD; + expect(cost).not.toBe(0); + }); + + it('excludes a session outside the window', () => { + writeSession('old', 60 * DAY, 1_000_000); + expect(scanCopilotHistory(new Date(Date.now() - 30 * DAY)).totalCostUSD).toBe(0); + }); + + it('counts everything when there is no window', () => { + writeSession('old', 60 * DAY, 1_000_000); + writeSession('recent', 5 * DAY, 1_000_000); + const all = scanCopilotHistory(null).totalCostUSD; + const windowed = scanCopilotHistory(new Date(Date.now() - 30 * DAY)).totalCostUSD; + expect(all).toBeGreaterThan(windowed); + expect(windowed).toBeGreaterThan(0); + }); + + it('stays at zero when a session never shut down, so there is no rollup', () => { + const d = path.join(home, '.copilot', 'session-state', 'live'); + fs.mkdirSync(d, { recursive: true }); + fs.writeFileSync( + path.join(d, 'events.jsonl'), + JSON.stringify({ type: 'session.start', data: { sessionId: 'live' } }) + ); + expect(scanCopilotHistory(null).totalCostUSD).toBe(0); + }); +}); diff --git a/src/cli/commands/scan.ts b/src/cli/commands/scan.ts index dc81752..2097bcf 100644 --- a/src/cli/commands/scan.ts +++ b/src/cli/commands/scan.ts @@ -38,6 +38,7 @@ import { scanArgs } from '../../dlp'; import { pricingFor } from '../../pricing/litellm'; import { geminiPriceFor } from '../../cost-gemini'; import { codexSessionCost } from '../../cost-codex'; +import { parseCopilotSession } from '../../cost-copilot'; import { canonicalToolInput } from '../../utils/hook-payload'; import type { SmartRule } from '../../core'; import { @@ -2053,6 +2054,25 @@ export function scanCopilotHistory( result.sessions++; + // Cost via the SHARED parseCopilotSession — the same function the upload + // path uses, so scan agrees with report and the cloud instead of inventing + // its own arithmetic. (The Codex mistake, one agent over.) + // + // This used to be a hardcoded `totalCostUSD: 0` with the comment "event + // logs carry no token/cost rollup". That was wrong: session.shutdown + // carries a modelMetrics rollup with inputTokens / outputTokens / + // cacheReadTokens / cacheWriteTokens, and costSync has been pricing it all + // along — $0.0426 of gpt-5-mini on this machine. `scan` was the only + // surface showing $0, and a $0 next to a real agent reads as "free", + // which is a conclusion the opposite of the truth. + // + // Windowed per row date, matching the Codex fix: rows carry their own + // `date`, so no session-level guard is needed here. + for (const row of parseCopilotSession(raw.split('\n'))) { + if (startDate && row.date && new Date(row.date) < startDate) continue; + result.totalCostUSD += row.costUSD; + } + for (const line of raw.split('\n')) { if (!line.trim()) continue; onLine?.();