Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 130 additions & 0 deletions src/__tests__/scan-codex-window.unit.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'fs';
import os from 'os';
import path from 'path';
import { scanCodexHistory } from '../cli/commands/scan';
import { _resetPricingCache } from '../pricing/litellm';

// `node9 scan --days 30` reported EVERY Codex session ever recorded, because
// total_token_usage is cumulative and nothing windowed the cost. Measured on
// real data: 41 sessions / $13.69 from scan, 25 / $1.04 from report — same
// pricing function, 13x apart.

let home: string;

const DAY = 86_400_000;
const iso = (msAgo: number): string => new Date(Date.now() - msAgo).toISOString();

/** One rollout-*.jsonl in the layout scanCodexHistory walks. */
function writeSession(id: string, startedMsAgo: number, tokens: number): void {
const d = path.join(home, '.codex', 'sessions', '2026', '09', '01');
fs.mkdirSync(d, { recursive: true });
const lines = [
{ type: 'session_meta', payload: { timestamp: iso(startedMsAgo), id, cwd: '/p' } },
{ type: 'turn_context', payload: { model: 'gpt-5', cwd: '/p' } },
{
type: 'event_msg',
payload: {
type: 'token_count',
info: {
total_token_usage: {
input_tokens: tokens,
cached_input_tokens: 0,
output_tokens: tokens,
},
},
},
},
];
fs.writeFileSync(
path.join(d, `rollout-${id}.jsonl`),
lines.map((l) => JSON.stringify(l)).join('\n')
);
}

function costFor(startDate: Date | null): number {
return scanCodexHistory(startDate).totalCostUSD;
}

beforeEach(() => {
_resetPricingCache(); // deterministic bundled snapshot
home = fs.mkdtempSync(path.join(os.tmpdir(), 'n9-codexwin-'));
vi.spyOn(os, 'homedir').mockReturnValue(home);
});

afterEach(() => {
vi.restoreAllMocks();
fs.rmSync(home, { recursive: true, force: true });
});

describe('scanCodexHistory — cost window', () => {
it('excludes a session that started before the window', () => {
writeSession('old', 60 * DAY, 1_000_000);
const cost = costFor(new Date(Date.now() - 30 * DAY));
expect(cost).toBe(0);
});

it('includes a session that started inside the window', () => {
writeSession('recent', 5 * DAY, 1_000_000);
const cost = costFor(new Date(Date.now() - 30 * DAY));
expect(cost).toBeGreaterThan(0);
});

it('counts only the in-window session when both exist', () => {
writeSession('old', 60 * DAY, 1_000_000);
writeSession('recent', 5 * DAY, 1_000_000);
const windowed = costFor(new Date(Date.now() - 30 * DAY));

// Same window, with the old session removed: the totals must match, which
// proves the excluded session contributed exactly nothing rather than a
// little less.
fs.rmSync(path.join(home, '.codex'), { recursive: true, force: true });
writeSession('recent', 5 * DAY, 1_000_000);
const only = costFor(new Date(Date.now() - 30 * DAY));

expect(windowed).toBeCloseTo(only, 10);
expect(only).toBeGreaterThan(0); // the comparison must not be 0 === 0
});

it('counts EVERYTHING when there is no window — `--all` must not regress', () => {
// The obvious way to write this filter also silences the full-history
// scan, which is the one place the whole lifetime is the right answer.
writeSession('old', 60 * DAY, 1_000_000);
writeSession('recent', 5 * DAY, 1_000_000);
const all = costFor(null);
const windowed = costFor(new Date(Date.now() - 30 * DAY));
expect(all).toBeGreaterThan(windowed);
});

it('drops a session with no start timestamp, exactly as report does', () => {
// No session_meta, so startTime stays ''. Found by mutation testing: the
// first version of this test only asserted "did not crash", which every
// variant passed. It cannot be placed in time, and report-audit.ts:656
// drops it outright — the whole point here is that the two paths agree, so
// scan must drop it too rather than invent a placement.
const d = path.join(home, '.codex', 'sessions', '2026', '09', '01');
fs.mkdirSync(d, { recursive: true });
fs.writeFileSync(
path.join(d, 'rollout-nometa.jsonl'),
[
JSON.stringify({ type: 'turn_context', payload: { model: 'gpt-5', cwd: '/p' } }),
JSON.stringify({
type: 'event_msg',
payload: {
type: 'token_count',
info: {
total_token_usage: {
input_tokens: 1_000_000,
cached_input_tokens: 0,
output_tokens: 1_000_000,
},
},
},
}),
].join('\n')
);
expect(costFor(new Date(Date.now() - 30 * DAY))).toBe(0);
// ...but with no window at all it still counts, since nothing is excluded.
expect(costFor(null)).toBeGreaterThan(0);
});
});
42 changes: 35 additions & 7 deletions src/cli/commands/scan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2496,13 +2496,41 @@ export function scanCodexHistory(
}
}

// Accumulate session cost via the SHARED codexSessionCost (price +
// arithmetic in one place) — the same source report + upload use.
result.totalCostUSD += codexSessionCost(model, {
input: lastTotalInput,
cached: lastTotalCached,
output: lastTotalOutput,
});
// Window the cost at SESSION level, matching report-audit.ts:658.
//
// Codex reports total_token_usage cumulatively — last row wins — so the
// figure below is the session's whole lifetime, not the part inside the
// window. The per-row guard above cannot help: it compares `startTime`
// (the session's own start) and only runs for `function_call` rows, so it
// filters findings and loops while token_count rows sail past it.
//
// Result before this: `node9 scan --days 30` reported every Codex session
// ever recorded. Measured on the founder's machine — 41 sessions, $13.69 —
// against `node9 report --period 30d` at 25 sessions, $1.04. Same data,
// same shared pricing function, 13x apart, because one of them windowed
// and the other did not.
//
// Deliberately NOT the ccusage per-event delta approach here. That is the
// more correct model for a session straddling the boundary, but measured:
// ZERO of 41 sessions straddle. Building it now would solve a case this
// data does not contain while leaving the actual gap — a missing filter —
// open. Recorded in doc/roadmap/active/cost-accuracy-and-plans.md instead.
// A session with no session_meta timestamp is EXCLUDED once a window is
// set, matching report-audit.ts:656 (`if (!sessionStart) return`). We
// cannot place it in time, and the point of this change is that the two
// paths agree. `new Date('')` is Invalid Date and every comparison against
// it is false, so the explicit `!== ''` documents the intent rather than
// relying on NaN semantics to carry it.
const withinWindow = !startDate || (startTime !== '' && new Date(startTime) >= startDate);
if (withinWindow) {
// Accumulate session cost via the SHARED codexSessionCost (price +
// arithmetic in one place) — the same source report + upload use.
result.totalCostUSD += codexSessionCost(model, {
input: lastTotalInput,
cached: lastTotalCached,
output: lastTotalOutput,
});
}

result.loopFindings.push(...detectLoops(sessionCalls, projLabel, sessionId, 'codex'));
}
Expand Down
Loading