diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 7340d85..1750075 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -33,6 +33,7 @@ type ArchiveRepository = Pick< | "getModel" | "getTest" | "compareLatest" + | "getRunHistory" >; export interface AppDependencies { @@ -199,6 +200,42 @@ export function createApp(deps: AppDependencies) { return c.json({ test }); }); + app.get("/v1/archive/history", async (c) => { + if (!deps.archive) { + return c.json({ error: "archive_unavailable" }, 503); + } + + const modelId = c.req.query("modelId"); + const testCaseId = c.req.query("testCaseId"); + const rawLimit = c.req.query("limit"); + + if (!modelId || !UUID_RE.test(modelId)) { + return c.json({ error: "invalid_model_id" }, 400); + } + if (!testCaseId || !UUID_RE.test(testCaseId)) { + return c.json({ error: "invalid_test_case_id" }, 400); + } + + const limit = rawLimit === undefined ? undefined : Number(rawLimit); + if ( + limit !== undefined && + (!Number.isInteger(limit) || limit < 1 || limit > 100) + ) { + return c.json({ error: "invalid_limit" }, 400); + } + + const history = await deps.archive.getRunHistory({ + modelId, + testCaseId, + ...(limit !== undefined ? { limit } : {}), + }); + if (!history) { + return c.json({ error: "archive_history_not_found" }, 404); + } + + return c.json({ history }); + }); + app.get("/v1/archive/compare", async (c) => { if (!deps.archive) { return c.json({ error: "archive_unavailable" }, 503); diff --git a/apps/api/test/archive-api.test.ts b/apps/api/test/archive-api.test.ts index 1881384..c989f93 100644 --- a/apps/api/test/archive-api.test.ts +++ b/apps/api/test/archive-api.test.ts @@ -4,6 +4,7 @@ import { createApp } from "../src/app.js"; const RUN_ID = "00000000-0000-4000-8000-000000000021"; const MODEL_ID = "00000000-0000-4000-8000-000000000022"; const TEST_CASE_ID = "00000000-0000-4000-8000-000000000023"; +const PREVIOUS_RUN_ID = "00000000-0000-4000-8000-000000000037"; function baseRuns() { return { @@ -103,6 +104,62 @@ const archiveRun = { }, }, ], + relations: [ + { + direction: "outgoing", + relationType: "repeat_of", + relatedRunId: PREVIOUS_RUN_ID, + createdAt: "2026-09-24T08:00:01.200Z", + }, + ], +} as const; + +const archiveRunHistory = { + model: { + id: MODEL_ID, + provider: archiveRun.provider, + canonicalSlug: "deepseek-flash", + marketingName: "DeepSeek Flash", + status: "active", + runCount: 2, + latestRunAt: archiveRun.completedAt, + }, + test: { + testCaseId: TEST_CASE_ID, + familySlug: archiveRun.test.familySlug, + familyName: archiveRun.test.familyName, + variantSlug: archiveRun.test.variantSlug, + variantName: archiveRun.test.variantName, + category: "smoke", + artifactType: "text", + version: archiveRun.test.version, + caseSlug: archiveRun.test.caseSlug, + evaluator: { + slug: "exact-text", + version: "1.0.0", + kind: "deterministic", + }, + runCount: 2, + }, + runs: [ + { + ...archiveRun, + id: PREVIOUS_RUN_ID, + createdAt: "2026-09-23T08:00:00.000Z", + completedAt: "2026-09-23T08:00:01.000Z", + sealedAt: "2026-09-23T08:00:01.100Z", + relations: [], + }, + archiveRun, + ], + relations: [ + { + fromRunId: RUN_ID, + toRunId: PREVIOUS_RUN_ID, + relationType: "repeat_of", + createdAt: "2026-09-24T08:00:01.200Z", + }, + ], } as const; const archiveModelDetail = { @@ -306,6 +363,7 @@ describe("Archive read API", () => { getTest: async (testCaseId) => testCaseId === TEST_CASE_ID ? archiveTestDetail : null, compareLatest: async () => archiveComparison, + getRunHistory: async () => archiveRunHistory, }, }); @@ -353,6 +411,14 @@ describe("Archive read API", () => { comparison: archiveComparison, }); + const history = await app.request( + `/v1/archive/history?modelId=${MODEL_ID}&testCaseId=${TEST_CASE_ID}&limit=20`, + ); + expect(history.status).toBe(200); + await expect(history.json()).resolves.toEqual({ + history: archiveRunHistory, + }); + const runs = await app.request( `/v1/archive/runs?modelId=${MODEL_ID}&testCaseId=${TEST_CASE_ID}&limit=10`, ); @@ -376,6 +442,7 @@ describe("Archive read API", () => { getModel: async () => null, getTest: async () => null, compareLatest: async () => null, + getRunHistory: async () => null, }, }); @@ -398,6 +465,20 @@ describe("Archive read API", () => { ) ).status, ).toBe(400); + expect( + ( + await app.request( + `/v1/archive/history?modelId=nope&testCaseId=${TEST_CASE_ID}`, + ) + ).status, + ).toBe(400); + expect( + ( + await app.request( + `/v1/archive/history?modelId=${MODEL_ID}&testCaseId=${TEST_CASE_ID}&limit=101`, + ) + ).status, + ).toBe(400); }); }); diff --git a/apps/web/src/modelapse.ts b/apps/web/src/modelapse.ts index 438cb68..06bd81d 100644 --- a/apps/web/src/modelapse.ts +++ b/apps/web/src/modelapse.ts @@ -251,6 +251,20 @@ export interface ArchiveRunEvidence { } | null; } +export interface ArchiveRunRelationEdge { + readonly fromRunId: string; + readonly toRunId: string; + readonly relationType: string; + readonly createdAt: string; +} + +export interface ArchiveRunRelation { + readonly direction: "outgoing" | "incoming"; + readonly relationType: string; + readonly relatedRunId: string; + readonly createdAt: string; +} + export interface ArchiveRunDetail extends ArchiveRun { readonly configJson: string | null; readonly requestBlob: ArchiveBlob | null; @@ -259,6 +273,7 @@ export interface ArchiveRunDetail extends ArchiveRun { readonly usageJson: string | null; readonly timingJson: string | null; readonly evidence: readonly ArchiveRunEvidence[]; + readonly relations: readonly ArchiveRunRelation[]; } interface ArchiveRunDetailWire extends ArchiveRun { @@ -269,6 +284,23 @@ interface ArchiveRunDetailWire extends ArchiveRun { readonly usage: unknown; readonly timing: unknown; readonly evidence: readonly ArchiveRunEvidence[]; + readonly relations: readonly ArchiveRunRelation[]; +} + +export interface ArchiveRunHistory { + readonly model: ArchiveModel; + readonly test: ArchiveTest; + readonly runs: readonly ArchiveRun[]; + readonly relations: readonly ArchiveRunRelationEdge[]; +} + +export interface ArchiveTemporalComparison { + readonly test: ArchiveTest; + readonly rows: readonly { + readonly model: ArchiveModel; + readonly runs: readonly ArchiveRun[]; + readonly relations: readonly ArchiveRunRelationEdge[]; + }[]; } export interface ControlJob { @@ -326,6 +358,13 @@ interface CompareArchiveInput { readonly testCaseId: string; } +interface ReadArchiveHistoryInput { + readonly modelId: string; + readonly testCaseId: string; + readonly limit?: number; +} + + interface ApiOptions { readonly method?: "GET" | "POST"; readonly control?: boolean; @@ -536,6 +575,36 @@ function parseArchiveTestInput(value: unknown): ReadArchiveTestInput { return { testCaseId }; } +function parseArchiveHistoryInput(value: unknown): ReadArchiveHistoryInput { + if (!isRecord(value)) throw new Error("Archive history request must be an object"); + + const modelId = value.modelId; + const testCaseId = value.testCaseId; + const limit = value.limit; + + if (typeof modelId !== "string" || !UUID_RE.test(modelId)) { + throw new Error("modelId must be a UUID"); + } + if (typeof testCaseId !== "string" || !UUID_RE.test(testCaseId)) { + throw new Error("testCaseId must be a UUID"); + } + if ( + limit !== undefined && + (typeof limit !== "number" || + !Number.isInteger(limit) || + limit < 1 || + limit > 100) + ) { + throw new Error("limit must be an integer between 1 and 100"); + } + + return { + modelId, + testCaseId, + ...(limit === undefined ? {} : { limit }), + }; +} + function parseArchiveComparisonInput(value: unknown): CompareArchiveInput { if (!isRecord(value)) throw new Error("Archive comparison request must be an object"); @@ -669,6 +738,71 @@ export const getArchiveTest = createServerFn({ method: "POST" }) } }); +export const getArchiveRunHistory = createServerFn({ method: "POST" }) + .validator(parseArchiveHistoryInput) + .handler(async ({ data }): Promise => { + const params = new URLSearchParams({ + modelId: data.modelId, + testCaseId: data.testCaseId, + limit: String(data.limit ?? 50), + }); + try { + const result = await requestJson<{ history: ArchiveRunHistory }>( + `/v1/archive/history?${params.toString()}`, + ); + return result.history; + } catch (error) { + if (error instanceof ApiRequestError && error.status === 404) { + return null; + } + throw error; + } + }); + +export const compareArchiveHistory = createServerFn({ method: "POST" }) + .validator(parseArchiveComparisonInput) + .handler(async ({ data }): Promise => { + const histories = await Promise.all( + data.modelIds.map(async (modelId) => { + const params = new URLSearchParams({ + modelId, + testCaseId: data.testCaseId, + limit: "20", + }); + try { + const result = await requestJson<{ history: ArchiveRunHistory }>( + `/v1/archive/history?${params.toString()}`, + ); + return result.history; + } catch (error) { + if (error instanceof ApiRequestError && error.status === 404) { + return null; + } + throw error; + } + }), + ); + + if (histories.some((history) => history === null)) { + return null; + } + + const complete = histories.filter( + (history): history is ArchiveRunHistory => history !== null, + ); + const first = complete[0]; + if (!first) return null; + + return { + test: first.test, + rows: complete.map((history) => ({ + model: history.model, + runs: history.runs, + relations: history.relations, + })), + }; + }); + export const compareArchive = createServerFn({ method: "POST" }) .validator(parseArchiveComparisonInput) .handler(async ({ data }): Promise => { diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index c4f8310..b0f1e05 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -7,6 +7,7 @@ import { Route as rootRouteImport } from "./routes/__root"; import { Route as TestsTestCaseIdRouteImport } from "./routes/tests.$testCaseId"; import { Route as RunsRunIdRouteImport } from "./routes/runs.$runId"; import { Route as ModelsModelIdRouteImport } from "./routes/models.$modelId"; +import { Route as HistoryModelIdTestCaseIdRouteImport } from "./routes/history.$modelId.$testCaseId"; import { Route as CompareRouteImport } from "./routes/compare"; import { Route as IndexRouteImport } from "./routes/index"; @@ -28,6 +29,12 @@ const ModelsModelIdRoute = ModelsModelIdRouteImport.update({ getParentRoute: () => rootRouteImport, } as any); +const HistoryModelIdTestCaseIdRoute = HistoryModelIdTestCaseIdRouteImport.update({ + id: "/history/$modelId/$testCaseId", + path: "/history/$modelId/$testCaseId", + getParentRoute: () => rootRouteImport, +} as any); + const CompareRoute = CompareRouteImport.update({ id: "/compare", path: "/compare", @@ -43,6 +50,7 @@ const IndexRoute = IndexRouteImport.update({ export interface FileRoutesByFullPath { "/": typeof IndexRoute; "/compare": typeof CompareRoute; + "/history/$modelId/$testCaseId": typeof HistoryModelIdTestCaseIdRoute; "/models/$modelId": typeof ModelsModelIdRoute; "/runs/$runId": typeof RunsRunIdRoute; "/tests/$testCaseId": typeof TestsTestCaseIdRoute; @@ -51,6 +59,7 @@ export interface FileRoutesByFullPath { export interface FileRoutesByTo { "/": typeof IndexRoute; "/compare": typeof CompareRoute; + "/history/$modelId/$testCaseId": typeof HistoryModelIdTestCaseIdRoute; "/models/$modelId": typeof ModelsModelIdRoute; "/runs/$runId": typeof RunsRunIdRoute; "/tests/$testCaseId": typeof TestsTestCaseIdRoute; @@ -60,6 +69,7 @@ export interface FileRoutesById { __root__: typeof rootRouteImport; "/": typeof IndexRoute; "/compare": typeof CompareRoute; + "/history/$modelId/$testCaseId": typeof HistoryModelIdTestCaseIdRoute; "/models/$modelId": typeof ModelsModelIdRoute; "/runs/$runId": typeof RunsRunIdRoute; "/tests/$testCaseId": typeof TestsTestCaseIdRoute; @@ -70,6 +80,7 @@ export interface FileRouteTypes { fullPaths: | "/" | "/compare" + | "/history/$modelId/$testCaseId" | "/models/$modelId" | "/runs/$runId" | "/tests/$testCaseId"; @@ -77,6 +88,7 @@ export interface FileRouteTypes { to: | "/" | "/compare" + | "/history/$modelId/$testCaseId" | "/models/$modelId" | "/runs/$runId" | "/tests/$testCaseId"; @@ -84,6 +96,7 @@ export interface FileRouteTypes { | "__root__" | "/" | "/compare" + | "/history/$modelId/$testCaseId" | "/models/$modelId" | "/runs/$runId" | "/tests/$testCaseId"; @@ -106,6 +119,13 @@ declare module "@tanstack/react-router" { preLoaderRoute: typeof CompareRouteImport; parentRoute: typeof rootRouteImport; }; + "/history/$modelId/$testCaseId": { + id: "/history/$modelId/$testCaseId"; + path: "/history/$modelId/$testCaseId"; + fullPath: "/history/$modelId/$testCaseId"; + preLoaderRoute: typeof HistoryModelIdTestCaseIdRouteImport; + parentRoute: typeof rootRouteImport; + }; "/models/$modelId": { id: "/models/$modelId"; path: "/models/$modelId"; @@ -133,6 +153,7 @@ declare module "@tanstack/react-router" { const rootRouteChildren = { IndexRoute, CompareRoute, + HistoryModelIdTestCaseIdRoute, ModelsModelIdRoute, RunsRunIdRoute, TestsTestCaseIdRoute, diff --git a/apps/web/src/routes/compare.tsx b/apps/web/src/routes/compare.tsx index 73f3baa..7502820 100644 --- a/apps/web/src/routes/compare.tsx +++ b/apps/web/src/routes/compare.tsx @@ -2,9 +2,11 @@ import { createFileRoute } from "@tanstack/react-router"; import { useMemo, useState } from "react"; import { compareArchive, + compareArchiveHistory, getArchiveCatalog, type ArchiveComparison, type ArchiveRun, + type ArchiveTemporalComparison, } from "../modelapse"; export const Route = createFileRoute("/compare")({ @@ -37,6 +39,7 @@ function ArchiveComparePage() { const [modelIds, setModelIds] = useState(defaultModelIds); const [testCaseId, setTestCaseId] = useState(catalog.tests[0]?.testCaseId ?? ""); const [comparison, setComparison] = useState(null); + const [temporal, setTemporal] = useState(null); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); @@ -47,6 +50,7 @@ function ArchiveComparePage() { function toggleModel(modelId: string): void { setComparison(null); + setTemporal(null); setError(null); setModelIds((current) => { if (current.includes(modelId)) { @@ -63,18 +67,28 @@ function ArchiveComparePage() { setBusy(true); setError(null); try { - const result = await compareArchive({ - data: { - modelIds, - testCaseId, - }, - }); + const [result, history] = await Promise.all([ + compareArchive({ + data: { + modelIds, + testCaseId, + }, + }), + compareArchiveHistory({ + data: { + modelIds, + testCaseId, + }, + }), + ]); setComparison(result); - if (!result) { + setTemporal(history); + if (!result || !history) { setError("No public Archive comparison exists for this selection."); } } catch (caught) { setComparison(null); + setTemporal(null); setError(caught instanceof Error ? caught.message : "Comparison failed"); } finally { setBusy(false); @@ -106,11 +120,11 @@ function ArchiveComparePage() {
-

COMPARISON FOUNDATION

-

Same Test. Latest sealed Run.

+

TEMPORAL COMPARISON

+

Same Test. Latest view + history.

- Compare 2–4 canonical models against one public Test Case without - turning the Archive into a ranking table. + Compare 2–4 canonical models against one public Test Case, then inspect + each model’s sealed Run history without turning the Archive into a ranking table.

@@ -125,6 +139,7 @@ function ArchiveComparePage() { onChange={(event) => { setTestCaseId(event.target.value); setComparison(null); + setTemporal(null); setError(null); }} > @@ -135,7 +150,7 @@ function ArchiveComparePage() { ))} - Comparison uses the latest sealed public Run for this exact Test Case. + Latest comparison and temporal lanes use this exact public Test Case. @@ -177,7 +192,7 @@ function ArchiveComparePage() { disabled={busy || modelIds.length < 2 || !testCaseId} onClick={() => void handleCompare()} > - {busy ? "Comparing…" : "Compare latest sealed Runs"} + {busy ? "Comparing…" : "Compare latest + history"} @@ -280,6 +295,81 @@ function ArchiveComparePage() { )} +
+
+
+

SAME-TEST HISTORY

+

+ {temporal + ? `${temporal.test.caseSlug} across capture time` + : "Temporal lanes appear after comparison"} +

+
+

+ Every lane is ordered by capture time. Result changes are displayed as observations, + not as an inferred trend, winner, or cause. +

+
+ + {temporal ? ( +
+ {temporal.rows.map((row) => ( +
+
+
+ {row.model.provider.slug} + {row.model.marketingName} + {row.model.canonicalSlug} +
+ + Full history → + +
+ +
+ {row.runs.map((run, index) => ( + + + {String(index + 1).padStart(2, "0")} + + {formatTimestamp(run.completedAt ?? run.sealedAt)} + {evaluationLabel(run)} + + {run.evidenceLevel ?? "—"} · {run.returnedModel ?? run.requestedModel} + + + ))} + {row.runs.length === 0 ? ( +
+ No sealed public Run for this model × Test Case pair. +
+ ) : null} +
+ +
+ {row.runs.length} Run(s) + {row.relations.length} explicit relation(s) +
+
+ ))} +
+ ) : ( +
+

+ Select one Test and 2–4 models to inspect both the latest sealed record + and the sequence of earlier sealed Runs for the same exact Test. +

+
+ )} +
+