diff --git a/apps/api/test/archive-api.test.ts b/apps/api/test/archive-api.test.ts index c989f93..544874e 100644 --- a/apps/api/test/archive-api.test.ts +++ b/apps/api/test/archive-api.test.ts @@ -5,6 +5,23 @@ const RUN_ID = "00000000-0000-4000-8000-000000000021"; const MODEL_ID = "00000000-0000-4000-8000-000000000022"; const TEST_CASE_ID = "00000000-0000-4000-8000-000000000023"; const PREVIOUS_RUN_ID = "00000000-0000-4000-8000-000000000037"; +const SOURCE_ID = "00000000-0000-4000-8000-000000000033"; +const BINDING_ID = "00000000-0000-4000-8000-000000000038"; +const ALIAS_ID = "00000000-0000-4000-8000-000000000039"; +const ALIAS_EVENT_ID = "00000000-0000-4000-8000-000000000040"; +const TEST_VERSION_ID = "00000000-0000-4000-8000-000000000041"; + +const archiveSource = { + id: SOURCE_ID, + sourceType: "provider_docs", + url: "https://api-docs.deepseek.com/guides/responses_api/", + title: "DeepSeek Responses API guide", + author: null, + publishedAt: null, + retrievedAt: "2026-09-01T00:00:00.000Z", + contentSha256: null, +} as const; + function baseRuns() { return { @@ -183,17 +200,65 @@ const archiveModelDetail = { }, releasedAt: "2026-09-01T00:00:00.000Z", retiredAt: null, - canonicalSourceId: "00000000-0000-4000-8000-000000000033", + canonicalSourceId: SOURCE_ID, + canonicalSource: archiveSource, snapshots: [ { id: "00000000-0000-4000-8000-000000000034", providerSnapshotId: "deepseek-flash-202609", validFrom: "2026-09-01T00:00:00.000Z", validTo: null, - sourceId: "00000000-0000-4000-8000-000000000033", + sourceId: SOURCE_ID, + source: archiveSource, }, ], relations: [], + aliasResolutions: [ + { + id: ALIAS_EVENT_ID, + alias: { + id: ALIAS_ID, + value: "deepseek-flash", + }, + observedAt: "2026-09-01T00:00:00.000Z", + sourceType: "provider_docs", + confidence: 1, + resolvedModelId: MODEL_ID, + resolvedSnapshot: null, + source: archiveSource, + }, + ], + executionBindings: [ + { + id: BINDING_ID, + apiModelId: "deepseek-flash", + validFrom: "2026-09-01T00:00:00.000Z", + validTo: null, + createdAt: "2026-09-01T00:00:00.000Z", + endpoint: { + id: archiveRun.provider.id, + path: "first_party_direct", + baseUrl: "https://api.deepseek.com", + hostname: "api.deepseek.com", + source: archiveSource, + }, + snapshot: null, + source: archiveSource, + }, + ], + identityTimeline: [ + { + id: `alias-resolution:${ALIAS_EVENT_ID}`, + kind: "alias_resolution", + occurredAt: "2026-09-01T00:00:00.000Z", + title: "Alias observed · deepseek-flash", + description: "resolved to canonical model deepseek-flash", + source: archiveSource, + aliasId: ALIAS_ID, + bindingId: null, + snapshotId: null, + }, + ], testCoverage: [ { testCaseId: TEST_CASE_ID, @@ -238,7 +303,29 @@ const archiveTestDetail = { }, runCount: 1, origin: "modelapse", - canonicalSourceId: null, + canonicalSourceId: SOURCE_ID, + canonicalSource: archiveSource, + versionSource: archiveSource, + versionHistory: [ + { + id: TEST_VERSION_ID, + version: "1.0.0", + status: "published", + definitionSha256: + "1111111111111111111111111111111111111111111111111111111111111111", + license: "Apache-2.0", + publishedAt: "2026-09-01T00:00:00.000Z", + createdAt: "2026-08-31T00:00:00.000Z", + source: archiveSource, + evaluator: { + slug: "exact-text", + version: "1.0.0", + kind: "deterministic", + }, + publicCaseCount: 1, + linkedTestCaseId: TEST_CASE_ID, + }, + ], versionStatus: "published", definitionSha256: "1111111111111111111111111111111111111111111111111111111111111111", @@ -389,6 +476,9 @@ describe("Archive read API", () => { await expect(model.json()).resolves.toMatchObject({ model: { id: MODEL_ID, + canonicalSource: { id: SOURCE_ID }, + aliasResolutions: [{ alias: { value: "deepseek-flash" } }], + executionBindings: [{ apiModelId: "deepseek-flash" }], timeline: [{ kind: "run", runId: RUN_ID }], }, }); @@ -399,6 +489,8 @@ describe("Archive read API", () => { test: { testCaseId: TEST_CASE_ID, definitionSha256: archiveTestDetail.definitionSha256, + canonicalSource: { id: SOURCE_ID }, + versionHistory: [{ version: "1.0.0", linkedTestCaseId: TEST_CASE_ID }], }, }); diff --git a/apps/web/src/modelapse.ts b/apps/web/src/modelapse.ts index 06bd81d..79cfc58 100644 --- a/apps/web/src/modelapse.ts +++ b/apps/web/src/modelapse.ts @@ -71,12 +71,97 @@ export interface ArchiveTest { readonly runCount: number; } +export interface ArchiveSource { + readonly id: string; + readonly sourceType: string; + readonly url: string | null; + readonly title: string | null; + readonly author: string | null; + readonly publishedAt: string | null; + readonly retrievedAt: string; + readonly contentSha256: string | null; +} + +export interface ArchiveModelAliasResolution { + readonly id: string; + readonly alias: { + readonly id: string; + readonly value: string; + }; + readonly observedAt: string; + readonly sourceType: string; + readonly confidence: number; + readonly resolvedModelId: string | null; + readonly resolvedSnapshot: { + readonly id: string; + readonly providerSnapshotId: string; + } | null; + readonly source: ArchiveSource | null; +} + +export interface ArchiveModelExecutionBinding { + readonly id: string; + readonly apiModelId: string; + readonly validFrom: string; + readonly validTo: string | null; + readonly createdAt: string; + readonly endpoint: { + readonly id: string; + readonly path: string; + readonly baseUrl: string; + readonly hostname: string; + readonly source: ArchiveSource | null; + }; + readonly snapshot: { + readonly id: string; + readonly providerSnapshotId: string; + } | null; + readonly source: ArchiveSource; +} + +export interface ArchiveIdentityTimelineEvent { + readonly id: string; + readonly kind: + | "canonical_source" + | "alias_resolution" + | "binding_started" + | "binding_ended" + | "snapshot_started" + | "snapshot_ended"; + readonly occurredAt: string; + readonly title: string; + readonly description: string; + readonly source: ArchiveSource | null; + readonly aliasId: string | null; + readonly bindingId: string | null; + readonly snapshotId: string | null; +} + +export interface ArchiveTestVersionHistory { + readonly id: string; + readonly version: string; + readonly status: string; + readonly definitionSha256: string; + readonly license: string | null; + readonly publishedAt: string | null; + readonly createdAt: string; + readonly source: ArchiveSource | null; + readonly evaluator: { + readonly slug: string; + readonly version: string; + readonly kind: string; + } | null; + readonly publicCaseCount: number; + readonly linkedTestCaseId: string | null; +} + export interface ArchiveModelSnapshot { readonly id: string; readonly providerSnapshotId: string; readonly validFrom: string | null; readonly validTo: string | null; readonly sourceId: string | null; + readonly source: ArchiveSource | null; } export interface ArchiveModelRelation { @@ -92,6 +177,7 @@ export interface ArchiveModelRelation { readonly validFrom: string | null; readonly validTo: string | null; readonly sourceId: string | null; + readonly source: ArchiveSource | null; readonly confidence: number; } @@ -128,8 +214,12 @@ export interface ArchiveModelDetail extends ArchiveModel { readonly releasedAt: string | null; readonly retiredAt: string | null; readonly canonicalSourceId: string | null; + readonly canonicalSource: ArchiveSource | null; readonly snapshots: readonly ArchiveModelSnapshot[]; readonly relations: readonly ArchiveModelRelation[]; + readonly aliasResolutions: readonly ArchiveModelAliasResolution[]; + readonly executionBindings: readonly ArchiveModelExecutionBinding[]; + readonly identityTimeline: readonly ArchiveIdentityTimelineEvent[]; readonly testCoverage: readonly { readonly testCaseId: string; readonly familySlug: string; @@ -146,6 +236,9 @@ export interface ArchiveModelDetail extends ArchiveModel { export interface ArchiveTestDetail extends ArchiveTest { readonly origin: string; readonly canonicalSourceId: string | null; + readonly canonicalSource: ArchiveSource | null; + readonly versionSource: ArchiveSource | null; + readonly versionHistory: readonly ArchiveTestVersionHistory[]; readonly versionStatus: string; readonly definitionSha256: string; readonly license: string | null; diff --git a/apps/web/src/routes/models.$modelId.tsx b/apps/web/src/routes/models.$modelId.tsx index 0e88857..9107ab4 100644 --- a/apps/web/src/routes/models.$modelId.tsx +++ b/apps/web/src/routes/models.$modelId.tsx @@ -20,6 +20,24 @@ function formatTimestamp(value: string | null): string { return new Date(value).toISOString().replace("T", " ").replace(".000Z", "Z"); } +function safeSourceHref(value: string | null): string | null { + if (!value) return null; + try { + const parsed = new URL(value); + return parsed.protocol === "https:" || parsed.protocol === "http:" + ? parsed.toString() + : null; + } catch { + return null; + } +} + +function shortHash(value: string | null): string { + if (!value) return "—"; + if (value.length <= 28) return value; + return `${value.slice(0, 14)}…${value.slice(-8)}`; +} + function evaluationLabel(run: ArchiveRun): string { if (!run.evaluation) return "not evaluated"; if (run.evaluation.exactMatch === true) return "exact match"; @@ -125,6 +143,168 @@ function ArchiveModelPage() { +
+
+
+

SOURCE & IDENTITY TIMELINE

+

What this model ID meant, when, and why

+
+

+ Alias observations, execution bindings and snapshots are shown only from stored + catalog facts. Source metadata excludes raw observation payloads. +

+
+ +
+
+ Canonical identity source + {model.canonicalSource?.title ?? model.canonicalSource?.sourceType ?? "—"} + + retrieved {formatTimestamp(model.canonicalSource?.retrievedAt ?? null)} + + {model.canonicalSource ? ( + <> + SHA-256 {shortHash(model.canonicalSource.contentSha256)} + {safeSourceHref(model.canonicalSource.url) ? ( + + Open source ↗ + + ) : null} + + ) : ( + No canonical source record is attached. + )} +
+ +
+ Observed aliases + {model.aliasResolutions.length} + + {model.aliasResolutions.length + ? `${new Set(model.aliasResolutions.map((item) => item.alias.value)).size} unique alias(es)` + : "No alias observation archived"} + +
+ +
+ Execution bindings + {model.executionBindings.length} + + {model.executionBindings.filter((binding) => !binding.validTo).length} current + +
+
+ +
+ {model.identityTimeline.map((event) => { + const sourceHref = safeSourceHref(event.source?.url ?? null); + return ( +
+
{formatTimestamp(event.occurredAt)}
+
+ {event.kind.replaceAll("_", " ")} + {event.title} + {event.description} +
+
+ {event.source?.sourceType ?? "no source"} + {event.source?.title ?? event.source?.url ?? "—"} + {sourceHref ? ( + + Source ↗ + + ) : null} +
+
+ ); + })} + {model.identityTimeline.length === 0 ? ( +
No dated identity observations are archived yet.
+ ) : null} +
+ +
+
+

Alias resolution history

+ {model.aliasResolutions.map((resolution) => ( +
+
+ {resolution.alias.value} + + {resolution.resolvedSnapshot + ? resolution.resolvedSnapshot.providerSnapshotId + : model.canonicalSlug} + +
+ + {formatTimestamp(resolution.observedAt)} · {resolution.sourceType} · confidence{" "} + {resolution.confidence} + + {safeSourceHref(resolution.source?.url ?? null) ? ( + + Evidence source ↗ + + ) : null} +
+ ))} + {model.aliasResolutions.length === 0 ? ( +

No alias-resolution observations archived.

+ ) : null} +
+ +
+

Execution binding history

+ {model.executionBindings.map((binding) => ( +
+
+ + {binding.validTo ? "historical" : "current"} + + {binding.apiModelId} +
+ + {binding.endpoint.path} · {binding.endpoint.hostname} + + + {formatTimestamp(binding.validFrom)} → {formatTimestamp(binding.validTo)} + + + snapshot {binding.snapshot?.providerSnapshotId ?? "—"} + + {safeSourceHref(binding.source.url) ? ( + + Binding source ↗ + + ) : null} +
+ ))} + {model.executionBindings.length === 0 ? ( +

No sourced execution binding archived.

+ ) : null} +
+
+
+
@@ -228,7 +408,19 @@ function ArchiveModelPage() { {formatTimestamp(snapshot.validFrom)} → {formatTimestamp(snapshot.validTo)} - source {snapshot.sourceId ?? "—"} + + source {snapshot.source?.title ?? snapshot.source?.sourceType ?? snapshot.sourceId ?? "—"} + + {safeSourceHref(snapshot.source?.url ?? null) ? ( + + Source ↗ + + ) : null}
))} {model.snapshots.length === 0 ?

No snapshots archived.

: null} @@ -245,6 +437,9 @@ function ArchiveModelPage() { {relation.direction} · confidence {relation.confidence} + + source {relation.source?.title ?? relation.source?.sourceType ?? relation.sourceId ?? "—"} +
))} {model.relations.length === 0 ?

No model relations archived.

: null} diff --git a/apps/web/src/routes/tests.$testCaseId.tsx b/apps/web/src/routes/tests.$testCaseId.tsx index d59ffe7..ce9ce28 100644 --- a/apps/web/src/routes/tests.$testCaseId.tsx +++ b/apps/web/src/routes/tests.$testCaseId.tsx @@ -22,6 +22,18 @@ function shortHash(value: string | null): string { return `${value.slice(0, 16)}…${value.slice(-10)}`; } +function safeSourceHref(value: string | null): string | null { + if (!value) return null; + try { + const parsed = new URL(value); + return parsed.protocol === "https:" || parsed.protocol === "http:" + ? parsed.toString() + : null; + } catch { + return null; + } +} + function evaluationLabel(run: ArchiveRun): string { if (!run.evaluation) return "not evaluated"; if (run.evaluation.exactMatch === true) return "exact match"; @@ -176,6 +188,134 @@ function ArchiveTestPage() {
+
+
+
+

SOURCE PROVENANCE

+

Definition lineage

+
+

+ Public source summaries expose citation metadata only; source-record metadata + and private blob locations remain outside the Archive contract. +

+
+ +
+
+ Test family source + {test.canonicalSource?.title ?? test.canonicalSource?.sourceType ?? "—"} + + retrieved {formatTimestamp(test.canonicalSource?.retrievedAt ?? null)} + + SHA-256 {shortHash(test.canonicalSource?.contentSha256 ?? null)} + {safeSourceHref(test.canonicalSource?.url ?? null) ? ( + + Open family source ↗ + + ) : null} +
+ +
+ Current version source + {test.versionSource?.title ?? test.versionSource?.sourceType ?? "—"} + v{test.version} · {test.versionStatus} + {safeSourceHref(test.versionSource?.url ?? null) ? ( + + Open version source ↗ + + ) : null} +
+ +
+ Public version records + {test.versionHistory.length} + {test.variantName} · {test.variantSlug} +
+
+
+ +
+
+
+

TEST VERSION HISTORY

+

Immutable definitions over time

+
+

+ Only versions with at least one public Test Case appear here. Matching case slugs + link across versions when that public case exists. +

+
+ +
+ {test.versionHistory.map((version) => { + const sourceHref = safeSourceHref(version.source?.url ?? null); + const current = version.version === test.version; + return ( +
+
+ + {current ? "current" : version.status} + + v{version.version} + {formatTimestamp(version.publishedAt ?? version.createdAt)} +
+
+ Definition SHA-256 + {shortHash(version.definitionSha256)} + {version.publicCaseCount} public case(s) · {version.license ?? "no license"} +
+
+ Evaluator + + {version.evaluator + ? `${version.evaluator.slug}@${version.evaluator.version}` + : "—"} + + {version.evaluator?.kind ?? "no evaluator binding"} +
+
+ Source + {version.source?.title ?? version.source?.sourceType ?? "—"} +
+ {version.linkedTestCaseId && version.linkedTestCaseId !== test.testCaseId ? ( + + Same case in v{version.version} → + + ) : null} + {sourceHref ? ( + + Source ↗ + + ) : null} +
+
+
+ ); + })} + {test.versionHistory.length === 0 ? ( +
No public Test version history is archived yet.
+ ) : null} +
+
+
diff --git a/apps/web/src/styles.css b/apps/web/src/styles.css index 8cefb87..1d690f9 100644 --- a/apps/web/src/styles.css +++ b/apps/web/src/styles.css @@ -1590,6 +1590,220 @@ button:disabled { font: 10px/1.4 ui-monospace, SFMono-Regular, Menlo, monospace; } + +.identity-section, +.test-provenance-section { + background: + linear-gradient(180deg, rgba(200, 255, 99, 0.018), transparent 260px), + rgba(255, 255, 255, 0.002); +} + +.provenance-summary { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + border-top: 1px solid var(--line); + border-left: 1px solid var(--line); +} + +.provenance-card { + min-width: 0; + min-height: 170px; + padding: 20px; + border-right: 1px solid var(--line); + border-bottom: 1px solid var(--line); + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 9px; + background: rgba(255, 255, 255, 0.006); +} + +.provenance-card-primary { + background: rgba(200, 255, 99, 0.025); +} + +.provenance-card > span, +.version-history-row span:not(.badge), +.identity-event-source > span { + color: var(--muted); + font: 700 10px/1.3 ui-monospace, SFMono-Regular, Menlo, monospace; + text-transform: uppercase; + letter-spacing: 0.07em; +} + +.provenance-card > strong { + margin-top: 6px; + font-size: 15px; + overflow-wrap: anywhere; +} + +.provenance-card > small { + color: var(--muted); + font: 10px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; + overflow-wrap: anywhere; +} + +.provenance-card .text-link { + margin-top: auto; +} + +.identity-timeline { + margin-top: 28px; + border-top: 1px solid var(--line); + border-left: 1px solid var(--line); +} + +.identity-event { + display: grid; + grid-template-columns: minmax(150px, 0.7fr) minmax(0, 1.25fr) minmax(210px, 1fr); + min-height: 112px; +} + +.identity-event > div { + min-width: 0; + padding: 18px; + border-right: 1px solid var(--line); + border-bottom: 1px solid var(--line); +} + +.identity-event-time { + color: var(--muted); + display: flex; + align-items: center; + font: 10px/1.55 ui-monospace, SFMono-Regular, Menlo, monospace; +} + +.identity-event-body, +.identity-event-source { + display: flex; + flex-direction: column; + align-items: flex-start; + justify-content: center; + gap: 7px; +} + +.identity-event-body strong, +.identity-event-source strong { + font-size: 13px; + overflow-wrap: anywhere; +} + +.identity-event-body small { + color: var(--muted); + font: 10px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; + overflow-wrap: anywhere; +} + +.identity-ledger-grid { + margin-top: 28px; + display: grid; + grid-template-columns: 1fr 1fr; + border-top: 1px solid var(--line); + border-left: 1px solid var(--line); +} + +.identity-ledger { + min-width: 0; + padding: 20px; + border-right: 1px solid var(--line); + border-bottom: 1px solid var(--line); +} + +.identity-ledger h3 { + margin: 0 0 18px; + font-size: 16px; +} + +.identity-ledger > p { + color: var(--muted); + font-size: 12px; +} + +.identity-ledger-row { + padding: 14px 0; + border-top: 1px solid var(--line); + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 7px; +} + +.identity-ledger-row > div { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 9px; +} + +.identity-ledger-row strong { + font-size: 12px; + overflow-wrap: anywhere; +} + +.identity-ledger-row small { + color: var(--muted); + font: 10px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; + overflow-wrap: anywhere; +} + +.version-history { + border-top: 1px solid var(--line); + border-left: 1px solid var(--line); +} + +.version-history-row { + display: grid; + grid-template-columns: + minmax(130px, 0.65fr) + minmax(200px, 1.25fr) + minmax(170px, 0.9fr) + minmax(220px, 1.2fr); +} + +.version-history-row > div { + min-width: 0; + min-height: 118px; + padding: 17px; + border-right: 1px solid var(--line); + border-bottom: 1px solid var(--line); + display: flex; + flex-direction: column; + justify-content: center; + align-items: flex-start; + gap: 7px; +} + +.version-history-current { + background: rgba(200, 255, 99, 0.018); +} + +.version-history-id strong { + font-size: 18px; +} + +.version-history-id small, +.version-history-definition small { + color: var(--muted); + font: 10px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace; +} + +.version-history-definition strong, +.version-history-source strong { + font: 12px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace; + overflow-wrap: anywhere; +} + +.version-history-actions { + margin-top: auto; + display: flex; + flex-wrap: wrap; + gap: 10px; +} + +.version-history-actions .text-link { + font-size: 10px; +} + footer { height: 110px; padding: 0 clamp(20px, 4vw, 64px); @@ -1653,13 +1867,27 @@ footer { .detail-grid, .lineage-grid, - .compare-controls { + .compare-controls, + .identity-ledger-grid { grid-template-columns: 1fr; } .entity-stats, .coverage-grid, - .comparison-grid { + .comparison-grid, + .provenance-summary { + grid-template-columns: 1fr 1fr; + } + + .identity-event { + grid-template-columns: minmax(135px, 0.65fr) minmax(0, 1.35fr); + } + + .identity-event-source { + grid-column: 1 / -1; + } + + .version-history-row { grid-template-columns: 1fr 1fr; } @@ -1716,10 +1944,17 @@ footer { .definition-grid, .comparison-grid, .model-picker-grid, - .comparison-facts { + .comparison-facts, + .provenance-summary, + .version-history-row, + .identity-event { grid-template-columns: 1fr; } + .identity-event-source { + grid-column: auto; + } + .timeline-event { grid-template-columns: 20px 1fr; } diff --git a/docs/web-operator-ui.md b/docs/web-operator-ui.md index 6acefe1..39fdf22 100644 --- a/docs/web-operator-ui.md +++ b/docs/web-operator-ui.md @@ -255,6 +255,88 @@ Migration `0006_archive_temporal_indexes.sql` adds: The migration changes indexing only; it does not alter historical records or introduce mutable summary tables. +## Archive v0.5: source and identity timeline + +Modelapse now treats catalog identity itself as a time-varying, sourced Archive record. + +No new privileged browser surface is introduced. The existing public Model and Test detail APIs return additional Archive-safe provenance projections. + +### Model identity timeline + +`GET /v1/archive/models/:modelId` now includes: + +- the canonical Model source record; +- provider snapshot sources; +- Model relation sources; +- alias-resolution observations that resolved to the Model or one of its snapshots; +- execution-binding history for provider endpoint + API model ID + optional snapshot; +- a dedicated identity timeline composed from canonical-source retrieval, alias observations, binding validity and snapshot validity. + +A public source summary contains only: + +```text +id +sourceType +url +title +author +publishedAt +retrievedAt +contentSha256 +``` + +The public projection deliberately excludes `source_records.metadata` and `alias_resolution_events.raw_observation`. + +This lets the Archive answer questions such as: + +```text +At time T, what did provider alias A resolve to? +Which provider endpoint/API model ID was bound to canonical Model M? +Which source record supports that identity claim? +Was the binding tied to a provider snapshot? +``` + +without exposing internal observation payloads. + +### Test version history + +`GET /v1/archive/tests/:testCaseId` now includes: + +- Test Family canonical source; +- current Test Version source; +- version history for the same Test Variant; +- immutable definition hash, publication/creation time, license and evaluator identity per version; +- the number of public Test Cases in each version; +- a linkable public Test Case ID when the current case slug also exists in another version. + +Only Test Versions that contain at least one public Test Case are included in public history. Private-only or case-less draft versions are not surfaced through this projection. + +### Version-level provenance + +Migration `0007_identity_provenance.sql` adds a nullable `test_versions.source_id` foreign key to `source_records`. + +Existing Test Versions are backfilled from their Test Family canonical source when one is available. Catalog bootstrap now writes version provenance directly for new Test definitions. + +The same migration adds indexes for: + +- alias observations resolved by canonical Model; +- alias observations resolved by provider snapshot; +- Test Variant version-history reads. + +### Identity vs execution evidence + +v0.5 keeps two provenance questions separate: + +```text +Catalog identity provenance + = why Modelapse says this alias/binding/snapshot identifies this Model + +Run evidence + = what happened during one actual model execution +``` + +A sourced alias or binding does not upgrade Run Evidence level, and a sealed Run does not by itself rewrite canonical catalog identity. + ## Deliberately deferred - general user login/session management; diff --git a/packages/catalog-admin/src/catalog.ts b/packages/catalog-admin/src/catalog.ts index b9a5c20..3751bb3 100644 --- a/packages/catalog-admin/src/catalog.ts +++ b/packages/catalog-admin/src/catalog.ts @@ -343,6 +343,7 @@ async function ensureVersionAndCase( client: PoolClient, variantId: string, prompt: BlobDescriptor, + definitionSourceId: string, ): Promise<{ readonly versionId: string; readonly caseId: string }> { const definitionSha256 = rawDefinitionSha256(); @@ -351,8 +352,9 @@ async function ensureVersionAndCase( id: string; status: "draft" | "published" | "retired"; definition_sha256: string; + source_id: string | null; }>( - `SELECT id, status, definition_sha256 + `SELECT id, status, definition_sha256, source_id FROM modelapse.test_versions WHERE variant_id = $1 AND version = $2`, @@ -365,12 +367,13 @@ async function ensureVersionAndCase( id: string; status: "draft"; definition_sha256: string; + source_id: string; }>( `INSERT INTO modelapse.test_versions - (variant_id, version, status, definition_sha256, license) - VALUES ($1, $2, 'draft', $3, NULL) - RETURNING id, status, definition_sha256`, - [variantId, OPENAI_SMOKE_VERSION, definitionSha256], + (variant_id, version, status, definition_sha256, license, source_id) + VALUES ($1, $2, 'draft', $3, NULL, $4) + RETURNING id, status, definition_sha256, source_id`, + [variantId, OPENAI_SMOKE_VERSION, definitionSha256, definitionSourceId], ); version = inserted.rows[0]; } @@ -384,6 +387,17 @@ async function ensureVersionAndCase( if (version.status === "retired") { throw new Error("Smoke Test Version is retired and cannot be bootstrapped"); } + if (!version.source_id) { + await client.query( + `UPDATE modelapse.test_versions + SET source_id = $2 + WHERE id = $1 + AND source_id IS NULL`, + [version.id, definitionSourceId], + ); + } else if (version.source_id !== definitionSourceId) { + throw new Error("Smoke Test Version provenance conflicts with existing catalog data"); + } let testCase = ( await client.query<{ @@ -657,6 +671,7 @@ async function ensureDirectSmokeVersionAndCase( client: PoolClient, variantId: string, prompt: BlobDescriptor, + definitionSourceId: string, ): Promise<{ readonly versionId: string; readonly caseId: string }> { const definitionSha256 = directSmokeDefinitionSha256(); @@ -665,8 +680,9 @@ async function ensureDirectSmokeVersionAndCase( id: string; status: "draft" | "published" | "retired"; definition_sha256: string; + source_id: string | null; }>( - `SELECT id, status, definition_sha256 + `SELECT id, status, definition_sha256, source_id FROM modelapse.test_versions WHERE variant_id = $1 AND version = $2`, @@ -679,12 +695,13 @@ async function ensureDirectSmokeVersionAndCase( id: string; status: "draft"; definition_sha256: string; + source_id: string; }>( `INSERT INTO modelapse.test_versions - (variant_id, version, status, definition_sha256, license) - VALUES ($1, $2, 'draft', $3, NULL) - RETURNING id, status, definition_sha256`, - [variantId, DIRECT_SMOKE_VERSION, definitionSha256], + (variant_id, version, status, definition_sha256, license, source_id) + VALUES ($1, $2, 'draft', $3, NULL, $4) + RETURNING id, status, definition_sha256, source_id`, + [variantId, DIRECT_SMOKE_VERSION, definitionSha256, definitionSourceId], ); version = inserted.rows[0]; } @@ -698,6 +715,19 @@ async function ensureDirectSmokeVersionAndCase( if (version.status === "retired") { throw new Error("Direct Smoke Test Version is retired"); } + if (!version.source_id) { + await client.query( + `UPDATE modelapse.test_versions + SET source_id = $2 + WHERE id = $1 + AND source_id IS NULL`, + [version.id, definitionSourceId], + ); + } else if (version.source_id !== definitionSourceId) { + throw new Error( + "Direct Smoke Test Version provenance conflicts with existing catalog data", + ); + } let testCase = ( await client.query<{ @@ -849,6 +879,7 @@ export class PgCatalogAdmin { client, testVariantId, prompt, + definitionSourceId, ); await client.query("COMMIT"); @@ -931,6 +962,7 @@ export class PgCatalogAdmin { client, testVariantId, prompt, + definitionSourceId, ); await client.query("COMMIT"); diff --git a/packages/catalog-admin/test/bootstrap.integration.test.ts b/packages/catalog-admin/test/bootstrap.integration.test.ts index 68450ad..ec9466d 100644 --- a/packages/catalog-admin/test/bootstrap.integration.test.ts +++ b/packages/catalog-admin/test/bootstrap.integration.test.ts @@ -160,11 +160,13 @@ describe("production catalog bootstrap", () => { version_status: string; endpoint_source_id: string | null; family_source_id: string | null; + version_source_id: string | null; }>( `SELECT tv.status AS version_status, pe.source_id AS endpoint_source_id, - tf.canonical_source_id AS family_source_id + tf.canonical_source_id AS family_source_id, + tv.source_id AS version_source_id FROM modelapse.test_cases tc JOIN modelapse.test_versions tv ON tv.id = tc.test_version_id JOIN modelapse.test_variants tvar ON tvar.id = tv.variant_id @@ -182,6 +184,7 @@ describe("production catalog bootstrap", () => { version_status: "published", endpoint_source_id: first.providerSourceId, family_source_id: first.definitionSourceId, + version_source_id: first.definitionSourceId, }); } finally { await verification.end(); diff --git a/packages/database/migrations/0007_identity_provenance.sql b/packages/database/migrations/0007_identity_provenance.sql new file mode 100644 index 0000000..eb47fbc --- /dev/null +++ b/packages/database/migrations/0007_identity_provenance.sql @@ -0,0 +1,27 @@ +BEGIN; + +SET search_path TO modelapse, public; + +ALTER TABLE test_versions + ADD COLUMN source_id uuid REFERENCES source_records(id); + +UPDATE test_versions tv + SET source_id = tf.canonical_source_id + FROM test_variants tvar + JOIN test_families tf ON tf.id = tvar.family_id + WHERE tv.variant_id = tvar.id + AND tv.source_id IS NULL + AND tf.canonical_source_id IS NOT NULL; + +CREATE INDEX alias_resolution_model_observed_idx + ON alias_resolution_events (resolved_model_id, observed_at DESC) + WHERE resolved_model_id IS NOT NULL; + +CREATE INDEX alias_resolution_snapshot_observed_idx + ON alias_resolution_events (resolved_snapshot_id, observed_at DESC) + WHERE resolved_snapshot_id IS NOT NULL; + +CREATE INDEX test_versions_variant_published_idx + ON test_versions (variant_id, published_at DESC NULLS LAST, created_at DESC); + +COMMIT; diff --git a/packages/persistence/src/archive-repository.ts b/packages/persistence/src/archive-repository.ts index 2274871..1d36988 100644 --- a/packages/persistence/src/archive-repository.ts +++ b/packages/persistence/src/archive-repository.ts @@ -126,12 +126,97 @@ export interface ArchiveRunDetailView extends ArchiveRunView { readonly relations: readonly ArchiveRunRelationView[]; } +export interface ArchiveSourceView { + readonly id: string; + readonly sourceType: string; + readonly url: string | null; + readonly title: string | null; + readonly author: string | null; + readonly publishedAt: string | null; + readonly retrievedAt: string; + readonly contentSha256: string | null; +} + +export interface ArchiveModelAliasResolutionView { + readonly id: string; + readonly alias: { + readonly id: string; + readonly value: string; + }; + readonly observedAt: string; + readonly sourceType: string; + readonly confidence: number; + readonly resolvedModelId: string | null; + readonly resolvedSnapshot: { + readonly id: string; + readonly providerSnapshotId: string; + } | null; + readonly source: ArchiveSourceView | null; +} + +export interface ArchiveModelExecutionBindingView { + readonly id: string; + readonly apiModelId: string; + readonly validFrom: string; + readonly validTo: string | null; + readonly createdAt: string; + readonly endpoint: { + readonly id: string; + readonly path: string; + readonly baseUrl: string; + readonly hostname: string; + readonly source: ArchiveSourceView | null; + }; + readonly snapshot: { + readonly id: string; + readonly providerSnapshotId: string; + } | null; + readonly source: ArchiveSourceView; +} + +export interface ArchiveIdentityTimelineEventView { + readonly id: string; + readonly kind: + | "canonical_source" + | "alias_resolution" + | "binding_started" + | "binding_ended" + | "snapshot_started" + | "snapshot_ended"; + readonly occurredAt: string; + readonly title: string; + readonly description: string; + readonly source: ArchiveSourceView | null; + readonly aliasId: string | null; + readonly bindingId: string | null; + readonly snapshotId: string | null; +} + +export interface ArchiveTestVersionHistoryView { + readonly id: string; + readonly version: string; + readonly status: string; + readonly definitionSha256: string; + readonly license: string | null; + readonly publishedAt: string | null; + readonly createdAt: string; + readonly source: ArchiveSourceView | null; + readonly evaluator: { + readonly slug: string; + readonly version: string; + readonly kind: string; + } | null; + readonly publicCaseCount: number; + readonly linkedTestCaseId: string | null; +} + export interface ArchiveModelSnapshotView { readonly id: string; readonly providerSnapshotId: string; readonly validFrom: string | null; readonly validTo: string | null; readonly sourceId: string | null; + readonly source: ArchiveSourceView | null; } export interface ArchiveModelRelationView { @@ -147,6 +232,7 @@ export interface ArchiveModelRelationView { readonly validFrom: string | null; readonly validTo: string | null; readonly sourceId: string | null; + readonly source: ArchiveSourceView | null; readonly confidence: number; } @@ -202,8 +288,12 @@ export interface ArchiveModelDetailView extends ArchiveModelView { readonly releasedAt: string | null; readonly retiredAt: string | null; readonly canonicalSourceId: string | null; + readonly canonicalSource: ArchiveSourceView | null; readonly snapshots: readonly ArchiveModelSnapshotView[]; readonly relations: readonly ArchiveModelRelationView[]; + readonly aliasResolutions: readonly ArchiveModelAliasResolutionView[]; + readonly executionBindings: readonly ArchiveModelExecutionBindingView[]; + readonly identityTimeline: readonly ArchiveIdentityTimelineEventView[]; readonly testCoverage: readonly ArchiveTestCoverageView[]; readonly recentRuns: readonly ArchiveRunView[]; readonly timeline: readonly ArchiveTimelineEventView[]; @@ -212,6 +302,9 @@ export interface ArchiveModelDetailView extends ArchiveModelView { export interface ArchiveTestDetailView extends ArchiveTestView { readonly origin: string; readonly canonicalSourceId: string | null; + readonly canonicalSource: ArchiveSourceView | null; + readonly versionSource: ArchiveSourceView | null; + readonly versionHistory: readonly ArchiveTestVersionHistoryView[]; readonly versionStatus: string; readonly definitionSha256: string; readonly license: string | null; @@ -244,6 +337,31 @@ export interface ArchiveRunHistoryView { } +function archiveSourceView(row: { + source_id: string | null; + source_type: string | null; + source_url: string | null; + source_title: string | null; + source_author: string | null; + source_published_at: Date | null; + source_retrieved_at: Date | null; + source_content_sha256: string | null; +}): ArchiveSourceView | null { + if (!row.source_id || !row.source_type || !row.source_retrieved_at) { + return null; + } + return { + id: row.source_id, + sourceType: row.source_type, + url: row.source_url, + title: row.source_title, + author: row.source_author, + publishedAt: row.source_published_at?.toISOString() ?? null, + retrievedAt: row.source_retrieved_at.toISOString(), + contentSha256: row.source_content_sha256, + }; +} + interface ArchiveRunRow { id: string; status: string; @@ -744,6 +862,14 @@ export class PgArchiveRepository { released_at: Date | null; retired_at: Date | null; canonical_source_id: string | null; + source_id: string | null; + source_type: string | null; + source_url: string | null; + source_title: string | null; + source_author: string | null; + source_published_at: Date | null; + source_retrieved_at: Date | null; + source_content_sha256: string | null; family_id: string | null; family_slug: string | null; family_name: string | null; @@ -765,6 +891,14 @@ export class PgArchiveRepository { m.released_at, m.retired_at, m.canonical_source_id, + canonical_source.id AS source_id, + canonical_source.source_type, + canonical_source.url AS source_url, + canonical_source.title AS source_title, + canonical_source.author AS source_author, + canonical_source.published_at AS source_published_at, + canonical_source.retrieved_at AS source_retrieved_at, + canonical_source.content_sha256 AS source_content_sha256, mf.id AS family_id, mf.slug AS family_slug, mf.display_name AS family_name, @@ -790,6 +924,8 @@ export class PgArchiveRepository { ) AS latest_run_at FROM modelapse.models m JOIN modelapse.providers p ON p.id = m.provider_id + LEFT JOIN modelapse.source_records canonical_source + ON canonical_source.id = m.canonical_source_id LEFT JOIN modelapse.model_families mf ON mf.id = m.family_id LEFT JOIN modelapse.model_tracks mt ON mt.id = m.track_id WHERE m.id = $1 @@ -800,19 +936,45 @@ export class PgArchiveRepository { const row = modelResult.rows[0]; if (!row) return null; - const [snapshotResult, relationResult, coverageResult, allRuns] = - await Promise.all([ + const [ + snapshotResult, + relationResult, + aliasResult, + bindingResult, + coverageResult, + allRuns, + ] = await Promise.all([ this.pool.query<{ id: string; provider_snapshot_id: string; valid_from: Date | null; valid_to: Date | null; source_id: string | null; + source_type: string | null; + source_url: string | null; + source_title: string | null; + source_author: string | null; + source_published_at: Date | null; + source_retrieved_at: Date | null; + source_content_sha256: string | null; }>( - `SELECT id, provider_snapshot_id, valid_from, valid_to, source_id - FROM modelapse.model_snapshots - WHERE model_id = $1 - ORDER BY valid_from DESC NULLS LAST, provider_snapshot_id`, + `SELECT + ms.id, + ms.provider_snapshot_id, + ms.valid_from, + ms.valid_to, + ms.source_id, + source.source_type, + source.url AS source_url, + source.title AS source_title, + source.author AS source_author, + source.published_at AS source_published_at, + source.retrieved_at AS source_retrieved_at, + source.content_sha256 AS source_content_sha256 + FROM modelapse.model_snapshots ms + LEFT JOIN modelapse.source_records source ON source.id = ms.source_id + WHERE ms.model_id = $1 + ORDER BY ms.valid_from DESC NULLS LAST, ms.provider_snapshot_id`, [modelId], ), this.pool.query<{ @@ -826,6 +988,13 @@ export class PgArchiveRepository { valid_from: Date | null; valid_to: Date | null; source_id: string | null; + source_type: string | null; + source_url: string | null; + source_title: string | null; + source_author: string | null; + source_published_at: Date | null; + source_retrieved_at: Date | null; + source_content_sha256: string | null; confidence: string; }>( `SELECT @@ -842,6 +1011,13 @@ export class PgArchiveRepository { mr.valid_from, mr.valid_to, mr.source_id, + source.source_type, + source.url AS source_url, + source.title AS source_title, + source.author AS source_author, + source.published_at AS source_published_at, + source.retrieved_at AS source_retrieved_at, + source.content_sha256 AS source_content_sha256, mr.confidence::text AS confidence FROM modelapse.model_relations mr JOIN modelapse.models related @@ -850,10 +1026,123 @@ export class PgArchiveRepository { ELSE mr.from_model_id END JOIN modelapse.providers rp ON rp.id = related.provider_id + LEFT JOIN modelapse.source_records source ON source.id = mr.source_id WHERE mr.from_model_id = $1 OR mr.to_model_id = $1 ORDER BY mr.valid_from DESC NULLS LAST, mr.relation_type`, [modelId], ), + this.pool.query<{ + id: string; + alias_id: string; + alias_value: string; + observed_at: Date; + source_type_observed: string; + confidence: string; + resolved_model_id: string | null; + resolved_snapshot_id: string | null; + provider_snapshot_id: string | null; + source_id: string | null; + source_type: string | null; + source_url: string | null; + source_title: string | null; + source_author: string | null; + source_published_at: Date | null; + source_retrieved_at: Date | null; + source_content_sha256: string | null; + }>( + `SELECT + are.id, + ma.id AS alias_id, + ma.alias AS alias_value, + are.observed_at, + are.source_type AS source_type_observed, + are.confidence::text AS confidence, + are.resolved_model_id, + are.resolved_snapshot_id, + ms.provider_snapshot_id, + source.id AS source_id, + source.source_type, + source.url AS source_url, + source.title AS source_title, + source.author AS source_author, + source.published_at AS source_published_at, + source.retrieved_at AS source_retrieved_at, + source.content_sha256 AS source_content_sha256 + FROM modelapse.alias_resolution_events are + JOIN modelapse.model_aliases ma ON ma.id = are.alias_id + LEFT JOIN modelapse.model_snapshots ms ON ms.id = are.resolved_snapshot_id + LEFT JOIN modelapse.source_records source ON source.id = are.source_id + WHERE are.resolved_model_id = $1 + OR ms.model_id = $1 + ORDER BY are.observed_at DESC, are.id`, + [modelId], + ), + this.pool.query<{ + id: string; + api_model_id: string; + valid_from: Date; + valid_to: Date | null; + created_at: Date; + endpoint_id: string; + endpoint_path: string; + endpoint_base_url: string; + endpoint_hostname: string; + endpoint_source_id: string | null; + endpoint_source_type: string | null; + endpoint_source_url: string | null; + endpoint_source_title: string | null; + endpoint_source_author: string | null; + endpoint_source_published_at: Date | null; + endpoint_source_retrieved_at: Date | null; + endpoint_source_content_sha256: string | null; + snapshot_id: string | null; + provider_snapshot_id: string | null; + source_id: string; + source_type: string; + source_url: string | null; + source_title: string | null; + source_author: string | null; + source_published_at: Date | null; + source_retrieved_at: Date; + source_content_sha256: string | null; + }>( + `SELECT + meb.id, + meb.api_model_id, + meb.valid_from, + meb.valid_to, + meb.created_at, + pe.id AS endpoint_id, + pe.path AS endpoint_path, + pe.base_url AS endpoint_base_url, + pe.hostname AS endpoint_hostname, + endpoint_source.id AS endpoint_source_id, + endpoint_source.source_type AS endpoint_source_type, + endpoint_source.url AS endpoint_source_url, + endpoint_source.title AS endpoint_source_title, + endpoint_source.author AS endpoint_source_author, + endpoint_source.published_at AS endpoint_source_published_at, + endpoint_source.retrieved_at AS endpoint_source_retrieved_at, + endpoint_source.content_sha256 AS endpoint_source_content_sha256, + ms.id AS snapshot_id, + ms.provider_snapshot_id, + source.id AS source_id, + source.source_type, + source.url AS source_url, + source.title AS source_title, + source.author AS source_author, + source.published_at AS source_published_at, + source.retrieved_at AS source_retrieved_at, + source.content_sha256 AS source_content_sha256 + FROM modelapse.model_execution_bindings meb + JOIN modelapse.provider_endpoints pe ON pe.id = meb.endpoint_id + JOIN modelapse.source_records source ON source.id = meb.source_id + LEFT JOIN modelapse.source_records endpoint_source ON endpoint_source.id = pe.source_id + LEFT JOIN modelapse.model_snapshots ms ON ms.id = meb.snapshot_id + WHERE meb.model_id = $1 + ORDER BY meb.valid_from DESC, meb.created_at DESC`, + [modelId], + ), this.pool.query<{ test_case_id: string; family_slug: string; @@ -893,6 +1182,7 @@ export class PgArchiveRepository { validFrom: snapshot.valid_from?.toISOString() ?? null, validTo: snapshot.valid_to?.toISOString() ?? null, sourceId: snapshot.source_id, + source: archiveSourceView(snapshot), }), ); @@ -910,10 +1200,170 @@ export class PgArchiveRepository { validFrom: relation.valid_from?.toISOString() ?? null, validTo: relation.valid_to?.toISOString() ?? null, sourceId: relation.source_id, + source: archiveSourceView(relation), confidence: Number(relation.confidence), }), ); + const canonicalSource = archiveSourceView(row); + + const aliasResolutions: ArchiveModelAliasResolutionView[] = + aliasResult.rows.map((alias) => ({ + id: alias.id, + alias: { + id: alias.alias_id, + value: alias.alias_value, + }, + observedAt: alias.observed_at.toISOString(), + sourceType: alias.source_type_observed, + confidence: Number(alias.confidence), + resolvedModelId: alias.resolved_model_id, + resolvedSnapshot: + alias.resolved_snapshot_id && alias.provider_snapshot_id + ? { + id: alias.resolved_snapshot_id, + providerSnapshotId: alias.provider_snapshot_id, + } + : null, + source: archiveSourceView(alias), + })); + + const executionBindings: ArchiveModelExecutionBindingView[] = + bindingResult.rows.map((binding) => { + const endpointSource = archiveSourceView({ + source_id: binding.endpoint_source_id, + source_type: binding.endpoint_source_type, + source_url: binding.endpoint_source_url, + source_title: binding.endpoint_source_title, + source_author: binding.endpoint_source_author, + source_published_at: binding.endpoint_source_published_at, + source_retrieved_at: binding.endpoint_source_retrieved_at, + source_content_sha256: binding.endpoint_source_content_sha256, + }); + const bindingSource = archiveSourceView(binding); + if (!bindingSource) { + throw new Error( + "Model execution binding is missing its required source record", + ); + } + return { + id: binding.id, + apiModelId: binding.api_model_id, + validFrom: binding.valid_from.toISOString(), + validTo: binding.valid_to?.toISOString() ?? null, + createdAt: binding.created_at.toISOString(), + endpoint: { + id: binding.endpoint_id, + path: binding.endpoint_path, + baseUrl: binding.endpoint_base_url, + hostname: binding.endpoint_hostname, + source: endpointSource, + }, + snapshot: + binding.snapshot_id && binding.provider_snapshot_id + ? { + id: binding.snapshot_id, + providerSnapshotId: binding.provider_snapshot_id, + } + : null, + source: bindingSource, + }; + }); + + const identityTimeline: ArchiveIdentityTimelineEventView[] = []; + + if (canonicalSource) { + identityTimeline.push({ + id: `source:${canonicalSource.id}:canonical`, + kind: "canonical_source", + occurredAt: canonicalSource.retrievedAt, + title: "Canonical identity source recorded", + description: + canonicalSource.title ?? canonicalSource.url ?? canonicalSource.sourceType, + source: canonicalSource, + aliasId: null, + bindingId: null, + snapshotId: null, + }); + } + + for (const alias of aliasResolutions) { + identityTimeline.push({ + id: `alias-resolution:${alias.id}`, + kind: "alias_resolution", + occurredAt: alias.observedAt, + title: `Alias observed · ${alias.alias.value}`, + description: alias.resolvedSnapshot + ? `resolved to snapshot ${alias.resolvedSnapshot.providerSnapshotId}` + : `resolved to canonical model ${row.canonical_slug}`, + source: alias.source, + aliasId: alias.alias.id, + bindingId: null, + snapshotId: alias.resolvedSnapshot?.id ?? null, + }); + } + + for (const binding of executionBindings) { + identityTimeline.push({ + id: `binding:${binding.id}:start`, + kind: "binding_started", + occurredAt: binding.validFrom, + title: "Execution binding became valid", + description: `${binding.apiModelId} · ${binding.endpoint.hostname}`, + source: binding.source, + aliasId: null, + bindingId: binding.id, + snapshotId: binding.snapshot?.id ?? null, + }); + if (binding.validTo) { + identityTimeline.push({ + id: `binding:${binding.id}:end`, + kind: "binding_ended", + occurredAt: binding.validTo, + title: "Execution binding validity ended", + description: `${binding.apiModelId} · ${binding.endpoint.hostname}`, + source: binding.source, + aliasId: null, + bindingId: binding.id, + snapshotId: binding.snapshot?.id ?? null, + }); + } + } + + for (const snapshot of snapshots) { + if (snapshot.validFrom) { + identityTimeline.push({ + id: `identity-snapshot:${snapshot.id}:start`, + kind: "snapshot_started", + occurredAt: snapshot.validFrom, + title: "Provider snapshot became valid", + description: snapshot.providerSnapshotId, + source: snapshot.source, + aliasId: null, + bindingId: null, + snapshotId: snapshot.id, + }); + } + if (snapshot.validTo) { + identityTimeline.push({ + id: `identity-snapshot:${snapshot.id}:end`, + kind: "snapshot_ended", + occurredAt: snapshot.validTo, + title: "Provider snapshot validity ended", + description: snapshot.providerSnapshotId, + source: snapshot.source, + aliasId: null, + bindingId: null, + snapshotId: snapshot.id, + }); + } + } + + identityTimeline.sort( + (left, right) => + Date.parse(right.occurredAt) - Date.parse(left.occurredAt), + ); + const timeline: ArchiveTimelineEventView[] = []; if (row.released_at) { @@ -1044,8 +1494,12 @@ export class PgArchiveRepository { releasedAt: row.released_at?.toISOString() ?? null, retiredAt: row.retired_at?.toISOString() ?? null, canonicalSourceId: row.canonical_source_id, + canonicalSource, snapshots, relations, + aliasResolutions, + executionBindings, + identityTimeline, testCoverage: coverageResult.rows.map((coverage) => ({ testCaseId: coverage.test_case_id, familySlug: coverage.family_slug, @@ -1067,11 +1521,28 @@ export class PgArchiveRepository { family_name: string; origin: string; canonical_source_id: string | null; + family_source_id: string | null; + family_source_type: string | null; + family_source_url: string | null; + family_source_title: string | null; + family_source_author: string | null; + family_source_published_at: Date | null; + family_source_retrieved_at: Date | null; + family_source_content_sha256: string | null; + variant_id: string; variant_slug: string; variant_name: string; category: string; artifact_type: string; version: string; + version_source_id: string | null; + version_source_type: string | null; + version_source_url: string | null; + version_source_title: string | null; + version_source_author: string | null; + version_source_published_at: Date | null; + version_source_retrieved_at: Date | null; + version_source_content_sha256: string | null; version_status: string; definition_sha256: string; license: string | null; @@ -1096,11 +1567,28 @@ export class PgArchiveRepository { tf.name AS family_name, tf.origin, tf.canonical_source_id, + family_source.id AS family_source_id, + family_source.source_type AS family_source_type, + family_source.url AS family_source_url, + family_source.title AS family_source_title, + family_source.author AS family_source_author, + family_source.published_at AS family_source_published_at, + family_source.retrieved_at AS family_source_retrieved_at, + family_source.content_sha256 AS family_source_content_sha256, + tvar.id AS variant_id, tvar.slug AS variant_slug, tvar.name AS variant_name, tvar.category, tvar.artifact_type, tv.version, + version_source.id AS version_source_id, + version_source.source_type AS version_source_type, + version_source.url AS version_source_url, + version_source.title AS version_source_title, + version_source.author AS version_source_author, + version_source.published_at AS version_source_published_at, + version_source.retrieved_at AS version_source_retrieved_at, + version_source.content_sha256 AS version_source_content_sha256, tv.status AS version_status, tv.definition_sha256, tv.license, @@ -1127,6 +1615,10 @@ export class PgArchiveRepository { JOIN modelapse.test_versions tv ON tv.id = tc.test_version_id JOIN modelapse.test_variants tvar ON tvar.id = tv.variant_id JOIN modelapse.test_families tf ON tf.id = tvar.family_id + LEFT JOIN modelapse.source_records family_source + ON family_source.id = tf.canonical_source_id + LEFT JOIN modelapse.source_records version_source + ON version_source.id = tv.source_id LEFT JOIN LATERAL ( SELECT e.slug, e.version, e.kind, e.definition_sha256 FROM modelapse.test_version_evaluators tve @@ -1144,7 +1636,7 @@ export class PgArchiveRepository { const row = testResult.rows[0]; if (!row) return null; - const [coverageResult, recentRuns] = await Promise.all([ + const [coverageResult, versionHistoryResult, recentRuns] = await Promise.all([ this.pool.query<{ model_id: string; canonical_slug: string; @@ -1169,9 +1661,132 @@ export class PgArchiveRepository { ORDER BY MAX(r.completed_at) DESC NULLS LAST, p.slug, m.marketing_name`, [testCaseId], ), + this.pool.query<{ + id: string; + version: string; + status: string; + definition_sha256: string; + license: string | null; + published_at: Date | null; + created_at: Date; + source_id: string | null; + source_type: string | null; + source_url: string | null; + source_title: string | null; + source_author: string | null; + source_published_at: Date | null; + source_retrieved_at: Date | null; + source_content_sha256: string | null; + evaluator_slug: string | null; + evaluator_version: string | null; + evaluator_kind: string | null; + public_case_count: string; + linked_test_case_id: string | null; + }>( + `SELECT + tv.id, + tv.version, + tv.status, + tv.definition_sha256, + tv.license, + tv.published_at, + tv.created_at, + source.id AS source_id, + source.source_type, + source.url AS source_url, + source.title AS source_title, + source.author AS source_author, + source.published_at AS source_published_at, + source.retrieved_at AS source_retrieved_at, + source.content_sha256 AS source_content_sha256, + evaluator.slug AS evaluator_slug, + evaluator.version AS evaluator_version, + evaluator.kind AS evaluator_kind, + ( + SELECT COUNT(*)::text + FROM modelapse.test_cases public_case + WHERE public_case.test_version_id = tv.id + AND public_case.visibility = 'public' + ) AS public_case_count, + linked_case.id AS linked_test_case_id + FROM modelapse.test_versions tv + LEFT JOIN modelapse.source_records source ON source.id = tv.source_id + LEFT JOIN LATERAL ( + SELECT e.slug, e.version, e.kind + FROM modelapse.test_version_evaluators tve + JOIN modelapse.evaluators e ON e.id = tve.evaluator_id + WHERE tve.test_version_id = tv.id + ORDER BY e.slug, e.version + LIMIT 1 + ) evaluator ON true + LEFT JOIN LATERAL ( + SELECT public_case.id + FROM modelapse.test_cases public_case + WHERE public_case.test_version_id = tv.id + AND public_case.visibility = 'public' + AND public_case.slug = $2 + ORDER BY public_case.id + LIMIT 1 + ) linked_case ON true + WHERE tv.variant_id = $1 + AND EXISTS ( + SELECT 1 + FROM modelapse.test_cases public_case + WHERE public_case.test_version_id = tv.id + AND public_case.visibility = 'public' + ) + ORDER BY tv.published_at DESC NULLS LAST, tv.created_at DESC, tv.version DESC`, + [row.variant_id, row.case_slug], + ), this.listRuns({ testCaseId, limit: 50 }), ]); + const canonicalSource = archiveSourceView({ + source_id: row.family_source_id, + source_type: row.family_source_type, + source_url: row.family_source_url, + source_title: row.family_source_title, + source_author: row.family_source_author, + source_published_at: row.family_source_published_at, + source_retrieved_at: row.family_source_retrieved_at, + source_content_sha256: row.family_source_content_sha256, + }); + + const versionSource = archiveSourceView({ + source_id: row.version_source_id, + source_type: row.version_source_type, + source_url: row.version_source_url, + source_title: row.version_source_title, + source_author: row.version_source_author, + source_published_at: row.version_source_published_at, + source_retrieved_at: row.version_source_retrieved_at, + source_content_sha256: row.version_source_content_sha256, + }); + + const versionHistory: ArchiveTestVersionHistoryView[] = + versionHistoryResult.rows.map((version) => ({ + id: version.id, + version: version.version, + status: version.status, + definitionSha256: version.definition_sha256, + license: version.license, + publishedAt: version.published_at?.toISOString() ?? null, + createdAt: version.created_at.toISOString(), + source: archiveSourceView(version), + evaluator: + version.evaluator_slug && + version.evaluator_version && + version.evaluator_kind + ? { + slug: version.evaluator_slug, + version: version.evaluator_version, + kind: version.evaluator_kind, + } + : null, + publicCaseCount: Number(version.public_case_count), + linkedTestCaseId: version.linked_test_case_id, + })); + return { testCaseId: row.test_case_id, familySlug: row.family_slug, @@ -1193,6 +1808,9 @@ export class PgArchiveRepository { runCount: Number(row.run_count), origin: row.origin, canonicalSourceId: row.canonical_source_id, + canonicalSource, + versionSource, + versionHistory, versionStatus: row.version_status, definitionSha256: row.definition_sha256, license: row.license, diff --git a/packages/persistence/test/integration.test.ts b/packages/persistence/test/integration.test.ts index 3479250..d0b4082 100644 --- a/packages/persistence/test/integration.test.ts +++ b/packages/persistence/test/integration.test.ts @@ -29,11 +29,14 @@ describe("PostgreSQL Run persistence", () => { let modelId = ""; let relatedModelId = ""; let snapshotId = ""; + let sourceId = ""; + let previousTestCaseId = ""; beforeAll(async () => { root = await mkdtemp(join(tmpdir(), "modelapse-integration-")); const suffix = randomUUID().slice(0, 8); const definitionHash = randomBytes(32).toString("hex"); + const previousDefinitionHash = randomBytes(32).toString("hex"); const promptHash = randomBytes(32).toString("hex"); await seedPool.query( @@ -51,6 +54,25 @@ describe("PostgreSQL Run persistence", () => { ); providerId = provider.rows[0]!.id; + const source = await seedPool.query<{ id: string }>( + `INSERT INTO modelapse.source_records + (source_type, url, title, published_at, retrieved_at, content_sha256) + VALUES ( + 'provider_docs', + $1, + 'Integration provider identity source', + '2026-08-31T00:00:00.000Z', + '2026-09-01T00:00:00.000Z', + $2 + ) + RETURNING id`, + [ + `https://router.fake.test/docs/${suffix}`, + randomBytes(32).toString("hex"), + ], + ); + sourceId = source.rows[0]!.id; + const modelFamily = await seedPool.query<{ id: string }>( `INSERT INTO modelapse.model_families (provider_id, slug, display_name) @@ -76,9 +98,10 @@ describe("PostgreSQL Run persistence", () => { canonical_slug, marketing_name, released_at, - status + status, + canonical_source_id ) - VALUES ($1, $2, $3, $4, 'Integration Model', $5, 'active') + VALUES ($1, $2, $3, $4, 'Integration Model', $5, 'active', $6) RETURNING id`, [ providerId, @@ -86,6 +109,7 @@ describe("PostgreSQL Run persistence", () => { modelTrack.rows[0]!.id, `integration-model-${suffix}`, "2026-09-01T00:00:00.000Z", + sourceId, ], ); modelId = model.rows[0]!.id; @@ -115,25 +139,96 @@ describe("PostgreSQL Run persistence", () => { const snapshot = await seedPool.query<{ id: string }>( `INSERT INTO modelapse.model_snapshots - (model_id, provider_snapshot_id, valid_from) - VALUES ($1, $2, $3) + (model_id, provider_snapshot_id, valid_from, source_id) + VALUES ($1, $2, $3, $4) RETURNING id`, - [modelId, `integration-snapshot-${suffix}`, "2026-09-01T00:00:00.000Z"], + [ + modelId, + `integration-snapshot-${suffix}`, + "2026-09-01T00:00:00.000Z", + sourceId, + ], ); snapshotId = snapshot.rows[0]!.id; await seedPool.query( `INSERT INTO modelapse.model_relations - (from_model_id, to_model_id, relation_type, valid_from, confidence) - VALUES ($1, $2, 'successor_of', $3, 1.0)`, - [relatedModelId, modelId, "2026-09-20T00:00:00.000Z"], + (from_model_id, to_model_id, relation_type, valid_from, source_id, confidence) + VALUES ($1, $2, 'successor_of', $3, $4, 1.0)`, + [relatedModelId, modelId, "2026-09-20T00:00:00.000Z", sourceId], + ); + + const endpoint = await seedPool.query<{ id: string }>( + `INSERT INTO modelapse.provider_endpoints + (provider_id, path, base_url, hostname, valid_from, source_id) + VALUES ($1, 'routed_provider', $2, 'router.fake.test', $3, $4) + RETURNING id`, + [ + providerId, + `https://router.fake.test/${suffix}`, + "2026-09-01T00:00:00.000Z", + sourceId, + ], + ); + + await seedPool.query( + `INSERT INTO modelapse.model_execution_bindings + ( + model_id, + endpoint_id, + api_model_id, + snapshot_id, + valid_from, + source_id + ) + VALUES ($1, $2, $3, $4, $5, $6)`, + [ + modelId, + endpoint.rows[0]!.id, + `integration-api-model-${suffix}`, + snapshotId, + "2026-09-01T00:00:00.000Z", + sourceId, + ], + ); + + const alias = await seedPool.query<{ id: string }>( + `INSERT INTO modelapse.model_aliases + (provider_id, alias) + VALUES ($1, $2) + RETURNING id`, + [providerId, `integration-api-model-${suffix}`], + ); + + await seedPool.query( + `INSERT INTO modelapse.alias_resolution_events + ( + alias_id, + resolved_model_id, + resolved_snapshot_id, + observed_at, + source_type, + source_id, + confidence, + raw_observation + ) + VALUES ($1, $2, $3, $4, 'provider_docs', $5, 1.0, $6::jsonb)`, + [ + alias.rows[0]!.id, + modelId, + snapshotId, + "2026-09-02T00:00:00.000Z", + sourceId, + JSON.stringify({ privateNote: "must-not-leak" }), + ], ); const family = await seedPool.query<{ id: string }>( - `INSERT INTO modelapse.test_families (slug, name, origin) - VALUES ($1, $2, 'modelapse') + `INSERT INTO modelapse.test_families + (slug, name, origin, canonical_source_id) + VALUES ($1, $2, 'modelapse', $3) RETURNING id`, - [`integration-${suffix}`, "Integration Test"], + [`integration-${suffix}`, "Integration Test", sourceId], ); const variant = await seedPool.query<{ id: string }>( @@ -144,12 +239,43 @@ describe("PostgreSQL Run persistence", () => { [family.rows[0]!.id], ); + const previousVersion = await seedPool.query<{ id: string }>( + `INSERT INTO modelapse.test_versions + ( + variant_id, + version, + status, + definition_sha256, + source_id + ) + VALUES ($1, '0.9.0', 'draft', $2, $3) + RETURNING id`, + [variant.rows[0]!.id, previousDefinitionHash, sourceId], + ); + + const previousCase = await seedPool.query<{ id: string }>( + `INSERT INTO modelapse.test_cases + (test_version_id, slug, case_type, visibility, prompt_blob_sha256) + VALUES ($1, 'icon', 'icon', 'public', $2) + RETURNING id`, + [previousVersion.rows[0]!.id, promptHash], + ); + previousTestCaseId = previousCase.rows[0]!.id; + + await seedPool.query( + `UPDATE modelapse.test_versions + SET status = 'published', + published_at = $2 + WHERE id = $1`, + [previousVersion.rows[0]!.id, "2026-08-15T00:00:00.000Z"], + ); + const version = await seedPool.query<{ id: string }>( `INSERT INTO modelapse.test_versions - (variant_id, version, status, definition_sha256) - VALUES ($1, '1.0.0', 'draft', $2) + (variant_id, version, status, definition_sha256, source_id) + VALUES ($1, '1.0.0', 'draft', $2, $3) RETURNING id`, - [variant.rows[0]!.id, definitionHash], + [variant.rows[0]!.id, definitionHash, sourceId], ); const testCase = await seedPool.query<{ id: string }>( @@ -366,7 +492,22 @@ describe("PostgreSQL Run persistence", () => { family: { displayName: "Integration Models" }, track: { displayName: "Main" }, runCount: 2, - snapshots: [{ id: snapshotId }], + canonicalSource: { id: sourceId, sourceType: "provider_docs" }, + snapshots: [{ id: snapshotId, source: { id: sourceId } }], + aliasResolutions: [ + { + alias: { value: expect.stringContaining("integration-api-model-") }, + resolvedSnapshot: { id: snapshotId }, + source: { id: sourceId }, + }, + ], + executionBindings: [ + { + apiModelId: expect.stringContaining("integration-api-model-"), + snapshot: { id: snapshotId }, + source: { id: sourceId }, + }, + ], testCoverage: [{ testCaseId, runCount: 2 }], }); expect( @@ -381,12 +522,42 @@ describe("PostgreSQL Run persistence", () => { (event) => event.kind === "run" && event.runId === result.run.id, ), ).toBe(true); + expect( + archivedModel?.identityTimeline.some( + (event) => + event.kind === "alias_resolution" && + event.snapshotId === snapshotId && + event.source?.id === sourceId, + ), + ).toBe(true); + expect( + archivedModel?.identityTimeline.some( + (event) => + event.kind === "binding_started" && + event.source?.id === sourceId, + ), + ).toBe(true); + const archivedTest = await archive.getTest(testCaseId); expect(archivedTest).toMatchObject({ testCaseId, origin: "modelapse", runCount: 2, + canonicalSource: { id: sourceId }, + versionSource: { id: sourceId }, + versionHistory: [ + { + version: "1.0.0", + source: { id: sourceId }, + linkedTestCaseId: testCaseId, + }, + { + version: "0.9.0", + source: { id: sourceId }, + linkedTestCaseId: previousTestCaseId, + }, + ], modelCoverage: [{ modelId, runCount: 2 }], }); expect(archivedTest?.recentRuns[0]?.id).toBe(repeat.run.id);