From a4e67ed7df4010eefb56efa148c08b0ec3379fcb Mon Sep 17 00:00:00 2001 From: 0xPratik Date: Mon, 31 Aug 2026 17:52:31 +0545 Subject: [PATCH 1/4] Add regression for seeded model priority --- packages/hub-client/test/seed.test.ts | 81 ++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/packages/hub-client/test/seed.test.ts b/packages/hub-client/test/seed.test.ts index 47fc1d1e5..5fdb4f1c1 100644 --- a/packages/hub-client/test/seed.test.ts +++ b/packages/hub-client/test/seed.test.ts @@ -1455,7 +1455,11 @@ describe("seedCatalog", () => { test("fresh run creates the full provider-to-offering chain", async () => { const { lines, log } = collector(); const modelPosts: string[] = []; - const offeringPosts: { modelId: string; providerId: string }[] = []; + const offeringPosts: { + modelId: string; + providerId: string; + priority: number; + }[] = []; const handler: FakeHandler = (method, path, body) => { if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) return { status: 201, data: providerRow("prv_1", "anthropic") }; @@ -1540,7 +1544,6 @@ describe("seedCatalog", () => { "mdl_5", "mdl_6", ]); - const output = lines.join("\n"); expect(output).toContain("created provider anthropic"); expect(output).toContain("created credential anthropic-default"); @@ -1552,6 +1555,80 @@ describe("seedCatalog", () => { ); }); + test.failing( + "fresh run gives the declared Anthropic default the lowest distinct priority", + async () => { + const { log } = collector(); + const modelNamesById = new Map(); + const offeringPosts: { modelId: string; priority: number }[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "anthropic") }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/credentials` + ) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "anthropic-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const canonicalName = (body as { canonicalName: string }) + .canonicalName; + const modelId = `mdl_${modelNamesById.size + 1}`; + modelNamesById.set(modelId, canonicalName); + return { + status: 201, + data: catalogModelRow(modelId, canonicalName), + }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + const offering = body as { modelId: string; priority: number }; + offeringPosts.push(offering); + return { + status: 201, + data: catalogOfferingRow( + `off_${offeringPosts.length}`, + offering.modelId, + "cpv_1", + ), + }; + } + return undefined; + }; + + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + apiKey: "sk-test", + log, + }); + + const priorities = offeringPosts.map((offering) => offering.priority); + const sonnet = offeringPosts.find( + (offering) => + modelNamesById.get(offering.modelId) === "claude-sonnet-5", + ); + expect(new Set(priorities).size).toBe(offeringPosts.length); + expect(sonnet?.priority).toBe(Math.min(...priorities)); + }, + ); + test("an Ollama offering's quirks carry that model's real context-window ceiling, not the built-in 4096 default", async () => { const { log } = collector(); const offeringBodies: Record[] = []; From d298238a44bf0b9b5b95bc0f521d061713548948 Mon Sep 17 00:00:00 2001 From: 0xPratik Date: Mon, 31 Aug 2026 18:21:57 +0545 Subject: [PATCH 2/4] Seed model offerings in declared priority order --- packages/hub-client/src/seed.ts | 20 ++-- packages/hub-client/test/seed.test.ts | 151 +++++++++++++------------- 2 files changed, 85 insertions(+), 86 deletions(-) diff --git a/packages/hub-client/src/seed.ts b/packages/hub-client/src/seed.ts index f3ddc5cc9..87642148d 100644 --- a/packages/hub-client/src/seed.ts +++ b/packages/hub-client/src/seed.ts @@ -1566,14 +1566,14 @@ export async function seedCatalog( }, log, ); - // Priority = the provider's CATALOG_SEEDS declaration index (anthropic - // first), so when several connected providers serve the same model the - // fallback order is deterministic instead of an all-zeroes tie broken - // by insertion accident. - const offeringPriority = Math.max( - 0, - Object.keys(CATALOG_SEEDS).indexOf(provider), - ); + // Flatten the curated provider/model declaration order into one priority + // sequence. Provider order still controls cross-provider fallback, while + // model order makes each provider's declared default the first choice. + let offeringPriorityOffset = 0; + for (const [seedProvider, providerSeed] of Object.entries(CATALOG_SEEDS)) { + if (seedProvider === provider) break; + offeringPriorityOffset += providerSeed.models.length; + } // What each deployment can do, resolved from the pinned catalog's probe // results. Until this, every seeded offering stored an empty capability // list, so no capability filter — this repo's concept resolution or the @@ -1585,7 +1585,7 @@ export async function seedCatalog( canonicalName: string; capabilities: readonly string[]; }[] = []; - for (const model of seededModels) { + for (const [modelIndex, model] of seededModels.entries()) { // Ollama's dynamic entries already carry their own live-probed // capabilities (`fetchOllamaModelCatalog`, CL-6366) — narrowed against // the real `Capability` enum here, the trust boundary, rather than @@ -1627,7 +1627,7 @@ export async function seedCatalog( tenantId, modelId: model.id, providerId: catalogProviderId, - priority: offeringPriority, + priority: offeringPriorityOffset + modelIndex, capabilities, ...(quirks !== undefined ? { quirks } : {}), }, diff --git a/packages/hub-client/test/seed.test.ts b/packages/hub-client/test/seed.test.ts index 5fdb4f1c1..54117a67f 100644 --- a/packages/hub-client/test/seed.test.ts +++ b/packages/hub-client/test/seed.test.ts @@ -1501,7 +1501,7 @@ describe("seedCatalog", () => { capabilities: string[]; }; expect(offeringBody.providerId).toBe("cpv_1"); - expect(offeringBody.priority).toBe(0); + expect(offeringBody.priority).toBe(offeringPosts.length); expect(offeringBody.capabilities.length).toBeGreaterThan(0); expect(offeringBody.capabilities).toContain("plain-text"); expect(offeringBody.capabilities).toContain( @@ -1555,79 +1555,71 @@ describe("seedCatalog", () => { ); }); - test.failing( - "fresh run gives the declared Anthropic default the lowest distinct priority", - async () => { - const { log } = collector(); - const modelNamesById = new Map(); - const offeringPosts: { modelId: string; priority: number }[] = []; - const handler: FakeHandler = (method, path, body) => { - if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) - return { status: 201, data: providerRow("prv_1", "anthropic") }; - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/credentials` - ) - return { - status: 201, - data: credentialRow("cre_1", "prv_1", "anthropic-default"), - }; - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/catalog/models` - ) { - const canonicalName = (body as { canonicalName: string }) - .canonicalName; - const modelId = `mdl_${modelNamesById.size + 1}`; - modelNamesById.set(modelId, canonicalName); - return { - status: 201, - data: catalogModelRow(modelId, canonicalName), - }; - } - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/catalog/providers` - ) - return { - status: 201, - data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), - }; - if ( - method === "POST" && - path === `/api/tenants/${TENANT_ID}/catalog/offerings` - ) { - const offering = body as { modelId: string; priority: number }; - offeringPosts.push(offering); - return { - status: 201, - data: catalogOfferingRow( - `off_${offeringPosts.length}`, - offering.modelId, - "cpv_1", - ), - }; - } - return undefined; - }; + test("fresh run gives the declared Anthropic default the lowest distinct priority", async () => { + const { log } = collector(); + const modelNamesById = new Map(); + const offeringPosts: { modelId: string; priority: number }[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "anthropic") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "anthropic-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const canonicalName = (body as { canonicalName: string }).canonicalName; + const modelId = `mdl_${modelNamesById.size + 1}`; + modelNamesById.set(modelId, canonicalName); + return { + status: 201, + data: catalogModelRow(modelId, canonicalName), + }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + const offering = body as { modelId: string; priority: number }; + offeringPosts.push(offering); + return { + status: 201, + data: catalogOfferingRow( + `off_${offeringPosts.length}`, + offering.modelId, + "cpv_1", + ), + }; + } + return undefined; + }; - await seedCatalog({ - api: fakeAPI(handler), - cookies: [], - tenantId: TENANT_ID, - apiKey: "sk-test", - log, - }); - - const priorities = offeringPosts.map((offering) => offering.priority); - const sonnet = offeringPosts.find( - (offering) => - modelNamesById.get(offering.modelId) === "claude-sonnet-5", - ); - expect(new Set(priorities).size).toBe(offeringPosts.length); - expect(sonnet?.priority).toBe(Math.min(...priorities)); - }, - ); + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + apiKey: "sk-test", + log, + }); + + const priorities = offeringPosts.map((offering) => offering.priority); + const sonnet = offeringPosts.find( + (offering) => modelNamesById.get(offering.modelId) === "claude-sonnet-5", + ); + expect(new Set(priorities).size).toBe(offeringPosts.length); + expect(sonnet?.priority).toBe(Math.min(...priorities)); + }); test("an Ollama offering's quirks carry that model's real context-window ceiling, not the built-in 4096 default", async () => { const { log } = collector(); @@ -1790,6 +1782,7 @@ describe("seedCatalog", () => { const { lines, log } = collector(); let patchCalls = 0; let postCredentialCalls = 0; + let offeringPosts = 0; let patchBody: unknown; const staleCredentialRow = () => ({ @@ -1862,11 +1855,17 @@ describe("seedCatalog", () => { method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { - // Priority = the provider's CATALOG_SEEDS declaration index, so - // multi-provider fallback order is deterministic, never a tie. - expect((body as { priority: number }).priority).toBe( + const providersBeforeHuggingFace = Object.entries(CATALOG_SEEDS).slice( + 0, Object.keys(CATALOG_SEEDS).indexOf("huggingface"), ); + expect((body as { priority: number }).priority).toBe( + providersBeforeHuggingFace.reduce( + (offset, [, providerSeed]) => offset + providerSeed.models.length, + 0, + ) + offeringPosts, + ); + offeringPosts += 1; return { status: 201, data: catalogOfferingRow("off_1", "mdl_1", "cpv_1"), From 5739ad6890f84d5184f9541585607865ea989240 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 2 Sep 2026 08:23:03 -0700 Subject: [PATCH 3/4] Reconcile catalog offering priorities --- packages/hub-client/src/seed.ts | 47 +++++++++- packages/hub-client/test/seed.test.ts | 119 ++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 1 deletion(-) diff --git a/packages/hub-client/src/seed.ts b/packages/hub-client/src/seed.ts index 87642148d..e53cb8869 100644 --- a/packages/hub-client/src/seed.ts +++ b/packages/hub-client/src/seed.ts @@ -1339,7 +1339,52 @@ async function ensureCatalogOffering( return; } if (created.status === 409) { - log("catalog offering already exists (skipped)"); + let cursor: string | null = null; + let existing: typeof ModelOfferingResponse.infer | undefined; + do { + const listed = await api( + "GET", + `/api/tenants/${args.tenantId}/catalog/offerings${cursor === null ? "" : `?cursor=${encodeURIComponent(cursor)}`}`, + undefined, + cookies, + ); + const page = parseAs( + paginatedSchema(ModelOfferingResponse), + listed.data, + "catalog offerings response", + ); + existing = page.data.find( + (offering) => + offering.modelId === args.modelId && + offering.providerId === args.providerId, + ); + cursor = page.nextCursor; + } while (existing === undefined && cursor !== null); + if (!existing) { + throw new CliError( + "catalog offering reported a conflict but is not listable on the bench", + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + if (existing.priority === args.priority) { + log("catalog offering already exists (skipped)"); + return; + } + + const updated = await api( + "PATCH", + `/api/tenants/${args.tenantId}/catalog/offerings/${existing.id}`, + { priority: args.priority }, + cookies, + ); + if (updated.status !== 200) { + throw new CliError( + `the hub rejected updating the catalog offering priority with status ${updated.status}: ${JSON.stringify(updated.data)}`, + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + parseAs(ModelOfferingResponse, updated.data, "catalog offering response"); + log("updated catalog offering priority"); return; } throw new CliError( diff --git a/packages/hub-client/test/seed.test.ts b/packages/hub-client/test/seed.test.ts index 54117a67f..3916e3b0e 100644 --- a/packages/hub-client/test/seed.test.ts +++ b/packages/hub-client/test/seed.test.ts @@ -1621,6 +1621,107 @@ describe("seedCatalog", () => { expect(sonnet?.priority).toBe(Math.min(...priorities)); }); + test("re-run updates a legacy offering to its computed priority", async () => { + const { log } = collector(); + const patchedOfferings: { id: string; priority: number }[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "anthropic") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "anthropic-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const canonicalName = (body as { canonicalName: string }).canonicalName; + const modelId = + canonicalName === "claude-opus-5" ? "mdl_legacy" : "mdl_new"; + return { status: 201, data: catalogModelRow(modelId, canonicalName) }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + const offering = body as { modelId: string }; + if (offering.modelId === "mdl_legacy") + return { status: 409, data: { error: "already exists" } }; + return { + status: 201, + data: catalogOfferingRow( + `off_${offering.modelId}`, + offering.modelId, + "cpv_1", + ), + }; + } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 200, + data: { + data: [ + catalogOfferingRow( + "off_other_provider", + "mdl_legacy", + "cpv_other", + ), + ], + nextCursor: "second-page", + }, + }; + if ( + method === "GET" && + path === + `/api/tenants/${TENANT_ID}/catalog/offerings?cursor=second-page` + ) + return { + status: 200, + data: { + data: [catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1")], + nextCursor: null, + }, + }; + if ( + method === "PATCH" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings/off_legacy` + ) { + const patch = body as { priority: number }; + patchedOfferings.push({ id: "off_legacy", priority: patch.priority }); + return { + status: 200, + data: { + ...catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1"), + priority: patch.priority, + }, + }; + } + return undefined; + }; + + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + apiKey: "sk-test", + log, + }); + + expect(patchedOfferings).toEqual([{ id: "off_legacy", priority: 1 }]); + }); + test("an Ollama offering's quirks carry that model's real context-window ceiling, not the built-in 4096 default", async () => { const { log } = collector(); const offeringBodies: Record[] = []; @@ -2266,6 +2367,24 @@ describe("seedCatalog", () => { offeringPosts += 1; return { status: 409, data: { error: "already exists" } }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 200, + data: { + data: anthropicModels.map((_, index) => ({ + ...catalogOfferingRow( + `off_${index + 1}`, + `mdl_${index + 1}`, + "cpv_1", + ), + priority: index, + })), + nextCursor: null, + }, + }; return undefined; }; From 8711245c880a5c7f78e9cc0b18fec9c3b3e5ffa6 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 00:23:08 -0700 Subject: [PATCH 4/4] Teach onboarding Hub mock GET catalog/offerings ensureCatalogOffering lists existing offerings after a 409 so it can reconcile priority. The onboarding fake Hub only stubbed the POST, so a second credential save threw instead of treating the catalog as already seeded. --- .../test/complete-credential.test.ts | 62 +++++++++++++++++-- .../test/huggingface-connect-routes.test.ts | 3 + .../test/openrouter-connect-routes.test.ts | 3 + 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index c754be3fe..68ea5a548 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -756,6 +756,17 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || + path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + ) { + return { + status: 200, + data: { data: [], nextCursor: null }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; @@ -949,7 +960,12 @@ describe("completeCredentialSetup", () => { const deployments: { definitionAssetId: string; id: string }[] = []; const catalogModels: Row[] = []; const catalogProviders: Row[] = []; - const catalogOfferings: { modelId: string; providerId: string }[] = []; + const catalogOfferings: { + id: string; + modelId: string; + providerId: string; + priority: number; + }[] = []; const providers: Row[] = []; const credentials: Row[] = []; let assetCreatePosts = 0; @@ -1331,21 +1347,31 @@ describe("completeCredentialSetup", () => { method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { - const b = body as { modelId: string; providerId: string }; + const b = body as { + modelId: string; + providerId: string; + priority: number; + }; const existing = catalogOfferings.find( (o) => o.modelId === b.modelId && o.providerId === b.providerId, ); if (existing) return { status: 409, data: {}, cookies: [] }; catalogOfferingCreatePosts += 1; - catalogOfferings.push({ modelId: b.modelId, providerId: b.providerId }); + const id = `off_${catalogOfferings.length + 1}`; + catalogOfferings.push({ + id, + modelId: b.modelId, + providerId: b.providerId, + priority: b.priority, + }); return { status: 201, data: { - id: `off_${catalogOfferings.length}`, + id, tenantId: TENANT_ID, modelId: b.modelId, providerId: b.providerId, - priority: 0, + priority: b.priority, deploymentTags: [], capabilities: [], quirks: null, @@ -1356,6 +1382,32 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || + path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + ) { + return { + status: 200, + data: { + data: catalogOfferings.map((o) => ({ + id: o.id, + tenantId: TENANT_ID, + modelId: o.modelId, + providerId: o.providerId, + priority: o.priority, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: TIMESTAMP, + updatedAt: TIMESTAMP, + })), + nextCursor: null, + }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; diff --git a/packages/onboarding/test/huggingface-connect-routes.test.ts b/packages/onboarding/test/huggingface-connect-routes.test.ts index c19a81db3..bdcfe2162 100644 --- a/packages/onboarding/test/huggingface-connect-routes.test.ts +++ b/packages/onboarding/test/huggingface-connect-routes.test.ts @@ -156,6 +156,9 @@ function mockHub() { 201, ), ); + hub.get("/api/tenants/ten_1/catalog/offerings", (c) => + c.json({ data: [], nextCursor: null }), + ); return hub; } diff --git a/packages/onboarding/test/openrouter-connect-routes.test.ts b/packages/onboarding/test/openrouter-connect-routes.test.ts index a18304407..27454ab4c 100644 --- a/packages/onboarding/test/openrouter-connect-routes.test.ts +++ b/packages/onboarding/test/openrouter-connect-routes.test.ts @@ -154,6 +154,9 @@ function mockHub() { 201, ), ); + hub.get("/api/tenants/ten_1/catalog/offerings", (c) => + c.json({ data: [], nextCursor: null }), + ); return hub; }