diff --git a/packages/hub-client/src/seed.ts b/packages/hub-client/src/seed.ts index f3ddc5cc9..e53cb8869 100644 --- a/packages/hub-client/src/seed.ts +++ b/packages/hub-client/src/seed.ts @@ -1339,7 +1339,52 @@ async function ensureCatalogOffering( return; } if (created.status === 409) { - log("catalog offering already exists (skipped)"); + let cursor: string | null = null; + let existing: typeof ModelOfferingResponse.infer | undefined; + do { + const listed = await api( + "GET", + `/api/tenants/${args.tenantId}/catalog/offerings${cursor === null ? "" : `?cursor=${encodeURIComponent(cursor)}`}`, + undefined, + cookies, + ); + const page = parseAs( + paginatedSchema(ModelOfferingResponse), + listed.data, + "catalog offerings response", + ); + existing = page.data.find( + (offering) => + offering.modelId === args.modelId && + offering.providerId === args.providerId, + ); + cursor = page.nextCursor; + } while (existing === undefined && cursor !== null); + if (!existing) { + throw new CliError( + "catalog offering reported a conflict but is not listable on the bench", + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + if (existing.priority === args.priority) { + log("catalog offering already exists (skipped)"); + return; + } + + const updated = await api( + "PATCH", + `/api/tenants/${args.tenantId}/catalog/offerings/${existing.id}`, + { priority: args.priority }, + cookies, + ); + if (updated.status !== 200) { + throw new CliError( + `the hub rejected updating the catalog offering priority with status ${updated.status}: ${JSON.stringify(updated.data)}`, + "check the hub logs for the underlying failure, then re-run: workbench seed", + ); + } + parseAs(ModelOfferingResponse, updated.data, "catalog offering response"); + log("updated catalog offering priority"); return; } throw new CliError( @@ -1566,14 +1611,14 @@ export async function seedCatalog( }, log, ); - // Priority = the provider's CATALOG_SEEDS declaration index (anthropic - // first), so when several connected providers serve the same model the - // fallback order is deterministic instead of an all-zeroes tie broken - // by insertion accident. - const offeringPriority = Math.max( - 0, - Object.keys(CATALOG_SEEDS).indexOf(provider), - ); + // Flatten the curated provider/model declaration order into one priority + // sequence. Provider order still controls cross-provider fallback, while + // model order makes each provider's declared default the first choice. + let offeringPriorityOffset = 0; + for (const [seedProvider, providerSeed] of Object.entries(CATALOG_SEEDS)) { + if (seedProvider === provider) break; + offeringPriorityOffset += providerSeed.models.length; + } // What each deployment can do, resolved from the pinned catalog's probe // results. Until this, every seeded offering stored an empty capability // list, so no capability filter — this repo's concept resolution or the @@ -1585,7 +1630,7 @@ export async function seedCatalog( canonicalName: string; capabilities: readonly string[]; }[] = []; - for (const model of seededModels) { + for (const [modelIndex, model] of seededModels.entries()) { // Ollama's dynamic entries already carry their own live-probed // capabilities (`fetchOllamaModelCatalog`, CL-6366) — narrowed against // the real `Capability` enum here, the trust boundary, rather than @@ -1627,7 +1672,7 @@ export async function seedCatalog( tenantId, modelId: model.id, providerId: catalogProviderId, - priority: offeringPriority, + priority: offeringPriorityOffset + modelIndex, capabilities, ...(quirks !== undefined ? { quirks } : {}), }, diff --git a/packages/hub-client/test/seed.test.ts b/packages/hub-client/test/seed.test.ts index 47fc1d1e5..3916e3b0e 100644 --- a/packages/hub-client/test/seed.test.ts +++ b/packages/hub-client/test/seed.test.ts @@ -1455,7 +1455,11 @@ describe("seedCatalog", () => { test("fresh run creates the full provider-to-offering chain", async () => { const { lines, log } = collector(); const modelPosts: string[] = []; - const offeringPosts: { modelId: string; providerId: string }[] = []; + const offeringPosts: { + modelId: string; + providerId: string; + priority: number; + }[] = []; const handler: FakeHandler = (method, path, body) => { if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) return { status: 201, data: providerRow("prv_1", "anthropic") }; @@ -1497,7 +1501,7 @@ describe("seedCatalog", () => { capabilities: string[]; }; expect(offeringBody.providerId).toBe("cpv_1"); - expect(offeringBody.priority).toBe(0); + expect(offeringBody.priority).toBe(offeringPosts.length); expect(offeringBody.capabilities.length).toBeGreaterThan(0); expect(offeringBody.capabilities).toContain("plain-text"); expect(offeringBody.capabilities).toContain( @@ -1540,7 +1544,6 @@ describe("seedCatalog", () => { "mdl_5", "mdl_6", ]); - const output = lines.join("\n"); expect(output).toContain("created provider anthropic"); expect(output).toContain("created credential anthropic-default"); @@ -1552,6 +1555,173 @@ describe("seedCatalog", () => { ); }); + test("fresh run gives the declared Anthropic default the lowest distinct priority", async () => { + const { log } = collector(); + const modelNamesById = new Map(); + const offeringPosts: { modelId: string; priority: number }[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "anthropic") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "anthropic-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const canonicalName = (body as { canonicalName: string }).canonicalName; + const modelId = `mdl_${modelNamesById.size + 1}`; + modelNamesById.set(modelId, canonicalName); + return { + status: 201, + data: catalogModelRow(modelId, canonicalName), + }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + const offering = body as { modelId: string; priority: number }; + offeringPosts.push(offering); + return { + status: 201, + data: catalogOfferingRow( + `off_${offeringPosts.length}`, + offering.modelId, + "cpv_1", + ), + }; + } + return undefined; + }; + + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + apiKey: "sk-test", + log, + }); + + const priorities = offeringPosts.map((offering) => offering.priority); + const sonnet = offeringPosts.find( + (offering) => modelNamesById.get(offering.modelId) === "claude-sonnet-5", + ); + expect(new Set(priorities).size).toBe(offeringPosts.length); + expect(sonnet?.priority).toBe(Math.min(...priorities)); + }); + + test("re-run updates a legacy offering to its computed priority", async () => { + const { log } = collector(); + const patchedOfferings: { id: string; priority: number }[] = []; + const handler: FakeHandler = (method, path, body) => { + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/providers`) + return { status: 201, data: providerRow("prv_1", "anthropic") }; + if (method === "POST" && path === `/api/tenants/${TENANT_ID}/credentials`) + return { + status: 201, + data: credentialRow("cre_1", "prv_1", "anthropic-default"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/models` + ) { + const canonicalName = (body as { canonicalName: string }).canonicalName; + const modelId = + canonicalName === "claude-opus-5" ? "mdl_legacy" : "mdl_new"; + return { status: 201, data: catalogModelRow(modelId, canonicalName) }; + } + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/providers` + ) + return { + status: 201, + data: catalogProviderRow("cpv_1", "anthropic", "cre_1"), + }; + if ( + method === "POST" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) { + const offering = body as { modelId: string }; + if (offering.modelId === "mdl_legacy") + return { status: 409, data: { error: "already exists" } }; + return { + status: 201, + data: catalogOfferingRow( + `off_${offering.modelId}`, + offering.modelId, + "cpv_1", + ), + }; + } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 200, + data: { + data: [ + catalogOfferingRow( + "off_other_provider", + "mdl_legacy", + "cpv_other", + ), + ], + nextCursor: "second-page", + }, + }; + if ( + method === "GET" && + path === + `/api/tenants/${TENANT_ID}/catalog/offerings?cursor=second-page` + ) + return { + status: 200, + data: { + data: [catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1")], + nextCursor: null, + }, + }; + if ( + method === "PATCH" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings/off_legacy` + ) { + const patch = body as { priority: number }; + patchedOfferings.push({ id: "off_legacy", priority: patch.priority }); + return { + status: 200, + data: { + ...catalogOfferingRow("off_legacy", "mdl_legacy", "cpv_1"), + priority: patch.priority, + }, + }; + } + return undefined; + }; + + await seedCatalog({ + api: fakeAPI(handler), + cookies: [], + tenantId: TENANT_ID, + apiKey: "sk-test", + log, + }); + + expect(patchedOfferings).toEqual([{ id: "off_legacy", priority: 1 }]); + }); + test("an Ollama offering's quirks carry that model's real context-window ceiling, not the built-in 4096 default", async () => { const { log } = collector(); const offeringBodies: Record[] = []; @@ -1713,6 +1883,7 @@ describe("seedCatalog", () => { const { lines, log } = collector(); let patchCalls = 0; let postCredentialCalls = 0; + let offeringPosts = 0; let patchBody: unknown; const staleCredentialRow = () => ({ @@ -1785,11 +1956,17 @@ describe("seedCatalog", () => { method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { - // Priority = the provider's CATALOG_SEEDS declaration index, so - // multi-provider fallback order is deterministic, never a tie. - expect((body as { priority: number }).priority).toBe( + const providersBeforeHuggingFace = Object.entries(CATALOG_SEEDS).slice( + 0, Object.keys(CATALOG_SEEDS).indexOf("huggingface"), ); + expect((body as { priority: number }).priority).toBe( + providersBeforeHuggingFace.reduce( + (offset, [, providerSeed]) => offset + providerSeed.models.length, + 0, + ) + offeringPosts, + ); + offeringPosts += 1; return { status: 201, data: catalogOfferingRow("off_1", "mdl_1", "cpv_1"), @@ -2190,6 +2367,24 @@ describe("seedCatalog", () => { offeringPosts += 1; return { status: 409, data: { error: "already exists" } }; } + if ( + method === "GET" && + path === `/api/tenants/${TENANT_ID}/catalog/offerings` + ) + return { + status: 200, + data: { + data: anthropicModels.map((_, index) => ({ + ...catalogOfferingRow( + `off_${index + 1}`, + `mdl_${index + 1}`, + "cpv_1", + ), + priority: index, + })), + nextCursor: null, + }, + }; return undefined; }; diff --git a/packages/onboarding/test/complete-credential.test.ts b/packages/onboarding/test/complete-credential.test.ts index c754be3fe..68ea5a548 100644 --- a/packages/onboarding/test/complete-credential.test.ts +++ b/packages/onboarding/test/complete-credential.test.ts @@ -756,6 +756,17 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || + path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + ) { + return { + status: 200, + data: { data: [], nextCursor: null }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; @@ -949,7 +960,12 @@ describe("completeCredentialSetup", () => { const deployments: { definitionAssetId: string; id: string }[] = []; const catalogModels: Row[] = []; const catalogProviders: Row[] = []; - const catalogOfferings: { modelId: string; providerId: string }[] = []; + const catalogOfferings: { + id: string; + modelId: string; + providerId: string; + priority: number; + }[] = []; const providers: Row[] = []; const credentials: Row[] = []; let assetCreatePosts = 0; @@ -1331,21 +1347,31 @@ describe("completeCredentialSetup", () => { method === "POST" && path === `/api/tenants/${TENANT_ID}/catalog/offerings` ) { - const b = body as { modelId: string; providerId: string }; + const b = body as { + modelId: string; + providerId: string; + priority: number; + }; const existing = catalogOfferings.find( (o) => o.modelId === b.modelId && o.providerId === b.providerId, ); if (existing) return { status: 409, data: {}, cookies: [] }; catalogOfferingCreatePosts += 1; - catalogOfferings.push({ modelId: b.modelId, providerId: b.providerId }); + const id = `off_${catalogOfferings.length + 1}`; + catalogOfferings.push({ + id, + modelId: b.modelId, + providerId: b.providerId, + priority: b.priority, + }); return { status: 201, data: { - id: `off_${catalogOfferings.length}`, + id, tenantId: TENANT_ID, modelId: b.modelId, providerId: b.providerId, - priority: 0, + priority: b.priority, deploymentTags: [], capabilities: [], quirks: null, @@ -1356,6 +1382,32 @@ describe("completeCredentialSetup", () => { cookies: [], }; } + if ( + method === "GET" && + (path === `/api/tenants/${TENANT_ID}/catalog/offerings` || + path.startsWith(`/api/tenants/${TENANT_ID}/catalog/offerings?`)) + ) { + return { + status: 200, + data: { + data: catalogOfferings.map((o) => ({ + id: o.id, + tenantId: TENANT_ID, + modelId: o.modelId, + providerId: o.providerId, + priority: o.priority, + deploymentTags: [], + capabilities: [], + quirks: null, + disabled: false, + createdAt: TIMESTAMP, + updatedAt: TIMESTAMP, + })), + nextCursor: null, + }, + cookies: [], + }; + } throw new Error(`unexpected call: ${method} ${path}`); }; diff --git a/packages/onboarding/test/huggingface-connect-routes.test.ts b/packages/onboarding/test/huggingface-connect-routes.test.ts index c19a81db3..bdcfe2162 100644 --- a/packages/onboarding/test/huggingface-connect-routes.test.ts +++ b/packages/onboarding/test/huggingface-connect-routes.test.ts @@ -156,6 +156,9 @@ function mockHub() { 201, ), ); + hub.get("/api/tenants/ten_1/catalog/offerings", (c) => + c.json({ data: [], nextCursor: null }), + ); return hub; } diff --git a/packages/onboarding/test/openrouter-connect-routes.test.ts b/packages/onboarding/test/openrouter-connect-routes.test.ts index a18304407..27454ab4c 100644 --- a/packages/onboarding/test/openrouter-connect-routes.test.ts +++ b/packages/onboarding/test/openrouter-connect-routes.test.ts @@ -154,6 +154,9 @@ function mockHub() { 201, ), ); + hub.get("/api/tenants/ten_1/catalog/offerings", (c) => + c.json({ data: [], nextCursor: null }), + ); return hub; }