From c76311e4daf2e767033d175d2ceab0d9a16c7b22 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 1 Sep 2026 21:03:44 -0700 Subject: [PATCH 1/3] Normalize OpenCode Go response deltas --- src/config.test.ts | 14 +++++ src/config/index.ts | 18 +++--- src/provider/inference-dependencies.ts | 12 +++- .../openai-compatible-adapter.test.ts | 14 +++++ src/provider/openai-compatible-adapter.ts | 7 ++- src/provider/opencode-go-adapter.test.ts | 61 +++++++++++++++++++ src/provider/opencode-go-adapter.ts | 42 +++++++++++++ tests/unit/inference-sources.test.ts | 3 +- 8 files changed, 159 insertions(+), 12 deletions(-) create mode 100644 src/provider/opencode-go-adapter.test.ts create mode 100644 src/provider/opencode-go-adapter.ts diff --git a/src/config.test.ts b/src/config.test.ts index 1dedfb626..629bdf15f 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -5,6 +5,7 @@ import { join, resolve } from "node:path"; import { buildBifrostSource, + buildGoSource, buildOpenAISource, buildXaiSource, buildProviderCatalog, @@ -1178,6 +1179,19 @@ describe("loadConfig", () => { }); }); +describe("buildGoSource", () => { + test("routes chat-completions models through the OpenCode Go adapter", () => { + const source = buildGoSource({ + id: "opencode-go", + apiKey: "sk-go", + model: "kimi-k2.7-code", + }); + + expect(source.provider).toBe("opencode-go"); + expect(source.quirks).toBeUndefined(); + }); +}); + describe("buildOpenAISource", () => { test("normalizes the runtime source baseURL", () => { const source = buildOpenAISource({ diff --git a/src/config/index.ts b/src/config/index.ts index 25c831353..cef36cd3b 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -46,6 +46,7 @@ import { import { xaiUserIdFromAccessToken } from "../auth/xai/session.js"; import { OPENCODE_GO_BASE_URL, + OPENCODE_GO_PROVIDER_ID, isOpenCodeGoProvider, resolveGoEndpoint, } from "../../packages/opencode-go/src/index.js"; @@ -372,13 +373,16 @@ export function buildGoSource(fields: { }; } // chat-completions (default) - return buildOpenAISource({ - id: fields.id, - baseURL: endpoint.baseURL.length > 0 ? endpoint.baseURL : OPENCODE_GO_BASE_URL, - apiKey, - model: fields.model, - ...(fields.reasoningEffort !== undefined ? { reasoningEffort: fields.reasoningEffort } : {}), - }); + return { + ...buildOpenAISource({ + id: fields.id, + baseURL: endpoint.baseURL.length > 0 ? endpoint.baseURL : OPENCODE_GO_BASE_URL, + apiKey, + model: fields.model, + ...(fields.reasoningEffort !== undefined ? { reasoningEffort: fields.reasoningEffort } : {}), + }), + provider: OPENCODE_GO_PROVIDER_ID, + }; } export interface Config { diff --git a/src/provider/inference-dependencies.ts b/src/provider/inference-dependencies.ts index f47401f96..4a38d0f1b 100644 --- a/src/provider/inference-dependencies.ts +++ b/src/provider/inference-dependencies.ts @@ -1,6 +1,7 @@ import { createDependencies, type Dependencies, type AdapterManifest } from "@intx/inference"; import { loadAdapterRegistry } from "@intx/inference/providers"; import * as openaiCompatible from "./openai-compatible-adapter.js"; +import * as opencodeGo from "./opencode-go-adapter.js"; import * as codexResponses from "./codex-responses-adapter.js"; import * as grokResponses from "./grok-responses-adapter.js"; import * as bifrostAdapter from "./bifrost-adapter.js"; @@ -8,18 +9,24 @@ import * as openaiResponses from "./openai-responses-adapter.js"; import { CODEX_RESPONSES_PROVIDER, withCodexContentTypeRepair } from "./codex-responses-adapter.js"; import { GROK_RESPONSES_PROVIDER } from "./grok-responses-adapter.js"; import { withReplaySanitizer } from "./replay-sanitizer.js"; +import { OPENCODE_GO_PROVIDER_ID } from "../../packages/opencode-go/src/index.js"; import { BIFROST_PROVIDER } from "./bifrost-adapter.js"; import { OPENAI_RESPONSES_PROVIDER } from "./openai-responses-adapter.js"; // Corbits Code ships first-party adapters on top of the built-in provider set: -// openai-compatible override, Codex/Grok responses, Bifrost, and generic -// openai-responses (OpenCode Go gpt-* Luna family). +// openai-compatible and OpenCode Go chat-completions adapters, Codex/Grok +// responses, Bifrost, and generic openai-responses (OpenCode Go gpt-* Luna family). const manifest: AdapterManifest = [ { provider: "openai-compatible", specifier: "openai-compatible-adapter", export: "createOpenAICompatibleAdapter", }, + { + provider: OPENCODE_GO_PROVIDER_ID, + specifier: "opencode-go-adapter", + export: "createOpenCodeGoAdapter", + }, { provider: CODEX_RESPONSES_PROVIDER, specifier: "codex-responses-adapter", @@ -44,6 +51,7 @@ const manifest: AdapterManifest = [ const localModules: Record = { "openai-compatible-adapter": openaiCompatible, + "opencode-go-adapter": opencodeGo, "codex-responses-adapter": codexResponses, "grok-responses-adapter": grokResponses, "bifrost-adapter": bifrostAdapter, diff --git a/src/provider/openai-compatible-adapter.test.ts b/src/provider/openai-compatible-adapter.test.ts index 29f628820..871631724 100644 --- a/src/provider/openai-compatible-adapter.test.ts +++ b/src/provider/openai-compatible-adapter.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect } from "bun:test"; +import { ProtocolMismatchError } from "@intx/inference"; import type { ConversationTurn, InferenceOptions } from "@intx/types/runtime"; import { createOpenAICompatibleAdapter } from "./openai-compatible-adapter.js"; @@ -80,6 +81,19 @@ describe("openai-compatible adapter SSE parse count", () => { }); }); +describe("openai-compatible adapter null delta fields", () => { + test.each(["role", "tool_calls"])("rejects null %s", (field) => { + const adapter = createOpenAICompatibleAdapter(source); + expect(() => + adapter.parseResponse( + JSON.stringify({ + choices: [{ index: 0, delta: { content: "hello", [field]: null } }], + }), + ), + ).toThrow(ProtocolMismatchError); + }); +}); + describe("openai-compatible adapter reasoning_content handling", () => { const withThinking: ConversationTurn[] = [ { role: "user", content: [{ type: "text", text: "hi" }] }, diff --git a/src/provider/openai-compatible-adapter.ts b/src/provider/openai-compatible-adapter.ts index e752d9794..987275979 100644 --- a/src/provider/openai-compatible-adapter.ts +++ b/src/provider/openai-compatible-adapter.ts @@ -15,8 +15,11 @@ import { createOpenAIAdapter } from "@intx/inference/providers"; // adapter for every source corbits builds. type AdapterSource = Parameters[0]; -export function createOpenAICompatibleAdapter(source: AdapterSource): ProviderAdapter { - const base = createOpenAIAdapter(source); +export function createOpenAICompatibleAdapter( + source: AdapterSource, + quirks?: unknown, +): ProviderAdapter { + const base = createOpenAIAdapter(source, quirks); // Set by buildRequest for the model the current request targets; only // DeepSeek/NIM streams need the null-delta-field patch below, so every // other provider's frames skip the reparse and hit base.parseResponse diff --git a/src/provider/opencode-go-adapter.test.ts b/src/provider/opencode-go-adapter.test.ts new file mode 100644 index 000000000..ac51586cc --- /dev/null +++ b/src/provider/opencode-go-adapter.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { ProtocolMismatchError } from "@intx/inference"; +import type { InferenceOptions } from "@intx/types/runtime"; +import { OPENCODE_GO_PROVIDER_ID } from "../../packages/opencode-go/src/index.js"; +import { createInferenceDependencies } from "./inference-dependencies.js"; +import { createOpenCodeGoAdapter } from "./opencode-go-adapter.js"; + +const source = { + sourceId: OPENCODE_GO_PROVIDER_ID, + provider: OPENCODE_GO_PROVIDER_ID, + model: "corbits", +}; + +describe("OpenCode Go adapter", () => { + test("normalizes null role and tool_calls through the runtime registry", async () => { + const deps = await createInferenceDependencies(); + const adapter = deps.adapters.resolve(source); + const chunk = JSON.stringify({ + choices: [{ index: 0, delta: { content: "hello", role: null, tool_calls: null } }], + }); + + expect(adapter.parseResponse(chunk)).toContainEqual({ + type: "inference.text.delta", + seq: 0, + data: { token: "hello", partial: { text: "" }, index: 0 }, + }); + }); + + test("keeps malformed non-null roles strict", () => { + const adapter = createOpenCodeGoAdapter(source); + const malformedChunk = JSON.stringify({ + choices: [{ index: 0, delta: { content: "hello", role: 42 } }], + }); + + expect(() => adapter.parseResponse(malformedChunk)).toThrow(ProtocolMismatchError); + }); + + test("keeps malformed non-null tool_calls strict", () => { + const adapter = createOpenCodeGoAdapter(source); + const malformedChunk = JSON.stringify({ + choices: [{ index: 0, delta: { content: "hello", tool_calls: "invalid" } }], + }); + + expect(() => adapter.parseResponse(malformedChunk)).toThrow(ProtocolMismatchError); + }); + + test("delegates request construction to the OpenAI-compatible adapter", () => { + const adapter = createOpenCodeGoAdapter(source); + const request = adapter.buildRequest( + [{ role: "user", timestamp: 0, content: [{ type: "text", text: "hi" }] }], + "corbits", + { providerOptions: { reasoning_effort: "high" } } as InferenceOptions, + ); + + expect(JSON.parse(request.body)).toMatchObject({ + model: "corbits", + reasoning_effort: "high", + stream: true, + }); + }); +}); diff --git a/src/provider/opencode-go-adapter.ts b/src/provider/opencode-go-adapter.ts new file mode 100644 index 000000000..02c1eef1c --- /dev/null +++ b/src/provider/opencode-go-adapter.ts @@ -0,0 +1,42 @@ +import type { ProviderAdapter } from "@intx/inference"; +import { createOpenAICompatibleAdapter } from "./openai-compatible-adapter.js"; + +type AdapterSource = Parameters[0]; + +const NULL_DELTA_FIELDS = ["role", "tool_calls"] as const; + +function normalizeNullDeltaFields(sseData: string): string { + let parsed: unknown; + try { + parsed = JSON.parse(sseData); + } catch { + return sseData; + } + if (parsed === null || typeof parsed !== "object") return sseData; + + const choices = (parsed as Record)["choices"]; + if (!Array.isArray(choices)) return sseData; + + let normalized = false; + for (const choice of choices) { + if (choice === null || typeof choice !== "object") continue; + const delta = (choice as Record)["delta"]; + if (delta === null || typeof delta !== "object") continue; + for (const field of NULL_DELTA_FIELDS) { + if ((delta as Record)[field] === null) { + Reflect.deleteProperty(delta, field); + normalized = true; + } + } + } + + return normalized ? JSON.stringify(parsed) : sseData; +} + +export function createOpenCodeGoAdapter(source: AdapterSource, quirks?: unknown): ProviderAdapter { + const base = createOpenAICompatibleAdapter(source, quirks); + return { + ...base, + parseResponse: (sseData) => base.parseResponse(normalizeNullDeltaFields(sseData)), + }; +} diff --git a/tests/unit/inference-sources.test.ts b/tests/unit/inference-sources.test.ts index 6276c6d78..43e87f1b6 100644 --- a/tests/unit/inference-sources.test.ts +++ b/tests/unit/inference-sources.test.ts @@ -178,7 +178,8 @@ test("buildInferenceSourceForRef routes OpenCode Go models by protocol", () => { ctx, undefined, ); - expect(chat?.provider).toBe("openai-compatible"); + expect(chat?.provider).toBe("opencode-go"); + expect(chat?.quirks).toBeUndefined(); expect(chat?.baseURL).toBe("https://opencode.ai/zen/go/v1"); expect(chat?.model).toBe("kimi-k2.7-code"); From a756c8acda00a2e218cf394b30800266ad0d1660 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 1 Sep 2026 21:03:50 -0700 Subject: [PATCH 2/3] Surface provider failure diagnostics in terminals --- src/agent/agent-search.ts | 4 +- src/agent/renderer.ts | 22 ++- src/exec/runner.ts | 43 ++++-- src/inference-error-message.test.ts | 132 ++++++++++++++++-- src/inference-error-message.ts | 91 ++++++++++-- src/mcp/plugin.ts | 4 +- src/plugins/result-truncation-plugin.ts | 4 +- .../tool-result-secret-scrub-plugin.ts | 6 +- src/plugins/tool-result-secret-scrub.test.ts | 42 ++++-- src/plugins/tool-result-secret-scrub.ts | 25 +++- src/renderer.test.ts | 13 +- src/subagent/agent-fleet.ts | 8 +- .../run-resolved-provider-failure.test.ts | 106 +++++++++++++- src/subagent/run.ts | 45 ++++-- src/tui/provider-failure-attempt.test.ts | 41 ++++++ src/tui/provider-failure-attempt.ts | 31 ++++ src/tui/runner.ts | 68 +++++++-- src/tui/runtime-bridge.test.ts | 12 +- src/tui/stream-event-map.test.ts | 52 +++++++ src/tui/stream-event-map.ts | 35 ++++- tests/unit/exec/runner.test.ts | 35 +++++ tests/unit/tui/runner.test.ts | 20 ++- 22 files changed, 730 insertions(+), 109 deletions(-) create mode 100644 src/tui/provider-failure-attempt.test.ts create mode 100644 src/tui/provider-failure-attempt.ts diff --git a/src/agent/agent-search.ts b/src/agent/agent-search.ts index 093e6c3ec..4a1c6ca83 100644 --- a/src/agent/agent-search.ts +++ b/src/agent/agent-search.ts @@ -2,7 +2,7 @@ import { stringTool } from "@intx/agent"; import type { AgentTool } from "@intx/agent"; import type { ToolDefinition } from "@intx/types/runtime"; import { type } from "arktype"; -import { scrubSecretShapedToolResultContent } from "../plugins/tool-result-secret-scrub.js"; +import { scrubSecretShapedContent } from "../plugins/tool-result-secret-scrub.js"; import type { AgentProfile } from "./profiles.js"; function tokenize(text: string): string[] { @@ -86,7 +86,7 @@ export function formatAgentSearchResults(profiles: readonly AgentProfile[]): str // SCRUBBABLE_TOOLS in tool-result-secret-scrub-plugin cannot reach it. Scrub here // before the formatted string becomes a tool result (marketplace/plugin bodies may // contain secret-shaped substrings). - return scrubSecretShapedToolResultContent( + return scrubSecretShapedContent( [ "Matching agent profiles (pass id to spawn_agent(agent=...)). Full system prompt / body is included so you do not need read_file on plugin roots outside the workspace:", "", diff --git a/src/agent/renderer.ts b/src/agent/renderer.ts index 7f2c2dca3..15fdb0bcd 100644 --- a/src/agent/renderer.ts +++ b/src/agent/renderer.ts @@ -9,7 +9,10 @@ import { createSessionCostAccumulator, type TurnBillingIdentity, } from "../cost/session-cost.js"; -import { inferenceErrorMessage } from "../inference-error-message.js"; +import { + inferenceErrorMessage, + terminalProviderFailureMessage, +} from "../inference-error-message.js"; export interface Renderer { render(event: ReactorEmittedEvent): void; @@ -258,16 +261,25 @@ export function createRenderer( case "inference.error": { const err = e.data?.error as Record | undefined; const rawMessage = String(err?.message ?? e.data?.error ?? "inference error"); - const message = + const classifiedError = typeof err?.category === "string" - ? inferenceErrorMessage({ + ? { category: err.category, message: rawMessage, ...(typeof err.statusCode === "number" ? { statusCode: err.statusCode } : {}), ...(err.raw !== undefined ? { raw: err.raw } : {}), ...(typeof err.providerId === "string" ? { providerId: err.providerId } : {}), - }) - : rawMessage; + } + : undefined; + const message = + classifiedError === undefined + ? rawMessage + : classifiedError.category === "quota_exhausted" + ? inferenceErrorMessage(classifiedError) + : terminalProviderFailureMessage( + classifiedError.providerId ?? "Unknown", + classifiedError, + ); writeErrorBlock(message); break; } diff --git a/src/exec/runner.ts b/src/exec/runner.ts index 531bc9c73..7399ff37b 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -67,6 +67,7 @@ import { isResolvedProviderFailureError, terminalProviderFailureMessage, } from "../inference-error-message.js"; +import type { InferenceErrorLike } from "../inference-gateway-error.js"; import { collectToolPlugins, resolveToolPlugins } from "../plugins/tool-plugins.js"; import { expandExistingPluginMembers, @@ -132,19 +133,34 @@ export async function refreshSelectedProviderCredential(refresh: () => Promis } } +function execTerminalProviderFailureMessage( + config: Config, + diagnostic: InferenceErrorLike, +): string { + const providerId = diagnostic.providerId ?? config.providerName; + const displayLabel = + providerId === config.providerName + ? config.settings?.providers[config.providerName]?.name + : undefined; + return terminalProviderFailureMessage(providerId, diagnostic, displayLabel); +} + export function execUserFailureMessage( config: Config, err: unknown, providerFailureObserved: boolean, + providerError?: InferenceErrorLike, ): string { if (err instanceof Error && err.name === SELECTED_PROVIDER_FAILURE) { return CREDENTIAL_FAILURE_USER_MESSAGE; } + if (providerError === undefined && isResolvedProviderFailureError(err)) return err.message; if (providerFailureObserved || isResolvedProviderFailureError(err)) { - return terminalProviderFailureMessage( - config.providerName, - config.settings?.providers[config.providerName]?.name, - ); + const diagnostic = providerError ?? { + category: "fatal", + message: err instanceof Error ? err.message : String(err), + }; + return execTerminalProviderFailureMessage(config, diagnostic); } return formatCaughtError(err); } @@ -327,6 +343,7 @@ export async function runExec(config: Config): Promise { let turnsUsed = 0; let runSink: RunSink | null = null; let providerFailureObserved = false; + let providerError: InferenceErrorLike | undefined; const persist = async ( status: "running" | "done" | "failed" | "cancelled", @@ -756,8 +773,18 @@ export async function runExec(config: Config): Promise { const sink = (event: ReactorEmittedEvent): void => { if (event.type === "inference.start" || event.type === "inference.done") { providerFailureObserved = false; + providerError = undefined; } else if (event.type === "inference.error") { providerFailureObserved = true; + const error = event.data.error; + providerError = { + category: error.category, + ...(error.message !== undefined ? { message: error.message } : {}), + ...(error.statusCode !== undefined ? { statusCode: error.statusCode } : {}), + ...("providerId" in error && typeof error.providerId === "string" + ? { providerId: error.providerId } + : {}), + }; } liveSink.sink(event); cycleRecorder.handleEvent(event); @@ -883,9 +910,9 @@ export async function runExec(config: Config): Promise { (summaryStatus === "cancelled" ? "run cancelled before completion" : "run failed"); const userMessage = summaryStatus === "failed" - ? terminalProviderFailureMessage( - config.providerName, - config.settings?.providers[config.providerName]?.name, + ? execTerminalProviderFailureMessage( + config, + providerError ?? { category: "unknown", message: diagnosticMessage }, ) : diagnosticMessage; stderr.write(`Error: ${userMessage}\n`); @@ -922,7 +949,7 @@ export async function runExec(config: Config): Promise { } catch (err) { const diagnosticMessage = formatCaughtError(err); logger.error("exec failed: {error}", { error: diagnosticMessage }); - const userMessage = execUserFailureMessage(config, err, providerFailureObserved); + const userMessage = execUserFailureMessage(config, err, providerFailureObserved, providerError); stderr.write(`Error: ${userMessage}\n`); await persist("failed", { error: diagnosticMessage }); return { diff --git a/src/inference-error-message.test.ts b/src/inference-error-message.test.ts index 17969610f..32746e59a 100644 --- a/src/inference-error-message.test.ts +++ b/src/inference-error-message.test.ts @@ -94,29 +94,141 @@ describe("inferenceErrorMessage", () => { }); describe("terminalProviderFailureMessage", () => { - test("uses the selected provider display label in the terminal guidance", () => { - expect(terminalProviderFailureMessage("openai", "OpenAI")).toBe( - 'OpenAI Provider failed. Try again or switch with "/model" and select another.', + test("surfaces a retryable HTTP failure with safe retry guidance", () => { + expect( + terminalProviderFailureMessage( + "openai", + { + category: "retryable", + message: "\u001b[31mupstream\n unavailable\u001b[0m", + statusCode: 500, + }, + "OpenAI", + ), + ).toBe("OpenAI Provider failed (retryable): upstream unavailable. Try again."); + }); + + test("surfaces protocol mismatches with switch-model guidance", () => { + expect( + terminalProviderFailureMessage("custom-provider", { + category: "protocol_mismatch", + message: "response did not match the expected schema", + }), + ).toBe( + 'custom-provider Provider failed (protocol_mismatch): response did not match the expected schema. Switch models with "/model".', + ); + }); + + test("treats an HTTP 503 protocol mismatch as transient", () => { + expect( + terminalProviderFailureMessage("custom-provider", { + category: "protocol_mismatch", + message: "gateway returned HTML", + statusCode: 503, + }), + ).toBe( + "custom-provider Provider failed (protocol_mismatch): gateway returned HTML. Try again.", + ); + }); + + test("uses the canonical category for a misclassified context overflow", () => { + expect( + terminalProviderFailureMessage("custom-provider", { + category: "retryable", + message: "input exceeds the maximum context window", + statusCode: 429, + }), + ).toBe( + "custom-provider Provider failed (context_overflow): input exceeds the maximum context window. Try /clear to start fresh.", ); }); - test("falls back to the selected provider id", () => { - expect(terminalProviderFailureMessage("custom-provider")).toBe( - 'custom-provider Provider failed. Try again or switch with "/model" and select another.', + test("tells the user to log in again after a credential failure", () => { + expect( + terminalProviderFailureMessage("custom-provider", { + category: "credential_failure", + message: "HTTP 401 Unauthorized", + statusCode: 401, + }), + ).toBe( + "custom-provider Provider failed (credential_failure): HTTP 401 Unauthorized. Authentication failed — log in again.", ); }); + test.each([ + { + name: "Bearer header", + secret: "bearer-secret-token-1234567890", + diagnostic: "Authorization: Bearer bearer-secret-token-1234567890", + }, + { + name: "Basic authorization header", + secret: "dXNlcjpwYXNzd29yZA==", + diagnostic: "Authorization: Basic dXNlcjpwYXNzd29yZA==", + }, + { + name: "api_key query parameter", + secret: "query-secret-value", + diagnostic: "GET https://provider.invalid/v1?api_key=query-secret-value&model=test", + }, + { + name: "JSON credential fields", + secret: "json-secret-value", + diagnostic: + '{"api_key":"json-secret-value","password":"json-secret-value","authorization":"json-secret-value"}', + }, + ])("scrubs $name before display", ({ secret, diagnostic }) => { + const message = terminalProviderFailureMessage("custom-provider", { + category: "fatal", + message: diagnostic, + }); + + expect(message).toContain("[redacted: looks like a credential]"); + expect(message).not.toContain(secret); + }); + + test("scrubs a Bearer token split by ANSI controls", () => { + const secret = "bearer-secret-token-1234567890"; + const message = terminalProviderFailureMessage("custom-provider", { + category: "fatal", + message: "Authorization: Bearer bearer-secret-\u001b[31mtoken-1234567890\u001b[0m", + }); + + expect(message).toContain("[redacted: looks like a credential]"); + expect(message).not.toContain(secret); + expect(message).not.toContain("\u001b"); + }); + test("does not duplicate Provider in configured display labels", () => { - expect(terminalProviderFailureMessage("codex/work", "Codex Provider")).toBe( - 'Codex Provider failed. Try again or switch with "/model" and select another.', + expect( + terminalProviderFailureMessage( + "codex/work", + { category: "fatal", message: "service rejected the request" }, + "Codex Provider", + ), + ).toBe( + 'Codex Provider failed (fatal): service rejected the request. Try again or switch models with "/model".', ); }); test("uses a safe label when the provider id contains only control sequences", () => { - const message = terminalProviderFailureMessage("\u001b[31m\u001b[0m"); + const message = terminalProviderFailureMessage("\u001b[31m\u001b[0m", { + category: "fatal", + message: "request failed", + }); expect(message).toBe( - 'Unknown Provider failed. Try again or switch with "/model" and select another.', + 'Unknown Provider failed (fatal): request failed. Try again or switch models with "/model".', ); expect(message).not.toContain("\u001b"); }); + + test("bounds provider-controlled display text", () => { + const message = terminalProviderFailureMessage("custom-provider", { + category: "fatal", + message: "x".repeat(1_000), + }); + + expect(message).toContain(`${"x".repeat(239)}…`); + expect(message).not.toContain("x".repeat(241)); + }); }); diff --git a/src/inference-error-message.ts b/src/inference-error-message.ts index 772add3eb..44bd0c159 100644 --- a/src/inference-error-message.ts +++ b/src/inference-error-message.ts @@ -12,6 +12,7 @@ import { } from "./auth/codex/usage-limit-error.js"; import { codexProfileFromProviderName, isCodexProviderName } from "./config/codex-providers.js"; import { stripTerminalControlSequences } from "./util/control-char-strip.js"; +import { scrubSecretShapedContent } from "./plugins/tool-result-secret-scrub.js"; import { gatewayOverloadUserMessage, isCodexShortRateLimitInferenceError, @@ -92,29 +93,90 @@ function codexUsageLimitLine(error: InferenceErrorLike): string | undefined { return undefined; } -export function terminalProviderFailureMessage(providerId: string, displayLabel?: string): string { +const TERMINAL_DIAGNOSTIC_MAX_CHARS = 240; +const TERMINAL_PROVIDER_LABEL_MAX_CHARS = 80; + +function safeDisplayText(text: string, maxChars: number): string { + const oneLine = scrubSecretShapedContent(stripTerminalControlSequences(text)) + .replace(/\s+/g, " ") + .trim(); + return oneLine.length > maxChars ? `${oneLine.slice(0, maxChars - 1)}…` : oneLine; +} + +function terminalProviderFailureCategory(error: InferenceErrorLike): string { + const category = classifyInferenceErrorCategory(error); + return /^[a-z][a-z0-9_]*$/i.test(category) ? category : "unknown"; +} + +export function terminalProviderFailureMessage( + providerId: string, + error: InferenceErrorLike, + displayLabel?: string, +): string { const preferred = displayLabel?.trim() || providerId; - const sanitized = stripTerminalControlSequences(preferred).replace(/\s+/g, " ").trim(); - const label = (sanitized.length > 0 ? sanitized : "Unknown").replace(/\s+Provider$/i, ""); - return `${label} Provider failed. Try again or switch with "/model" and select another.`; + const sanitizedLabel = safeDisplayText(preferred, TERMINAL_PROVIDER_LABEL_MAX_CHARS); + const label = (sanitizedLabel.length > 0 ? sanitizedLabel : "Unknown").replace( + /\s+Provider$/i, + "", + ); + const category = terminalProviderFailureCategory(error); + const message = safeDisplayText(error.message ?? "", TERMINAL_DIAGNOSTIC_MAX_CHARS); + const diagnostic = message.length > 0 ? message : "inference error"; + const diagnosticSentence = /[.!?]$/.test(diagnostic) ? diagnostic : `${diagnostic}.`; + const guidance = terminalProviderFailureGuidance(error, category); + return `${label} Provider failed (${category}): ${diagnosticSentence} ${guidance}`; +} + +function terminalProviderFailureGuidance(error: InferenceErrorLike, category: string): string { + if (category === "credential_failure") return CREDENTIAL_FAILURE_USER_MESSAGE; + if (category === "context_overflow") return "Try /clear to start fresh."; + if ( + category === "retryable" || + (error.statusCode !== undefined && error.statusCode >= 500 && error.statusCode <= 599) + ) { + return "Try again."; + } + return category === "protocol_mismatch" + ? 'Switch models with "/model".' + : 'Try again or switch models with "/model".'; +} + +function terminalProviderFailureSummary( + providerId: string, + error: InferenceErrorLike, + displayLabel?: string, +): string { + const preferred = displayLabel?.trim() || providerId; + const sanitizedLabel = safeDisplayText(preferred, TERMINAL_PROVIDER_LABEL_MAX_CHARS); + const label = (sanitizedLabel.length > 0 ? sanitizedLabel : "Unknown").replace( + /\s+Provider$/i, + "", + ); + const category = terminalProviderFailureCategory(error); + return `${label} Provider failed (${category}). ${terminalProviderFailureGuidance(error, category)}`; } export type ResolvedProviderFailureError = Error & { readonly name: "ResolvedProviderFailureError"; readonly providerId: string; - readonly diagnosticMessage: string; + readonly category: string; + readonly statusCode?: number; }; export function createResolvedProviderFailureError( providerId: string, - diagnosticMessage: string, + providerError: InferenceErrorLike, displayLabel?: string, ): ResolvedProviderFailureError { - return Object.assign(new Error(terminalProviderFailureMessage(providerId, displayLabel)), { - name: "ResolvedProviderFailureError" as const, - providerId, - diagnosticMessage, - }); + return Object.assign( + new Error(terminalProviderFailureSummary(providerId, providerError, displayLabel)), + { + name: "ResolvedProviderFailureError" as const, + providerId, + category: terminalProviderFailureCategory(providerError), + ...(providerError.statusCode !== undefined ? { statusCode: providerError.statusCode } : {}), + }, + ); } export function isResolvedProviderFailureError( @@ -125,8 +187,8 @@ export function isResolvedProviderFailureError( error.name === "ResolvedProviderFailureError" && "providerId" in error && typeof error.providerId === "string" && - "diagnosticMessage" in error && - typeof error.diagnosticMessage === "string" + "category" in error && + typeof error.category === "string" ); } @@ -145,5 +207,6 @@ export function inferenceErrorMessage(error: InferenceErrorLike): string { if (codexLine !== undefined) return codexLine; } - return FRIENDLY_BY_CATEGORY[category] ?? error.message ?? "inference error"; + const fallback = safeDisplayText(error.message ?? "", TERMINAL_DIAGNOSTIC_MAX_CHARS); + return FRIENDLY_BY_CATEGORY[category] ?? (fallback.length > 0 ? fallback : "inference error"); } diff --git a/src/mcp/plugin.ts b/src/mcp/plugin.ts index 1a7302316..02c38daf2 100644 --- a/src/mcp/plugin.ts +++ b/src/mcp/plugin.ts @@ -2,7 +2,7 @@ import type { AgentTool } from "@intx/agent"; import type { ToolCall, ToolResult } from "@intx/types/runtime"; import type { PermissionGate } from "../permission/gate.js"; import { gateToolCall } from "../plugins/permission-plugin.js"; -import { scrubSecretShapedToolResultContent } from "../plugins/tool-result-secret-scrub.js"; +import { scrubSecretShapedContent } from "../plugins/tool-result-secret-scrub.js"; import { truncateToolResultContent, type SpillBlobWriter, @@ -24,7 +24,7 @@ function sanitizeMcpResultContent( content: string, spill?: { callId: string; writeBlob: SpillBlobWriter; contextDir?: string }, ): Promise { - return truncateToolResultContent(scrubSecretShapedToolResultContent(content), undefined, spill); + return truncateToolResultContent(scrubSecretShapedContent(content), undefined, spill); } // Convert a connected client's tools into AgentTools for the dynamic runner used diff --git a/src/plugins/result-truncation-plugin.ts b/src/plugins/result-truncation-plugin.ts index 561abe011..2e255e25a 100644 --- a/src/plugins/result-truncation-plugin.ts +++ b/src/plugins/result-truncation-plugin.ts @@ -5,7 +5,7 @@ import { toolOutputAbsolutePath, type MaterializedToolResult, } from "./tool-result-materialize.js"; -import { scrubSecretShapedToolResultContent } from "./tool-result-secret-scrub.js"; +import { scrubSecretShapedContent } from "./tool-result-secret-scrub.js"; const TRUNCATABLE_TOOLS = new Set(["read_file", "grep", "run_shell", "search_files", "web_fetch"]); @@ -108,7 +108,7 @@ async function spillAndTruncate( spill?: TruncationSpillOptions, ): Promise { const { contentType } = materialized; - const text = scrubSecretShapedToolResultContent(materialized.text); + const text = scrubSecretShapedContent(materialized.text); if (text.length <= maxChars) return text; if (spill === undefined) { diff --git a/src/plugins/tool-result-secret-scrub-plugin.ts b/src/plugins/tool-result-secret-scrub-plugin.ts index 5524dea24..a45d6eddd 100644 --- a/src/plugins/tool-result-secret-scrub-plugin.ts +++ b/src/plugins/tool-result-secret-scrub-plugin.ts @@ -1,5 +1,5 @@ import type { ToolPlugin } from "@intx/tools-posix"; -import { scrubSecretShapedToolResultContent } from "./tool-result-secret-scrub.js"; +import { scrubSecretShapedContent } from "./tool-result-secret-scrub.js"; // Posix-middleware scrub path only. search_agents is listed for future unified // scrubbing if it ever rides this middleware; live scrub for profile bodies is in @@ -20,14 +20,14 @@ export function toolResultSecretScrubPlugin(): ToolPlugin { if (!SCRUBBABLE_TOOLS.has(call.name) || result.isError) return result; if (typeof result.content === "string") { - const scrubbed = scrubSecretShapedToolResultContent(result.content); + const scrubbed = scrubSecretShapedContent(result.content); if (scrubbed === result.content) return result; return { ...result, content: scrubbed }; } if (result.content !== null && typeof result.content === "object") { const serialized = JSON.stringify(result.content); - const scrubbed = scrubSecretShapedToolResultContent(serialized); + const scrubbed = scrubSecretShapedContent(serialized); if (scrubbed === serialized) return result; return { ...result, content: scrubbed }; } diff --git a/src/plugins/tool-result-secret-scrub.test.ts b/src/plugins/tool-result-secret-scrub.test.ts index 43368fe0e..0a84cab92 100644 --- a/src/plugins/tool-result-secret-scrub.test.ts +++ b/src/plugins/tool-result-secret-scrub.test.ts @@ -1,15 +1,13 @@ import { describe, test, expect } from "bun:test"; -import { - CREDENTIAL_REDACTION, - scrubSecretShapedToolResultContent, -} from "./tool-result-secret-scrub.js"; +import { CREDENTIAL_REDACTION, scrubSecretShapedContent } from "./tool-result-secret-scrub.js"; import { toolResultSecretScrubPlugin } from "./tool-result-secret-scrub-plugin.js"; +import { resultTruncationPlugin } from "./result-truncation-plugin.js"; import type { ToolCall, ToolResult } from "@intx/types/runtime"; -describe("scrubSecretShapedToolResultContent", () => { +describe("scrubSecretShapedContent", () => { test("redacts grep-surfaced .env assignment", () => { const text = "./app/.env:3:API_KEY=sk-live-abc123xyz789012345678"; - const out = scrubSecretShapedToolResultContent(text); + const out = scrubSecretShapedContent(text); expect(out).toContain(`API_KEY=${CREDENTIAL_REDACTION}`); expect(out).not.toContain("sk-live-abc123"); }); @@ -19,15 +17,23 @@ describe("scrubSecretShapedToolResultContent", () => { MIIEpAIBAAKCAQEA7 -----END RSA PRIVATE KEY-----`; const text = `wrote key:\n${pem}\n`; - const out = scrubSecretShapedToolResultContent(text); + const out = scrubSecretShapedContent(text); expect(out).toBe(`wrote key:\n${CREDENTIAL_REDACTION}\n`); expect(out).not.toContain("MIIEpAIBAAKCAQEA7"); }); + test("is idempotent for redacted query parameters", () => { + const text = "GET https://provider.invalid/v1?api_key=plain-value&model=test"; + const once = scrubSecretShapedContent(text); + + expect(scrubSecretShapedContent(once)).toBe(once); + expect(once).toBe(`GET https://provider.invalid/v1?api_key=${CREDENTIAL_REDACTION}&model=test`); + }); + test("passes benign documentation mentioning API keys", () => { const text = "See docs/authentication.md for how API keys are issued and rotated. No secrets in this paragraph."; - expect(scrubSecretShapedToolResultContent(text)).toBe(text); + expect(scrubSecretShapedContent(text)).toBe(text); }); }); @@ -47,6 +53,26 @@ describe("toolResultSecretScrubPlugin", () => { expect(result.content).not.toContain("supersecretvalue"); }); + test("preserves query redaction through the long-result middleware chain", async () => { + const content = + "GET https://provider.invalid/v1?api_key=plain-value&model=test\n" + "x".repeat(11_000); + const scrub = toolResultSecretScrubPlugin(); + const truncate = resultTruncationPlugin(); + if (scrub.middleware === undefined || truncate.middleware === undefined) { + throw new Error("expected middleware plugins"); + } + const handler = truncate.middleware(scrub.middleware(next(content))); + + const result = await handler( + { id: "c-long", name: "grep", arguments: { pattern: "api_key" } }, + new AbortController().signal, + ); + + if (typeof result.content !== "string") throw new Error("expected text tool result"); + expect(result.content).toContain(`api_key=${CREDENTIAL_REDACTION}&model=test`); + expect(result.content.match(/\[redacted: looks like a credential\]/g)).toHaveLength(1); + }); + // search_agents is listed in SCRUBBABLE_TOOLS for future unified scrubbing, but // it is not on the posix middleware path today. Live scrub is in // formatAgentSearchResults — see agent-search.test.ts. This case only documents diff --git a/src/plugins/tool-result-secret-scrub.ts b/src/plugins/tool-result-secret-scrub.ts index 172ee93f8..a8fac5f48 100644 --- a/src/plugins/tool-result-secret-scrub.ts +++ b/src/plugins/tool-result-secret-scrub.ts @@ -1,6 +1,6 @@ -// Scrub secret-shaped substrings from tool result text before it enters the -// transcript. Complements secret-guard (path denylist) for secrets that surface -// via directory-wide grep/shell output. +// Scrub secret-shaped substrings before untrusted text reaches a transcript or +// terminal. Complements secret-guard (path denylist) for secrets that surface +// in tool output and upstream diagnostics. export const CREDENTIAL_REDACTION = "[redacted: looks like a credential]"; @@ -15,8 +15,15 @@ const API_KEY_PATTERNS: RegExp[] = [ /\bxox[baprs]-[0-9A-Za-z-]{10,}\b/g, /\bAKIA[0-9A-Z]{16}\b/g, /\bBearer\s+[a-zA-Z0-9_\-.]{20,}\b/g, + /\bBasic\s+[a-zA-Z0-9+/]{8,}={0,2}/g, ]; +const SECRET_QUERY_PARAMETER = + /([?&](?:api[_-]?key|access[_-]?token|token|password|secret|credential)=)(?!\[redacted: looks like a credential\])[^&#\s]+/gi; + +const JSON_CREDENTIAL_FIELD = + /("(?:api[_-]?key|access[_-]?token|token|password|secret|credential|authorization)"\s*:\s*")([^"\r\n]+)(")/gi; + // Grep/shell lines often look like path:line:KEY=value const ENV_ASSIGNMENT = /(?:^|:)([A-Z][A-Z0-9_]+)=([^\n]+)/gm; @@ -28,7 +35,7 @@ function replaceAll(text: string, pattern: RegExp, replacement: string): string return text.replace(pattern, replacement); } -export function scrubSecretShapedToolResultContent(text: string): string { +export function scrubSecretShapedContent(text: string): string { let result = text; result = replaceAll(result, PEM_BLOCK, CREDENTIAL_REDACTION); @@ -37,6 +44,16 @@ export function scrubSecretShapedToolResultContent(text: string): string { result = replaceAll(result, pattern, CREDENTIAL_REDACTION); } + result = result.replace( + SECRET_QUERY_PARAMETER, + (_match, prefix: string) => `${prefix}${CREDENTIAL_REDACTION}`, + ); + result = result.replace( + JSON_CREDENTIAL_FIELD, + (_match, prefix: string, _value: string, suffix: string) => + `${prefix}${CREDENTIAL_REDACTION}${suffix}`, + ); + result = result.replace(ENV_ASSIGNMENT, (match, key: string) => { if (!isSecretEnvKey(key)) return match; const prefixEnd = match.lastIndexOf(key); diff --git a/src/renderer.test.ts b/src/renderer.test.ts index 72305ec74..cd7a8ede0 100644 --- a/src/renderer.test.ts +++ b/src/renderer.test.ts @@ -194,12 +194,16 @@ describe("renderer — submit_output / reactor.done journal block", () => { }); describe("renderer — error blocks", () => { - test("inference.error writes error block in red to stdout", () => { + test("inference.error writes an actionable provider diagnostic in red to stdout", () => { const cap = captureOutput(); const renderer = createRenderer(Date.now()); renderer.render( event("inference.error", { - error: { category: "timeout", message: "request timed out" }, + error: { + category: "protocol_mismatch", + message: "\u001b[31mresponse\n shape changed\u001b[0m", + providerId: "openai", + }, partial: {}, }), ); @@ -207,7 +211,10 @@ describe("renderer — error blocks", () => { const out = cap.stdout.join(""); expect(out).toContain("error"); expect(out).toContain("\x1b[31m"); // red - expect(out).toContain("Request timed out"); + expect(out).toContain( + 'openai Provider failed (protocol_mismatch): response shape changed. Switch models with "/model".', + ); + expect(out).not.toContain("\u001b[31mresponse"); }); test("inference.error surfaces Codex usage_limit_reached with reset ETA", () => { diff --git a/src/subagent/agent-fleet.ts b/src/subagent/agent-fleet.ts index 555691fa6..5bf3c45d6 100644 --- a/src/subagent/agent-fleet.ts +++ b/src/subagent/agent-fleet.ts @@ -1094,14 +1094,10 @@ export function createSpawnAgentTool(deps: AgentFleetDeps): AgentTool { deps.sessions.settleRun(session.id); return; } - const diagnosticMessage = isResolvedProviderFailureError(err) - ? err.diagnosticMessage - : err instanceof Error - ? err.message - : String(err); const isProviderFailure = isResolvedProviderFailureError(err); + const diagnosticMessage = err instanceof Error ? err.message : String(err); const authMessage = formatSubAgentSpawnAuthFailureMessage(description, err); - const failReason = authMessage ?? (isProviderFailure ? err.message : diagnosticMessage); + const failReason = authMessage ?? diagnosticMessage; if (isProviderFailure || providerFailureObserved) { deps.fleetRecords.markProviderFailure(session.id); } diff --git a/src/subagent/run-resolved-provider-failure.test.ts b/src/subagent/run-resolved-provider-failure.test.ts index 2163e5d95..e943fb210 100644 --- a/src/subagent/run-resolved-provider-failure.test.ts +++ b/src/subagent/run-resolved-provider-failure.test.ts @@ -10,14 +10,17 @@ import { isResolvedProviderFailureError, type ResolvedProviderFailureError, } from "../inference-error-message.js"; +import type { InferenceErrorLike } from "../inference-gateway-error.js"; import { createPermissionGate } from "../permission/gate.js"; import { createFleetMailbox, createSpawnAgentTool, createWaitAgentsTool } from "./agent-fleet.js"; import { createSubAgentSessionStore } from "./session-store.js"; import type { RunSubAgentParams, RunSubAgentResult } from "./types.js"; -const RAW_DIAGNOSTIC = "POST https://provider.invalid returned secret response body"; +const RAW_DIAGNOSTIC = + "\u001b[31mPOST https://provider.invalid returned\n secret response body\u001b[0m"; +const NORMALIZED_DIAGNOSTIC = "POST https://provider.invalid returned secret response body"; const SAFE_MESSAGE = - 'test-provider Provider failed. Try again or switch with "/model" and select another.'; + 'test-provider Provider failed (fatal). Try again or switch models with "/model".'; const provider = { providerName: "test-provider", baseURL: "http://localhost", @@ -33,9 +36,15 @@ type Run = (params: RunSubAgentParams) => Promise; async function withResolvedProviderRun( callback: (run: Run, cwd: string, observed: ReactorEmittedEvent[]) => Promise, + providerError: InferenceErrorLike = { category: "fatal", message: RAW_DIAGNOSTIC }, + sendFailure?: Error, ): Promise { const cwd = await mkdtemp(join(tmpdir(), "resolved-provider-failure-")); const observed: ReactorEmittedEvent[] = []; + let inferenceErrorConsumed: (() => void) | undefined; + const inferenceErrorWasConsumed = new Promise((resolve) => { + inferenceErrorConsumed = resolve; + }); try { return await withMockedModuleDuring( import.meta.resolve("../agent/live-tool-dispatch.js"), @@ -44,6 +53,10 @@ async function withResolvedProviderRun( createAgentWithLiveToolDispatch: async () => ({ send: async () => { + if (sendFailure !== undefined) { + await inferenceErrorWasConsumed; + throw sendFailure; + } await new Promise((resolve) => queueMicrotask(resolve)); return { reply: RAW_DIAGNOSTIC, turn: { role: "assistant", content: [] } }; }, @@ -58,10 +71,11 @@ async function withResolvedProviderRun( type: "inference.error", seq: 2, data: { - error: { category: "fatal", message: RAW_DIAGNOSTIC }, + error: providerError, partial: { text: "" }, }, } as unknown as ReactorEmittedEvent; + inferenceErrorConsumed?.(); yield { type: "connector.reply", seq: 3, @@ -125,8 +139,14 @@ describe("resolved sub-agent provider failures", () => { expect(isResolvedProviderFailureError(caught)).toBe(true); expect((caught as ResolvedProviderFailureError).message).toBe(SAFE_MESSAGE); - expect((caught as ResolvedProviderFailureError).diagnosticMessage).toBe(RAW_DIAGNOSTIC); - expect(observed.some((event) => JSON.stringify(event).includes(RAW_DIAGNOSTIC))).toBe(true); + expect((caught as ResolvedProviderFailureError).category).toBe("fatal"); + expect(JSON.stringify(caught)).not.toContain(RAW_DIAGNOSTIC); + expect(JSON.stringify(caught)).not.toContain(NORMALIZED_DIAGNOSTIC); + expect( + observed.some( + (event) => event.type === "inference.error" && event.data.error.message === RAW_DIAGNOSTIC, + ), + ).toBe(true); }); test("split spawn_agent and wait_agents return only the safe message", async () => { @@ -168,7 +188,83 @@ describe("resolved sub-agent provider failures", () => { provider_failure: true, }); expect(String(waited.content)).not.toContain(RAW_DIAGNOSTIC); + expect(String(waited.content)).not.toContain(NORMALIZED_DIAGNOSTIC); expect(sessions.get(spawnPayload.agent_id)?.error).toBe(SAFE_MESSAGE); + expect(sessions.get(spawnPayload.agent_id)?.error).not.toContain(RAW_DIAGNOSTIC); + expect(sessions.get(spawnPayload.agent_id)?.error).not.toContain(NORMALIZED_DIAGNOSTIC); }); }); + + test("a rejected send after inference.error stores only safe classified failure text", async () => { + const providerError = { + category: "retryable", + message: RAW_DIAGNOSTIC, + statusCode: 502, + } satisfies InferenceErrorLike; + await withResolvedProviderRun( + async (run, cwd) => { + const sessions = createSubAgentSessionStore(); + const fleetRecords = createFleetMailbox(sessions); + const deps = { + ...runParams(cwd), + getWorkdirBase: () => join(cwd, ".ctx"), + sessions, + fleetRecords, + run, + }; + const spawned = await callTool(createSpawnAgentTool(deps), "spawn_agent", { + description: "rejected provider failure", + prompt: "trigger it", + intent: "explore", + }); + const spawnPayload = JSON.parse(String(spawned.content)) as { agent_id?: unknown }; + if (typeof spawnPayload.agent_id !== "string") throw new Error("missing agent_id"); + const waited = await callTool( + createWaitAgentsTool({ sessions, fleetRecords }), + "wait_agents", + { targets: [spawnPayload.agent_id], timeout_ms: 5000 }, + ); + const safeFailure = "test-provider Provider failed (retryable). Try again."; + + expect(String(waited.content)).toContain(safeFailure); + expect(String(waited.content)).not.toContain(RAW_DIAGNOSTIC); + expect(String(waited.content)).not.toContain(NORMALIZED_DIAGNOSTIC); + expect(sessions.get(spawnPayload.agent_id)?.error).toBe(safeFailure); + expect(sessions.get(spawnPayload.agent_id)?.error).not.toContain(RAW_DIAGNOSTIC); + expect(sessions.get(spawnPayload.agent_id)?.error).not.toContain(NORMALIZED_DIAGNOSTIC); + }, + providerError, + new Error(RAW_DIAGNOSTIC), + ); + }); + + test.each([ + { + error: { category: "retryable", message: RAW_DIAGNOSTIC, statusCode: 500 }, + expected: "test-provider Provider failed (retryable). Try again.", + }, + { + error: { category: "protocol_mismatch", message: RAW_DIAGNOSTIC }, + expected: 'test-provider Provider failed (protocol_mismatch). Switch models with "/model".', + }, + ] satisfies { error: InferenceErrorLike; expected: string }[])( + "preserves $error.category guidance without exposing its diagnostic", + async ({ error, expected }) => { + const caught = await withResolvedProviderRun(async (run, cwd) => { + try { + await run(runParams(cwd)); + } catch (failure) { + return failure; + } + throw new Error("expected runSubAgent to reject"); + }, error); + + expect(isResolvedProviderFailureError(caught)).toBe(true); + expect((caught as ResolvedProviderFailureError).category).toBe(error.category); + expect((caught as ResolvedProviderFailureError).statusCode).toBe(error.statusCode); + expect((caught as ResolvedProviderFailureError).message).toBe(expected); + expect((caught as ResolvedProviderFailureError).message).not.toContain(RAW_DIAGNOSTIC); + expect((caught as ResolvedProviderFailureError).message).not.toContain(NORMALIZED_DIAGNOSTIC); + }, + ); }); diff --git a/src/subagent/run.ts b/src/subagent/run.ts index 95e0e96b2..a8b6a2672 100644 --- a/src/subagent/run.ts +++ b/src/subagent/run.ts @@ -77,6 +77,7 @@ import { createCycleTextRecorder } from "../session/stream-journal.js"; import { onTurnBoundary } from "../agent/reactor-events.js"; import { refreshInferenceSourceBundle } from "./refresh-inference-source.js"; import { createResolvedProviderFailureError } from "../inference-error-message.js"; +import type { InferenceErrorLike } from "../inference-gateway-error.js"; import { createRunEventSettlement } from "./run-event-settlement.js"; import type { CapabilityFilter } from "../agent/profiles.js"; @@ -877,7 +878,7 @@ async function runSubAgentInner( // salvage Findings keep substantive mid-run text, not only the final cycle. const TURN_PROSE_CAP = 12_000; let accumulatedProse = ""; - let terminalProviderDiagnostic: string | undefined; + let terminalProviderError: InferenceErrorLike | undefined; // Thrash paths from tool.start so mid-tool cancel still lists files touched. let thrashState = EMPTY_THRASH_STATE; const withTelemetry = (result: RunSubAgentResult): RunSubAgentResult => ({ @@ -912,16 +913,19 @@ async function runSubAgentInner( } if (event.type === "inference.start") { settlementState.latestModel = event.data.model; - terminalProviderDiagnostic = undefined; + terminalProviderError = undefined; } if (event.type === "inference.done") { settlementState.latestModel = event.data.source.model; - terminalProviderDiagnostic = undefined; + terminalProviderError = undefined; } if (event.type === "inference.error") { - const message = (event.data as { error?: { message?: unknown } }).error?.message; - terminalProviderDiagnostic = - typeof message === "string" && message.length > 0 ? message : "inference error"; + const error = event.data.error; + terminalProviderError = { + category: error.category, + ...(error.message !== undefined ? { message: error.message } : {}), + ...(error.statusCode !== undefined ? { statusCode: error.statusCode } : {}), + }; } if (onTurnBoundary(event)) { telemetryRollup.turn_count += 1; @@ -984,6 +988,23 @@ async function runSubAgentInner( } }; + const sendWithProviderFailure = async ( + message: string, + options: { signal: AbortSignal }, + ): ReturnType["send"]> => { + try { + return await sendAndSettle(message, options); + } catch (cause) { + if (terminalProviderError !== undefined) { + throw createResolvedProviderFailureError( + params.provider.providerName, + terminalProviderError, + ); + } + throw cause; + } + }; + // Aborting the send signal only rejects the promise; the child reactor keeps // running until close() (same hard-stop rule as the parent in runner.ts). closeOnAbort = (): void => { @@ -1045,11 +1066,11 @@ async function runSubAgentInner( // agent object, reusing full context rather than starting fresh. const followup = async (message: string): Promise => { interruptController = new AbortController(); - const result = await sendAndSettle(message, { signal: sendAbortSignal() }); - if (terminalProviderDiagnostic !== undefined) { + const result = await sendWithProviderFailure(message, { signal: sendAbortSignal() }); + if (terminalProviderError !== undefined) { throw createResolvedProviderFailureError( params.provider.providerName, - terminalProviderDiagnostic, + terminalProviderError, ); } return result.reply.trim().length > 0 @@ -1107,11 +1128,11 @@ async function runSubAgentInner( params.catalog, ); agent.setSources(fresh.sources, fresh.defaultSource); - const result = await sendAndSettle(fullPrompt, sendOpts); - if (terminalProviderDiagnostic !== undefined) { + const result = await sendWithProviderFailure(fullPrompt, sendOpts); + if (terminalProviderError !== undefined) { throw createResolvedProviderFailureError( params.provider.providerName, - terminalProviderDiagnostic, + terminalProviderError, ); } // A successful non-empty reply must not be clobbered by a late cancel that diff --git a/src/tui/provider-failure-attempt.test.ts b/src/tui/provider-failure-attempt.test.ts new file mode 100644 index 000000000..fa061ae4c --- /dev/null +++ b/src/tui/provider-failure-attempt.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, test } from "bun:test"; +import { createProviderFailureAttemptTracker } from "./provider-failure-attempt.js"; + +describe("provider failure attempt tracker", () => { + test("does not carry a settled attempt's diagnostic into the next attempt", () => { + const tracker = createProviderFailureAttemptTracker(); + const first = tracker.begin(); + tracker.observe({ + providerId: "openai", + category: "protocol_mismatch", + message: "stale diagnostic", + }); + tracker.settle(first); + + const second = tracker.begin(); + + expect(second).toEqual({ observed: false, error: undefined }); + }); + + test("settling an older attempt does not clear the active attempt", () => { + const tracker = createProviderFailureAttemptTracker(); + const first = tracker.begin(); + const second = tracker.begin(); + + tracker.settle(first); + tracker.observe({ + providerId: "xai/work", + category: "credential_failure", + message: "HTTP 401", + }); + + expect(second).toEqual({ + observed: true, + error: { + providerId: "xai/work", + category: "credential_failure", + message: "HTTP 401", + }, + }); + }); +}); diff --git a/src/tui/provider-failure-attempt.ts b/src/tui/provider-failure-attempt.ts new file mode 100644 index 000000000..5bc82cd3b --- /dev/null +++ b/src/tui/provider-failure-attempt.ts @@ -0,0 +1,31 @@ +import type { InferenceErrorLike } from "../inference-gateway-error.js"; + +export interface ProviderFailureAttempt { + observed: boolean; + error: InferenceErrorLike | undefined; +} + +export function createProviderFailureAttemptTracker() { + let active: ProviderFailureAttempt | undefined; + + return { + begin(): ProviderFailureAttempt { + const attempt = { observed: false, error: undefined }; + active = attempt; + return attempt; + }, + reset(): void { + if (active === undefined) return; + active.observed = false; + active.error = undefined; + }, + observe(error: InferenceErrorLike): void { + if (active === undefined) return; + active.observed = true; + active.error = error; + }, + settle(attempt: ProviderFailureAttempt): void { + if (active === attempt) active = undefined; + }, + }; +} diff --git a/src/tui/runner.ts b/src/tui/runner.ts index 0cd082fc1..5dc849d66 100644 --- a/src/tui/runner.ts +++ b/src/tui/runner.ts @@ -159,6 +159,10 @@ import type { } from "@intx/types/runtime"; import { OPERATOR_ORIGINATED_FLAG } from "../agent/message-provenance.js"; import { createSessionOperationQueue } from "./session-operation-queue.js"; +import { + createProviderFailureAttemptTracker, + type ProviderFailureAttempt, +} from "./provider-failure-attempt.js"; import { setAgentSourceUnlessClosed } from "./agent-source-sync.js"; import { createChatDirector, hydrateTasksFromTurns } from "../agent/director.js"; import { onTurnBoundary } from "../agent/reactor-events.js"; @@ -221,6 +225,7 @@ import { isResolvedProviderFailureError, terminalProviderFailureMessage, } from "../inference-error-message.js"; +import type { InferenceErrorLike } from "../inference-gateway-error.js"; import { listPathSuggestions } from "./components/at-mention/list.js"; import { imageAttachmentFromPath, type PendingImageAttachment } from "./image-attachments.js"; import { appendSentMessage, loadSentMessages } from "../session/sent-messages.js"; @@ -604,9 +609,10 @@ export function setUpCommandRegistry( export function surfaceTerminalProviderFailure( shell: Parameters[0], providerId: string, + error: InferenceErrorLike, displayLabel?: string, ): void { - surfaceSystemNotice(shell, terminalProviderFailureMessage(providerId, displayLabel)); + surfaceSystemNotice(shell, terminalProviderFailureMessage(providerId, error, displayLabel)); } export interface InferenceAttemptIdentity { @@ -619,6 +625,7 @@ export function tuiSendFailureMessage( failureKind: "auth" | "error", providerFailureObserved: boolean, attempt: InferenceAttemptIdentity, + providerError?: InferenceErrorLike, ): string { if (failureKind === "auth") { return CREDENTIAL_FAILURE_USER_MESSAGE; @@ -628,7 +635,12 @@ export function tuiSendFailureMessage( } const providerId = isResolvedProviderFailureError(error) ? error.providerId : attempt.providerId; const displayLabel = providerId === attempt.providerId ? attempt.displayLabel : undefined; - return terminalProviderFailureMessage(providerId, displayLabel); + if (providerError === undefined && isResolvedProviderFailureError(error)) return error.message; + const diagnostic = providerError ?? { + category: "fatal", + message: error instanceof Error ? error.message : String(error), + }; + return terminalProviderFailureMessage(providerId, diagnostic, displayLabel); } export async function runTUI(initialConfig: Config): Promise { @@ -1666,13 +1678,21 @@ export async function runTUI(initialConfig: Config): Promise { // keeps the in-flight cycle's text so an errored or interrupted turn leaves // its partial output in partial.jsonl instead of vanishing. const cycleRecorder = createCycleTextRecorder(() => workdir); - let providerFailureObserved = false; + const providerFailureAttempts = createProviderFailureAttemptTracker(); flushPartialOnCrash = () => cycleRecorder.dispose("crashed").then(() => undefined); const streamSink = (event: Parameters[0]): void => { if (event.type === "inference.start" || event.type === "inference.done") { - providerFailureObserved = false; + providerFailureAttempts.reset(); } else if (event.type === "inference.error") { - providerFailureObserved = true; + const error = event.data.error; + providerFailureAttempts.observe({ + category: error.category, + message: error.message, + ...(error.statusCode !== undefined ? { statusCode: error.statusCode } : {}), + ...("providerId" in error && typeof error.providerId === "string" + ? { providerId: error.providerId } + : {}), + }); } runSink.sink(event); cycleRecorder.handleEvent(event); @@ -2147,13 +2167,25 @@ export async function runTUI(initialConfig: Config): Promise { approvalPersistNotice.notify = systemNotice; /** Settle the shell after a rejected send so the run does not look live. */ - const handleSendFailure = (err: unknown, attempt: InferenceAttemptIdentity): void => { + const handleSendFailure = ( + err: unknown, + attempt: InferenceAttemptIdentity, + providerFailure: ProviderFailureAttempt, + ): void => { const failure = classifyAgentSendFailure(err, sendAborted, isCodexAuthError, isXaiAuthError); captureAuthFailure(getTelemetry(), failure); if (!shouldSettleUiAfterSendFailure(failure.kind)) return; if (failure.kind === "abort") return; recordRunError(err); - systemNotice(tuiSendFailureMessage(err, failure.kind, providerFailureObserved, attempt)); + systemNotice( + tuiSendFailureMessage( + err, + failure.kind, + providerFailure.observed, + attempt, + providerFailure.error, + ), + ); setShellRunState(host.shell, "idle"); }; @@ -2167,10 +2199,13 @@ export async function runTUI(initialConfig: Config): Promise { const sendWithAttemptIdentity = async (message: InboundMessage): Promise => { const attempt = currentAttemptIdentity(); + const providerFailure = providerFailureAttempts.begin(); try { await agentProxy.send(message); } catch (error) { - handleSendFailure(error, attempt); + handleSendFailure(error, attempt, providerFailure); + } finally { + providerFailureAttempts.settle(providerFailure); } }; @@ -2270,7 +2305,10 @@ export async function runTUI(initialConfig: Config): Promise { dispatchCommand: (name, args) => dispatchCommand(name, args), sendPrompt: (text, attachments) => { void sendUserPrompt(text, attachments ?? []).catch((error: unknown) => { - handleSendFailure(error, currentAttemptIdentity()); + handleSendFailure(error, currentAttemptIdentity(), { + observed: false, + error: undefined, + }); }); }, onPromptSubmitted: () => { @@ -2355,7 +2393,11 @@ export async function runTUI(initialConfig: Config): Promise { }); }, captureGeneration: deliveryGeneration.capture, - onFailure: (error) => handleSendFailure(error, currentAttemptIdentity()), + onFailure: (error) => + handleSendFailure(error, currentAttemptIdentity(), { + observed: false, + error: undefined, + }), }), parentCycleLive: () => host.bridge.parentCycleLive, deliverSteer: createLiveSteerDeliver({ @@ -2366,7 +2408,11 @@ export async function runTUI(initialConfig: Config): Promise { agentProxy.deliver(userInboundMessage(text, pending)); }, captureGeneration: deliveryGeneration.capture, - onFailure: (error) => handleSendFailure(error, currentAttemptIdentity()), + onFailure: (error) => + handleSendFailure(error, currentAttemptIdentity(), { + observed: false, + error: undefined, + }), }), }), // Consent by proceeding requires the disclosure to be on screen before the diff --git a/src/tui/runtime-bridge.test.ts b/src/tui/runtime-bridge.test.ts index c86cdcca6..9a5dae2db 100644 --- a/src/tui/runtime-bridge.test.ts +++ b/src/tui/runtime-bridge.test.ts @@ -723,7 +723,7 @@ describe("attachSessionBridge", () => { }); describe("failed sends", () => { - test("a resolved terminal provider failure replaces the raw reply and resets", async () => { + test("a resolved terminal provider failure is render-only and resets", async () => { await withTestRenderer( async (h) => { const shell = createAppShell(h.renderer, { @@ -731,8 +731,9 @@ describe("failed sends", () => { wireKeys: false, run: "idle", }); - const bridge = attachSessionBridge(shell, createRecordingPort()); - const rawDiagnostic = "upstream 401: secret response body"; + const port = createRecordingPort(); + const bridge = attachSessionBridge(shell, port); + const rawDiagnostic = "\u001b[31mupstream 401:\n secret response body\u001b[0m"; const normalReply = "The next request worked."; try { bridge.setInferenceProviderId("codex/default", "Codex"); @@ -746,10 +747,11 @@ describe("failed sends", () => { bridge.handle({ type: "connector.reply", data: { content: normalReply } }); const safeMessage = - 'Codex Provider failed. Try again or switch with "/model" and select another.'; + "Codex Provider failed (credential_failure): upstream 401: secret response body. Authentication failed — log in again."; expect(shell.streamLog.filter((row) => row.text === safeMessage)).toHaveLength(1); expect(shell.streamLog.filter((row) => row.text === normalReply)).toHaveLength(1); - expect(shell.streamLog.map((row) => row.text).join("\n")).not.toContain(rawDiagnostic); + expect(shell.streamLog.map((row) => row.text).join("\n")).not.toContain("\u001b"); + expect(port.calls).toEqual([]); } finally { bridge.dispose(); shell.dispose(); diff --git a/src/tui/stream-event-map.test.ts b/src/tui/stream-event-map.test.ts index 0d6c34bee..ff5f4af62 100644 --- a/src/tui/stream-event-map.test.ts +++ b/src/tui/stream-event-map.test.ts @@ -344,4 +344,56 @@ describe("inference.error text", () => { expect(out).toEqual([]); expect(JSON.stringify(out)).not.toContain(rawDiagnostic); }); + + test("terminal replies surface the classified sanitized provider diagnostic", () => { + const out = mapProductionSequence( + [ + { + type: "inference.error", + data: { + error: { + category: "retryable", + message: "\u001b[31mupstream\n unavailable\u001b[0m", + statusCode: 500, + }, + }, + }, + { type: "connector.reply", data: { content: "generic director reply" } }, + ], + createStreamMapContext({ providerId: "openai", providerLabel: "OpenAI" }), + ); + + expect(out).toEqual([ + { + type: "assistant", + text: "OpenAI Provider failed (retryable): upstream unavailable. Try again.", + }, + ]); + }); + + test("an explicit failing provider overrides the selected provider identity", () => { + const out = mapProductionSequence( + [ + { + type: "inference.error", + data: { + error: { + providerId: "xai/work", + category: "credential_failure", + message: "HTTP 401", + }, + }, + }, + { type: "connector.reply", data: { content: "generic director reply" } }, + ], + createStreamMapContext({ providerId: "openai", providerLabel: "OpenAI" }), + ); + + expect(out).toEqual([ + { + type: "assistant", + text: "xai/work Provider failed (credential_failure): HTTP 401. Authentication failed — log in again.", + }, + ]); + }); }); diff --git a/src/tui/stream-event-map.ts b/src/tui/stream-event-map.ts index 1773de085..a34b4644e 100644 --- a/src/tui/stream-event-map.ts +++ b/src/tui/stream-event-map.ts @@ -10,6 +10,7 @@ import { stripTerminalControlSequences, } from "../util/control-char-strip.js"; import { terminalProviderFailureMessage } from "../inference-error-message.js"; +import type { InferenceErrorLike } from "../inference-gateway-error.js"; import type { RunState } from "./session-queue.js"; /** Canonical inbound events the bridge understands (fixtures + mapped reactor). */ @@ -122,8 +123,9 @@ export interface StreamMapContext { */ providerId?: string; providerLabel?: string; - /** Raw diagnostics stay on the reactor event; only this marker reaches reply mapping. */ + /** Classified diagnostics are held only for render-time terminal presentation. */ pendingProviderFailure: boolean; + pendingProviderError: InferenceErrorLike | undefined; } export function createStreamMapContext(opts?: { @@ -140,6 +142,7 @@ export function createStreamMapContext(opts?: { attemptCallIds: new Set(), errorRollbackArmed: false, pendingProviderFailure: false, + pendingProviderError: undefined, ...(opts?.providerId !== undefined ? { providerId: opts.providerId } : {}), ...(opts?.providerLabel !== undefined ? { providerLabel: opts.providerLabel } : {}), }; @@ -345,6 +348,7 @@ function mapEvent( ctx.attemptArmed = true; ctx.attemptCallIds = new Set(ctx.callIdToName.keys()); ctx.pendingProviderFailure = false; + ctx.pendingProviderError = undefined; } return [ ...(recovered ? [ATTEMPT_ROLLBACK] : []), @@ -356,7 +360,10 @@ function mapEvent( case "inference.done": // Cycle settled: disarm so a pre-commit retry belonging to the *next* // cycle cannot retract this one's rows. - if (ctx) ctx.pendingProviderFailure = false; + if (ctx) { + ctx.pendingProviderFailure = false; + ctx.pendingProviderError = undefined; + } return disarmAttempt(ctx); case "inference.retry": { @@ -462,10 +469,20 @@ function mapEvent( if (ctx?.pendingProviderFailure === true) { ctx.pendingProviderFailure = false; ctx.hadTextDelta = false; + const error = ctx.pendingProviderError ?? { + category: "unknown", + message: "inference error", + }; + ctx.pendingProviderError = undefined; + const providerId = error.providerId ?? ctx.providerId ?? "Unknown"; + const providerLabel = + error.providerId === undefined || error.providerId === ctx.providerId + ? ctx.providerLabel + : undefined; return [ { type: "assistant", - text: terminalProviderFailureMessage(ctx.providerId ?? "Unknown", ctx.providerLabel), + text: terminalProviderFailureMessage(providerId, error, providerLabel), }, ]; } @@ -482,6 +499,7 @@ function mapEvent( if (ctx) { ctx.hadTextDelta = false; ctx.pendingProviderFailure = false; + ctx.pendingProviderError = undefined; } return [...disarmAttempt(ctx), { type: "run", state: "idle" }, { type: "tool.boundary" }]; @@ -504,7 +522,16 @@ function mapEvent( ctx.attemptArmed = false; ctx.errorRollbackArmed = true; } - if (ctx) ctx.pendingProviderFailure = true; + if (ctx) { + ctx.pendingProviderFailure = true; + const rawError = asRecord(data.error); + ctx.pendingProviderError = { + category: typeof rawError?.category === "string" ? rawError.category : "unknown", + message: typeof rawError?.message === "string" ? rawError.message : "inference error", + ...(typeof rawError?.statusCode === "number" ? { statusCode: rawError.statusCode } : {}), + ...(typeof rawError?.providerId === "string" ? { providerId: rawError.providerId } : {}), + }; + } return []; } diff --git a/tests/unit/exec/runner.test.ts b/tests/unit/exec/runner.test.ts index f6250c4b2..4685c4cc0 100644 --- a/tests/unit/exec/runner.test.ts +++ b/tests/unit/exec/runner.test.ts @@ -62,6 +62,41 @@ describe("selected provider refresh failures", () => { expect(userMessage).not.toContain(rawDiagnostic); } }); + + test("terminal provider failures use the shared classified diagnostic", () => { + const config = { + ...bareConfig("hello"), + providerName: "codex/work", + settings: { providers: { "codex/work": { name: "Codex" } } }, + } as unknown as Config; + + expect( + execUserFailureMessage(config, new Error("send failed"), true, { + category: "protocol_mismatch", + message: "\u001b[31mresponse\n shape changed\u001b[0m", + }), + ).toBe( + 'Codex Provider failed (protocol_mismatch): response shape changed. Switch models with "/model".', + ); + }); + + test("terminal provider failures prefer an explicit failing provider", () => { + const config = { + ...bareConfig("hello"), + providerName: "openai", + settings: { providers: { openai: { name: "OpenAI" } } }, + } as unknown as Config; + + expect( + execUserFailureMessage(config, new Error("send failed"), true, { + providerId: "xai/work", + category: "credential_failure", + message: "HTTP 401", + }), + ).toBe( + "xai/work Provider failed (credential_failure): HTTP 401. Authentication failed — log in again.", + ); + }); }); describe("runExec", () => { diff --git a/tests/unit/tui/runner.test.ts b/tests/unit/tui/runner.test.ts index 667558e84..325377ae6 100644 --- a/tests/unit/tui/runner.test.ts +++ b/tests/unit/tui/runner.test.ts @@ -51,11 +51,21 @@ test("TUI send failures keep non-provider errors distinct", () => { test("TUI send failures retain the in-flight provider identity across model switches", () => { expect( - tuiSendFailureMessage(new Error("raw provider body"), "error", true, { - providerId: "codex/work", - displayLabel: "Codex", - }), - ).toBe('Codex Provider failed. Try again or switch with "/model" and select another.'); + tuiSendFailureMessage( + new Error("send failed"), + "error", + true, + { + providerId: "codex/work", + displayLabel: "Codex", + }, + { + category: "retryable", + message: "\u001b[31mupstream\n unavailable\u001b[0m", + statusCode: 500, + }, + ), + ).toBe("Codex Provider failed (retryable): upstream unavailable. Try again."); }); test("TUI auth failures tell the user to log in again instead of switching models", () => { From ab2cebe257e4c9be420c6b5547676bd2aa91c739 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Tue, 1 Sep 2026 22:09:48 -0700 Subject: [PATCH 3/3] Fix terminal diagnostic attempt attribution --- src/agent/retry-policy.test.ts | 27 ++++++++++ src/inference-gateway-error.ts | 35 ++++++++++++ src/tui/provider-failure-attempt.test.ts | 38 +++++++++++-- src/tui/provider-failure-attempt.ts | 69 ++++++++++++++++++++++-- src/tui/runner.ts | 66 +++++++++++++++-------- src/tui/stream-event-map.test.ts | 54 +++++++++++++++++++ src/tui/stream-event-map.ts | 55 ++++++++++++++++--- tests/unit/tui/runner.test.ts | 18 +++++++ 8 files changed, 322 insertions(+), 40 deletions(-) diff --git a/src/agent/retry-policy.test.ts b/src/agent/retry-policy.test.ts index 2c686fe80..3fb5a72c0 100644 --- a/src/agent/retry-policy.test.ts +++ b/src/agent/retry-policy.test.ts @@ -18,6 +18,33 @@ describe("createCorbitsRetryPolicy", () => { expect(decision).toEqual({ kind: "retry", delayMs: 500 }); }); + test("aborts an OpenCode Go malformed streamed SSE schema response", async () => { + const policy = createCorbitsRetryPolicy({ providerId: "opencode-go/corbits" }); + const decision = await policy({ + attempt: 1, + elapsedMs: 0, + error: { + category: "protocol_mismatch", + message: + "openai parseResponse: SSE chunk failed schema validation: choices0.delta.role must be a string (was null)", + }, + }); + expect(decision).toEqual({ kind: "abort" }); + }); + + test("aborts a generic non-overload protocol mismatch", async () => { + const policy = createCorbitsRetryPolicy(); + const decision = await policy({ + attempt: 1, + elapsedMs: 0, + error: { + category: "protocol_mismatch", + message: "response did not match the provider protocol", + }, + }); + expect(decision).toEqual({ kind: "abort" }); + }); + test("aborts long-window quota exhaustion", async () => { const policy = createCorbitsRetryPolicy(); const decision = await policy({ diff --git a/src/inference-gateway-error.ts b/src/inference-gateway-error.ts index a0598b841..818432ab4 100644 --- a/src/inference-gateway-error.ts +++ b/src/inference-gateway-error.ts @@ -378,6 +378,41 @@ export function normalizeInferenceErrorForRetry( }; } +function isInferenceErrorCategory(category: string): category is InferenceError["category"] { + return ( + category === "fatal" || + category === "retryable" || + category === "context_overflow" || + category === "credential_failure" || + category === "quota_exhausted" || + category === "aborted" || + category === "timeout" || + category === "protocol_mismatch" + ); +} + +/** Normalize a provider diagnostic once for terminal presentation without dropping context fields. */ +export function normalizeInferenceErrorForTerminal( + error: InferenceErrorLike, + fallbackProviderId: string, +): InferenceErrorLike { + const contextual = { + ...error, + providerId: error.providerId ?? fallbackProviderId, + }; + if (!isInferenceErrorCategory(contextual.category)) return contextual; + const normalized = normalizeInferenceErrorForRetry({ + ...contextual, + category: contextual.category, + message: contextual.message ?? "Inference error", + }); + return { + ...contextual, + ...normalized, + providerId: contextual.providerId, + }; +} + export function gatewayOverloadUserMessage(error: InferenceErrorLike): string { if (!isGatewayOverloadInferenceError(error)) return error.message ?? "Inference error"; return GATEWAY_OVERLOAD_USER_MESSAGE; diff --git a/src/tui/provider-failure-attempt.test.ts b/src/tui/provider-failure-attempt.test.ts index fa061ae4c..2540143ae 100644 --- a/src/tui/provider-failure-attempt.test.ts +++ b/src/tui/provider-failure-attempt.test.ts @@ -14,28 +14,58 @@ describe("provider failure attempt tracker", () => { const second = tracker.begin(); - expect(second).toEqual({ observed: false, error: undefined }); + expect(second).toEqual({ observed: false, presented: false, error: undefined }); }); - test("settling an older attempt does not clear the active attempt", () => { + test("a settled send remains current until its terminal stream event is consumed", () => { const tracker = createProviderFailureAttemptTracker(); const first = tracker.begin(); + tracker.sendSettled(first); const second = tracker.begin(); - tracker.settle(first); tracker.observe({ providerId: "xai/work", category: "credential_failure", message: "HTTP 401", }); + const reply = tracker.consumeConnectorReply(); - expect(second).toEqual({ + expect(reply).toEqual({ attempt: first, suppressPresentation: false }); + expect(first).toEqual({ observed: true, + presented: true, error: { providerId: "xai/work", category: "credential_failure", message: "HTTP 401", }, }); + expect(second).toEqual({ observed: false, presented: false, error: undefined }); + expect(tracker.current()).toBe(first); + tracker.consumeTerminal(); + expect(tracker.current()).toBe(second); + }); + + test("a rejected-send fallback advances when the next message starts and suppresses a late reply", () => { + const tracker = createProviderFailureAttemptTracker(); + const first = tracker.begin(); + tracker.markPresented(first); + tracker.sendSettled(first); + const second = tracker.begin(); + + tracker.advanceToNextMessage(); + expect(tracker.current()).toBe(second); + + tracker.observe({ + providerId: "openai", + category: "fatal", + message: "second failure", + }); + tracker.markPresented(second); + + expect(tracker.consumeConnectorReply()).toEqual({ + attempt: second, + suppressPresentation: true, + }); }); }); diff --git a/src/tui/provider-failure-attempt.ts b/src/tui/provider-failure-attempt.ts index 5bc82cd3b..c817f8e4f 100644 --- a/src/tui/provider-failure-attempt.ts +++ b/src/tui/provider-failure-attempt.ts @@ -2,30 +2,89 @@ import type { InferenceErrorLike } from "../inference-gateway-error.js"; export interface ProviderFailureAttempt { observed: boolean; + presented: boolean; error: InferenceErrorLike | undefined; + providerId?: string; + displayLabel?: string; +} + +const presentationSuppressedEvents = new WeakSet(); + +export function suppressProviderFailurePresentation(event: T): T { + presentationSuppressedEvents.add(event); + return event; +} + +export function isProviderFailurePresentationSuppressed(event: object): boolean { + return presentationSuppressedEvents.has(event); } export function createProviderFailureAttemptTracker() { - let active: ProviderFailureAttempt | undefined; + const attempts: ProviderFailureAttempt[] = []; + const settledSends = new Set(); + const consumedTerminals = new Set(); + + const remove = (attempt: ProviderFailureAttempt): void => { + const index = attempts.indexOf(attempt); + if (index !== -1) attempts.splice(index, 1); + settledSends.delete(attempt); + consumedTerminals.delete(attempt); + }; return { - begin(): ProviderFailureAttempt { - const attempt = { observed: false, error: undefined }; - active = attempt; + begin(identity?: { providerId: string; displayLabel?: string }): ProviderFailureAttempt { + const attempt = { + observed: false, + presented: false, + error: undefined, + ...(identity !== undefined ? identity : {}), + }; + attempts.push(attempt); return attempt; }, + current(): ProviderFailureAttempt | undefined { + return attempts[0]; + }, reset(): void { + const active = attempts[0]; if (active === undefined) return; active.observed = false; + active.presented = false; active.error = undefined; }, observe(error: InferenceErrorLike): void { + const active = attempts[0]; if (active === undefined) return; active.observed = true; active.error = error; }, + markPresented(attempt: ProviderFailureAttempt): void { + attempt.presented = true; + }, + consumeConnectorReply(): + { attempt: ProviderFailureAttempt; suppressPresentation: boolean } | undefined { + const active = attempts[0]; + if (active === undefined) return undefined; + const suppressPresentation = active.presented; + if (active.observed) active.presented = true; + return { attempt: active, suppressPresentation }; + }, + consumeTerminal(): void { + const active = attempts[0]; + if (active === undefined) return; + consumedTerminals.add(active); + if (settledSends.has(active)) remove(active); + }, + sendSettled(attempt: ProviderFailureAttempt): void { + settledSends.add(attempt); + if (consumedTerminals.has(attempt)) remove(attempt); + }, + advanceToNextMessage(): void { + const active = attempts[0]; + if (attempts.length > 1 && active !== undefined && settledSends.has(active)) remove(active); + }, settle(attempt: ProviderFailureAttempt): void { - if (active === attempt) active = undefined; + remove(attempt); }, }; } diff --git a/src/tui/runner.ts b/src/tui/runner.ts index 5dc849d66..7c9976167 100644 --- a/src/tui/runner.ts +++ b/src/tui/runner.ts @@ -161,6 +161,7 @@ import { OPERATOR_ORIGINATED_FLAG } from "../agent/message-provenance.js"; import { createSessionOperationQueue } from "./session-operation-queue.js"; import { createProviderFailureAttemptTracker, + suppressProviderFailurePresentation, type ProviderFailureAttempt, } from "./provider-failure-attempt.js"; import { setAgentSourceUnlessClosed } from "./agent-source-sync.js"; @@ -225,7 +226,10 @@ import { isResolvedProviderFailureError, terminalProviderFailureMessage, } from "../inference-error-message.js"; -import type { InferenceErrorLike } from "../inference-gateway-error.js"; +import { + normalizeInferenceErrorForTerminal, + type InferenceErrorLike, +} from "../inference-gateway-error.js"; import { listPathSuggestions } from "./components/at-mention/list.js"; import { imageAttachmentFromPath, type PendingImageAttachment } from "./image-attachments.js"; import { appendSentMessage, loadSentMessages } from "../session/sent-messages.js"; @@ -633,7 +637,9 @@ export function tuiSendFailureMessage( if (!providerFailureObserved && !isResolvedProviderFailureError(error)) { return error instanceof Error ? error.message : String(error); } - const providerId = isResolvedProviderFailureError(error) ? error.providerId : attempt.providerId; + const providerId = + providerError?.providerId ?? + (isResolvedProviderFailureError(error) ? error.providerId : attempt.providerId); const displayLabel = providerId === attempt.providerId ? attempt.displayLabel : undefined; if (providerError === undefined && isResolvedProviderFailureError(error)) return error.message; const diagnostic = providerError ?? { @@ -1681,20 +1687,28 @@ export async function runTUI(initialConfig: Config): Promise { const providerFailureAttempts = createProviderFailureAttemptTracker(); flushPartialOnCrash = () => cycleRecorder.dispose("crashed").then(() => undefined); const streamSink = (event: Parameters[0]): void => { - if (event.type === "inference.start" || event.type === "inference.done") { + let eventForSink = event; + if (event.type === "message.received") { + providerFailureAttempts.advanceToNextMessage(); + } else if (event.type === "inference.start" || event.type === "inference.done") { providerFailureAttempts.reset(); } else if (event.type === "inference.error") { const error = event.data.error; - providerFailureAttempts.observe({ - category: error.category, - message: error.message, - ...(error.statusCode !== undefined ? { statusCode: error.statusCode } : {}), - ...("providerId" in error && typeof error.providerId === "string" - ? { providerId: error.providerId } - : {}), - }); + const executingAttempt = providerFailureAttempts.current(); + const providerId = + "providerId" in error && typeof error.providerId === "string" + ? error.providerId + : (executingAttempt?.providerId ?? config.providerName); + providerFailureAttempts.observe(normalizeInferenceErrorForTerminal(error, providerId)); + } else if (event.type === "connector.reply") { + const reply = providerFailureAttempts.consumeConnectorReply(); + if (reply?.suppressPresentation === true) { + eventForSink = suppressProviderFailurePresentation(event); + } + } else if (event.type === "message.run.ended") { + providerFailureAttempts.consumeTerminal(); } - runSink.sink(event); + runSink.sink(eventForSink); cycleRecorder.handleEvent(event); if (onTurnBoundary(event)) { sessionCost.addTurn(event.data.usage, billingIdentityFromSource(event.data.source)); @@ -2177,15 +2191,18 @@ export async function runTUI(initialConfig: Config): Promise { if (!shouldSettleUiAfterSendFailure(failure.kind)) return; if (failure.kind === "abort") return; recordRunError(err); - systemNotice( - tuiSendFailureMessage( - err, - failure.kind, - providerFailure.observed, - attempt, - providerFailure.error, - ), - ); + if (!providerFailure.presented) { + systemNotice( + tuiSendFailureMessage( + err, + failure.kind, + providerFailure.observed, + attempt, + providerFailure.error, + ), + ); + providerFailureAttempts.markPresented(providerFailure); + } setShellRunState(host.shell, "idle"); }; @@ -2199,13 +2216,13 @@ export async function runTUI(initialConfig: Config): Promise { const sendWithAttemptIdentity = async (message: InboundMessage): Promise => { const attempt = currentAttemptIdentity(); - const providerFailure = providerFailureAttempts.begin(); + const providerFailure = providerFailureAttempts.begin(attempt); try { await agentProxy.send(message); } catch (error) { handleSendFailure(error, attempt, providerFailure); } finally { - providerFailureAttempts.settle(providerFailure); + providerFailureAttempts.sendSettled(providerFailure); } }; @@ -2307,6 +2324,7 @@ export async function runTUI(initialConfig: Config): Promise { void sendUserPrompt(text, attachments ?? []).catch((error: unknown) => { handleSendFailure(error, currentAttemptIdentity(), { observed: false, + presented: false, error: undefined, }); }); @@ -2396,6 +2414,7 @@ export async function runTUI(initialConfig: Config): Promise { onFailure: (error) => handleSendFailure(error, currentAttemptIdentity(), { observed: false, + presented: false, error: undefined, }), }), @@ -2411,6 +2430,7 @@ export async function runTUI(initialConfig: Config): Promise { onFailure: (error) => handleSendFailure(error, currentAttemptIdentity(), { observed: false, + presented: false, error: undefined, }), }), diff --git a/src/tui/stream-event-map.test.ts b/src/tui/stream-event-map.test.ts index ff5f4af62..d82369424 100644 --- a/src/tui/stream-event-map.test.ts +++ b/src/tui/stream-event-map.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test"; +import { suppressProviderFailurePresentation } from "./provider-failure-attempt.js"; import { createStreamMapContext, mapProductionEvent, @@ -371,6 +372,59 @@ describe("inference.error text", () => { ]); }); + test("does not repeat a fallback diagnostic on the terminal connector reply", () => { + const ctx = createStreamMapContext({ providerId: "openai", providerLabel: "OpenAI" }); + mapProductionEvent( + { + type: "inference.error", + data: { error: { category: "retryable", message: "upstream unavailable" } }, + }, + ctx, + ); + const reply = suppressProviderFailurePresentation({ + type: "connector.reply", + data: { content: "generic director reply" }, + }); + + expect(mapProductionEvent(reply, ctx)).toEqual([]); + expect(ctx.pendingProviderFailure).toBe(false); + }); + + test("latches the executing provider when selection changes mid-cycle", () => { + const ctx = createStreamMapContext({ providerId: "xai/work", providerLabel: "Work" }); + mapProductionEvent({ type: "inference.start", data: { model: "grok" } }, ctx); + + ctx.providerId = "openai"; + ctx.providerLabel = "OpenAI"; + mapProductionEvent( + { + type: "inference.error", + data: { + error: { + category: "quota_exhausted", + message: "Too many requests", + statusCode: 429, + raw: { requestId: "req-1" }, + }, + }, + }, + ctx, + ); + + expect(ctx.pendingProviderError).toMatchObject({ + providerId: "xai/work", + category: "retryable", + statusCode: 429, + raw: { requestId: "req-1" }, + }); + expect( + mapProductionEvent({ type: "connector.reply", data: { content: "generic reply" } }, ctx), + ).toContainEqual({ + type: "assistant", + text: "Work Provider failed (retryable): Rate limited — retrying…. Try again.", + }); + }); + test("an explicit failing provider overrides the selected provider identity", () => { const out = mapProductionSequence( [ diff --git a/src/tui/stream-event-map.ts b/src/tui/stream-event-map.ts index a34b4644e..c1c87d246 100644 --- a/src/tui/stream-event-map.ts +++ b/src/tui/stream-event-map.ts @@ -10,7 +10,11 @@ import { stripTerminalControlSequences, } from "../util/control-char-strip.js"; import { terminalProviderFailureMessage } from "../inference-error-message.js"; -import type { InferenceErrorLike } from "../inference-gateway-error.js"; +import { + normalizeInferenceErrorForTerminal, + type InferenceErrorLike, +} from "../inference-gateway-error.js"; +import { isProviderFailurePresentationSuppressed } from "./provider-failure-attempt.js"; import type { RunState } from "./session-queue.js"; /** Canonical inbound events the bridge understands (fixtures + mapped reactor). */ @@ -123,6 +127,9 @@ export interface StreamMapContext { */ providerId?: string; providerLabel?: string; + /** Provider selection captured when the active inference cycle started. */ + inferenceProviderId?: string; + inferenceProviderLabel?: string; /** Classified diagnostics are held only for render-time terminal presentation. */ pendingProviderFailure: boolean; pendingProviderError: InferenceErrorLike | undefined; @@ -347,6 +354,16 @@ function mapEvent( ctx.hadTextDelta = false; ctx.attemptArmed = true; ctx.attemptCallIds = new Set(ctx.callIdToName.keys()); + if (ctx.providerId === undefined) { + delete ctx.inferenceProviderId; + } else { + ctx.inferenceProviderId = ctx.providerId; + } + if (ctx.providerLabel === undefined) { + delete ctx.inferenceProviderLabel; + } else { + ctx.inferenceProviderLabel = ctx.providerLabel; + } ctx.pendingProviderFailure = false; ctx.pendingProviderError = undefined; } @@ -363,6 +380,8 @@ function mapEvent( if (ctx) { ctx.pendingProviderFailure = false; ctx.pendingProviderError = undefined; + delete ctx.inferenceProviderId; + delete ctx.inferenceProviderLabel; } return disarmAttempt(ctx); @@ -466,6 +485,14 @@ function mapEvent( case "connector.reply": { const content = typeof data.content === "string" ? data.content : ""; + if (isProviderFailurePresentationSuppressed(event)) { + if (ctx) { + ctx.pendingProviderFailure = false; + ctx.pendingProviderError = undefined; + ctx.hadTextDelta = false; + } + return []; + } if (ctx?.pendingProviderFailure === true) { ctx.pendingProviderFailure = false; ctx.hadTextDelta = false; @@ -474,10 +501,12 @@ function mapEvent( message: "inference error", }; ctx.pendingProviderError = undefined; - const providerId = error.providerId ?? ctx.providerId ?? "Unknown"; + const providerId = + error.providerId ?? ctx.inferenceProviderId ?? ctx.providerId ?? "Unknown"; + const selectedProviderId = ctx.inferenceProviderId ?? ctx.providerId; const providerLabel = - error.providerId === undefined || error.providerId === ctx.providerId - ? ctx.providerLabel + error.providerId === undefined || error.providerId === selectedProviderId + ? (ctx.inferenceProviderLabel ?? ctx.providerLabel) : undefined; return [ { @@ -515,9 +544,9 @@ function mapEvent( case "inference.error": { // Hand the armed boundary to the next event rather than disarming: a - // committed retry start must still retract the failed attempt. Terminal - // failures are surfaced once by the runner after agent.send rejects; - // provider diagnostics remain in the event stream for observability only. + // committed retry start must still retract the failed attempt. Hold the + // provider diagnostic for connector.reply, where terminal presentation is + // coordinated with any rejected-send fallback from the runner. if (ctx?.attemptArmed === true) { ctx.attemptArmed = false; ctx.errorRollbackArmed = true; @@ -525,12 +554,22 @@ function mapEvent( if (ctx) { ctx.pendingProviderFailure = true; const rawError = asRecord(data.error); - ctx.pendingProviderError = { + const error = { category: typeof rawError?.category === "string" ? rawError.category : "unknown", message: typeof rawError?.message === "string" ? rawError.message : "inference error", ...(typeof rawError?.statusCode === "number" ? { statusCode: rawError.statusCode } : {}), + ...(rawError?.raw !== undefined ? { raw: rawError.raw } : {}), + ...(typeof rawError?.retryAfterMs === "number" + ? { retryAfterMs: rawError.retryAfterMs } + : {}), + ...(typeof rawError?.requestURL === "string" ? { requestURL: rawError.requestURL } : {}), + ...(typeof rawError?.opencodeGo === "boolean" ? { opencodeGo: rawError.opencodeGo } : {}), ...(typeof rawError?.providerId === "string" ? { providerId: rawError.providerId } : {}), }; + ctx.pendingProviderError = normalizeInferenceErrorForTerminal( + error, + ctx.inferenceProviderId ?? ctx.providerId ?? "Unknown", + ); } return []; } diff --git a/tests/unit/tui/runner.test.ts b/tests/unit/tui/runner.test.ts index 325377ae6..31d9fe790 100644 --- a/tests/unit/tui/runner.test.ts +++ b/tests/unit/tui/runner.test.ts @@ -68,6 +68,24 @@ test("TUI send failures retain the in-flight provider identity across model swit ).toBe("Codex Provider failed (retryable): upstream unavailable. Try again."); }); +test("TUI send failures prefer an explicitly reported provider", () => { + expect( + tuiSendFailureMessage( + new Error("send failed"), + "error", + true, + { providerId: "codex/work", displayLabel: "Codex" }, + { + providerId: "xai/work", + category: "credential_failure", + message: "HTTP 401", + }, + ), + ).toBe( + "xai/work Provider failed (credential_failure): HTTP 401. Authentication failed — log in again.", + ); +}); + test("TUI auth failures tell the user to log in again instead of switching models", () => { expect( tuiSendFailureMessage(new Error("401 refresh token rejected"), "auth", false, {