diff --git a/src/tui/runtime-bridge.test.ts b/src/tui/runtime-bridge.test.ts index c77270e1b..471686862 100644 --- a/src/tui/runtime-bridge.test.ts +++ b/src/tui/runtime-bridge.test.ts @@ -814,6 +814,332 @@ describe("committed inference retry", () => { }); }); +describe("same-turn failover after inference.error", () => { + const errorRows = (shell: { + streamLog: readonly { role: string; meta?: string; text: string }[]; + }) => shell.streamLog.filter((r) => r.meta === "error").map((r) => r.text); + + test("a recovered quota error does not stay in the transcript", async () => { + await withTestRenderer( + async (h) => { + const shell = createAppShell(h.renderer, { + terminal: { columns: 80, rows: 24 }, + wireKeys: false, + run: "idle", + }); + const bridge = attachSessionBridge(shell, createRecordingPort()); + try { + for (const event of [ + { type: "inference.start", data: {} }, + { + type: "inference.error", + data: { + error: { + category: "quota_exhausted", + message: "The usage limit has been reached", + statusCode: 429, + }, + }, + }, + { type: "inference.start", data: {} }, + { type: "inference.text.delta", data: { token: "recovered" } }, + { type: "inference.done", data: {} }, + { type: "reactor.done", data: {} }, + ] as const) { + bridge.handle(event); + } + + expect(errorRows(shell)).toEqual([]); + expect(shell.streamLog.map((r) => r.text).join("\n")).toContain("recovered"); + } finally { + bridge.dispose(); + shell.dispose(); + } + }, + { width: 80, height: 24 }, + ); + }); + + test("a recovered credential error does not stay in the transcript", async () => { + await withTestRenderer( + async (h) => { + const shell = createAppShell(h.renderer, { + terminal: { columns: 80, rows: 24 }, + wireKeys: false, + run: "idle", + }); + const bridge = attachSessionBridge(shell, createRecordingPort()); + try { + for (const event of [ + { type: "inference.start", data: {} }, + { + type: "inference.error", + data: { + error: { category: "credential_failure", message: "Forbidden", statusCode: 403 }, + }, + }, + { type: "inference.start", data: {} }, + { type: "inference.text.delta", data: { token: "recovered" } }, + { type: "inference.done", data: {} }, + { type: "reactor.done", data: {} }, + ] as const) { + bridge.handle(event); + } + + expect(errorRows(shell)).toEqual([]); + expect(shell.streamLog.map((r) => r.text).join("\n")).toContain("recovered"); + } finally { + bridge.dispose(); + shell.dispose(); + } + }, + { width: 80, height: 24 }, + ); + }); + + test("an echoed auto-retry prompt does not expire recovery", async () => { + await withTestRenderer( + async (h) => { + const shell = createAppShell(h.renderer, { + terminal: { columns: 80, rows: 24 }, + wireKeys: false, + run: "idle", + }); + const port = createRecordingPort(); + const bridge = attachSessionBridge(shell, port); + try { + bridge.submit("retry this", "immediate"); + bridge.handle({ type: "message.received", data: { message: { content: "retry this" } } }); + bridge.handle({ type: "inference.start", data: {} }); + bridge.handle({ + type: "inference.error", + data: { + error: { + category: "quota_exhausted", + message: "The usage limit has been reached", + statusCode: 429, + }, + }, + }); + bridge.submit("retry this", "immediate"); + bridge.handle({ type: "message.received", data: { message: { content: "retry this" } } }); + bridge.handle({ type: "inference.start", data: {} }); + bridge.handle({ type: "inference.text.delta", data: { token: "recovered" } }); + bridge.handle({ type: "inference.done", data: {} }); + bridge.handle({ type: "reactor.done", data: {} }); + + expect(errorRows(shell)).toEqual([]); + expect(shell.streamLog.map((r) => r.text).join("\n")).toContain("recovered"); + // The replay duplicates the operator's prompt; rollback drops the copy. + expect(shell.streamLog.filter((r) => r.role === "user").map((r) => r.text)).toEqual([ + "retry this", + ]); + // Same-turn failover, not an operator stop — recovery must not borrow interrupt. + expect(port.calls.some((c) => c.op === "interrupt")).toBe(false); + expect(shell.streamLog.some((r) => r.meta === "stop")).toBe(false); + } finally { + bridge.dispose(); + shell.dispose(); + } + }, + { width: 80, height: 24 }, + ); + }); + + test("a steer echo at a tool boundary opens a new thinking row", async () => { + await withTestRenderer( + async (h) => { + const shell = createAppShell(h.renderer, { + terminal: { columns: 80, rows: 24 }, + wireKeys: false, + run: "idle", + }); + const bridge = attachSessionBridge(shell, createRecordingPort()); + try { + bridge.submit("first prompt", "immediate"); + bridge.handle({ + type: "message.received", + data: { message: { content: "first prompt" } }, + }); + bridge.handle({ type: "inference.start", data: {} }); + bridge.handle({ type: "inference.thinking.delta", data: { token: "planning" } }); + bridge.handle({ + type: "inference.tool_call.end", + data: { name: "run_shell", callId: "c1", arguments: "{}" }, + }); + bridge.handle({ type: "inference.done", data: {} }); + bridge.submit("steer this", "steer"); + bridge.handle({ + type: "tool.done", + data: { result: { callId: "c1", content: "ok", isError: false } }, + }); + bridge.handle({ type: "message.received", data: { message: { content: "steer this" } } }); + bridge.handle({ type: "inference.start", data: {} }); + bridge.handle({ type: "inference.thinking.delta", data: { token: "after steer" } }); + bridge.handle({ type: "inference.text.delta", data: { token: "done" } }); + + const rows = shell.streamLog.map((r) => `${r.meta ?? r.role}:${r.text}`); + expect(rows.indexOf("thinking:planning")).toBeGreaterThan(-1); + expect(rows.indexOf("steering:steer this")).toBeGreaterThan( + rows.indexOf("thinking:planning"), + ); + expect(rows.indexOf("thinking:after steer")).toBeGreaterThan( + rows.indexOf("steering:steer this"), + ); + expect(shell.streamLog.filter((r) => r.meta === "thinking")).toHaveLength(2); + } finally { + bridge.dispose(); + shell.dispose(); + } + }, + { width: 80, height: 24 }, + ); + }); + + test("interrupt then a new prompt keeps the prompt and the classified error", async () => { + await withTestRenderer( + async (h) => { + const shell = createAppShell(h.renderer, { + terminal: { columns: 80, rows: 24 }, + wireKeys: false, + run: "idle", + }); + const port = createRecordingPort(); + const bridge = attachSessionBridge(shell, port); + try { + bridge.handle({ type: "inference.start", data: {} }); + bridge.handle({ + type: "inference.error", + data: { + error: { category: "credential_failure", message: "Forbidden", statusCode: 403 }, + }, + }); + bridge.interrupt(); + bridge.submit("next prompt", "immediate"); + bridge.handle({ + type: "message.received", + data: { message: { content: "next prompt" } }, + }); + bridge.handle({ type: "inference.start", data: {} }); + + const text = shell.streamLog.map((r) => r.text).join("\n"); + expect(text).toContain("next prompt"); + expect(errorRows(shell)).toContain("Session expired — re-authenticating…"); + } finally { + bridge.dispose(); + shell.dispose(); + } + }, + { width: 80, height: 24 }, + ); + }); + + test("a queued steer row survives failover rollback", async () => { + await withTestRenderer( + async (h) => { + const shell = createAppShell(h.renderer, { + terminal: { columns: 80, rows: 24 }, + wireKeys: false, + run: "idle", + }); + const bridge = attachSessionBridge(shell, createRecordingPort()); + try { + bridge.handle({ type: "inference.start", data: {} }); + bridge.handle({ + type: "inference.error", + data: { + error: { category: "credential_failure", message: "Forbidden", statusCode: 403 }, + }, + }); + bridge.submit("steer this", "steer"); + bridge.handle({ type: "inference.start", data: {} }); + bridge.handle({ type: "inference.text.delta", data: { token: "recovered" } }); + bridge.handle({ type: "inference.done", data: {} }); + bridge.handle({ type: "reactor.done", data: {} }); + + const text = shell.streamLog.map((r) => r.text).join("\n"); + expect(errorRows(shell)).toEqual([]); + expect(text).toContain("recovered"); + expect(text).toContain("steer this"); + } finally { + bridge.dispose(); + shell.dispose(); + } + }, + { width: 80, height: 24 }, + ); + }); + + test("reinject interrupt keeps the prompt and the classified error", async () => { + await withTestRenderer( + async (h) => { + const shell = createAppShell(h.renderer, { + terminal: { columns: 80, rows: 24 }, + wireKeys: false, + run: "idle", + }); + const bridge = attachSessionBridge(shell, createRecordingPort()); + try { + bridge.handle({ type: "inference.start", data: {} }); + bridge.handle({ + type: "inference.error", + data: { + error: { category: "credential_failure", message: "Forbidden", statusCode: 403 }, + }, + }); + bridge.submit("restart from here", "reinject"); + bridge.handle({ type: "inference.start", data: {} }); + bridge.handle({ type: "inference.text.delta", data: { token: "recovered" } }); + bridge.handle({ type: "inference.done", data: {} }); + bridge.handle({ type: "reactor.done", data: {} }); + + const text = shell.streamLog.map((r) => r.text).join("\n"); + expect(text).toContain("restart from here"); + expect(text).toContain("stop — restarting from your message"); + expect(errorRows(shell)).toContain("Session expired — re-authenticating…"); + } finally { + bridge.dispose(); + shell.dispose(); + } + }, + { width: 80, height: 24 }, + ); + }); + + test("a terminal inference.error with no recovery still surfaces", async () => { + await withTestRenderer( + async (h) => { + const shell = createAppShell(h.renderer, { + terminal: { columns: 80, rows: 24 }, + wireKeys: false, + run: "idle", + }); + const bridge = attachSessionBridge(shell, createRecordingPort()); + try { + for (const event of [ + { type: "inference.start", data: {} }, + { + type: "inference.error", + data: { + error: { category: "credential_failure", message: "Forbidden", statusCode: 403 }, + }, + }, + { type: "reactor.error", data: { error: "failed" } }, + ] as const) { + bridge.handle(event); + } + + expect(errorRows(shell)).toContain("Session expired — re-authenticating…"); + } finally { + bridge.dispose(); + shell.dispose(); + } + }, + { width: 80, height: 24 }, + ); + }); +}); + describe("parallel sub-agent dispatch on the live session bridge", () => { // The live main-session path tracks a call's row by callId in its own map // (applyToolCall/applyToolResult), independent of tool-rows.ts's name-based diff --git a/src/tui/runtime-bridge.ts b/src/tui/runtime-bridge.ts index 6dd61305b..fe9945f89 100644 --- a/src/tui/runtime-bridge.ts +++ b/src/tui/runtime-bridge.ts @@ -417,6 +417,24 @@ function consumeEcho(bag: BridgeBag, text: string): boolean { return true; } +function messageReceivedContent(event: { readonly data?: unknown }): string | undefined { + const data = event.data; + if (data === null || typeof data !== "object" || Array.isArray(data)) return undefined; + const message = (data as { readonly message?: unknown }).message; + if (message === null || typeof message !== "object" || Array.isArray(message)) return undefined; + const content = (message as { readonly content?: unknown }).content; + return typeof content === "string" ? content : undefined; +} + +function consumePendingEchoEvent( + bag: BridgeBag, + event: { readonly type: string; readonly data?: unknown }, +): boolean { + if (event.type !== "message.received") return false; + const content = messageReceivedContent(event); + return content !== undefined && consumeEcho(bag, content); +} + function openRowContent( kind: OpenRowKind, text: string, @@ -686,6 +704,17 @@ function syncToolElapsed(shell: AppShell, bag: BridgeBag, nowMs: number): void { } } +function isLocallyQueuedUserRow(row: StreamRow): boolean { + return ( + row.role === "user" && + (row.meta === "queue" || + row.meta === "steer" || + row.meta === "steering" || + row.meta === "following-up" || + row.meta === "reinject") + ); +} + /** * Retract everything the failed attempt painted, then forget the row * bookkeeping that pointed into it — a rolled-back tool call has no row left @@ -695,7 +724,11 @@ function rollbackAttempt(shell: AppShell, bag: BridgeBag): void { const boundary = bag.attemptRow; bag.attemptRow = null; if (boundary === null || boundary >= streamRowCount(shell)) return; + const localRows = Array.from({ length: streamRowCount(shell) - boundary }, (_, i) => + streamRowAt(shell, boundary + i), + ).filter((row): row is StreamRow => row !== undefined && isLocallyQueuedUserRow(row)); truncateStreamRows(shell, boundary); + for (const row of localRows) appendStreamRow(shell, row); for (const [callId, index] of [...bag.toolRows]) { if (index >= boundary) { bag.toolRows.delete(callId); @@ -1045,6 +1078,13 @@ export function attachSessionBridge( const settled = noteEvent(event); // Reactor-shaped types always map first (avoids tool.done name collision). if (PRODUCTION_REACTOR_TYPES.has(event.type)) { + if (consumePendingEchoEvent(bag, event)) { + // The echo skips the mapper so it cannot expire a recovery handoff, + // but it still starts a new turn: the next reasoning gets its own row. + closeOpenRow(shell, bag); + bag.turnThinking = null; + return; + } for (const mapped of mapProductionEvent(event as ReactorLikeEvent, bag.mapCtx)) { applyInbound(shell, bag, mapped); } @@ -1105,6 +1145,8 @@ export function attachSessionBridge( if (shell.session.run !== "busy") return; closeOpenRow(shell, bag); bag.pendingEchoes.length = 0; + bag.mapCtx.errorRollbackArmed = false; + bag.attemptRow = null; shell.session = interrupt(shell.session); appendStreamRow(shell, { role: "system", @@ -1162,6 +1204,11 @@ export function attachSessionBridge( if (bag.disposed) return; closeOpenRow(shell, bag); bag.pendingEchoes.length = 0; + // The stopped attempt is no longer in flight. Expire the error-recovery + // handoff so a later new-turn inference.start cannot roll back the + // classified error, the stop row, or the operator's next prompt. + bag.mapCtx.errorRollbackArmed = false; + bag.attemptRow = null; applyShellInterrupt(shell); bag.port.interrupt(); // The stop settles the turn without necessarily producing an idle event to diff --git a/src/tui/stream-event-map.test.ts b/src/tui/stream-event-map.test.ts index e4e4a5995..ae5df33d9 100644 --- a/src/tui/stream-event-map.test.ts +++ b/src/tui/stream-event-map.test.ts @@ -258,6 +258,33 @@ describe("inference.retry", () => { expect(actions(out)).toEqual(["mark", "rollback"]); }); + test("a same-turn failover start consumes the boundary handed off by inference.error", () => { + const out = mapProductionSequence([ + { type: "inference.start" }, + { type: "inference.text.delta", data: { token: "partial" } }, + { + type: "inference.error", + data: { + error: { category: "quota_exhausted", message: "The usage limit has been reached" }, + }, + }, + { type: "inference.start" }, + ]); + expect(actions(out)).toEqual(["mark", "rollback", "mark"]); + }); + + test("a same-turn failover start after credential_failure also rolls back", () => { + const out = mapProductionSequence([ + { type: "inference.start" }, + { + type: "inference.error", + data: { error: { category: "credential_failure", message: "Forbidden" } }, + }, + { type: "inference.start" }, + ]); + expect(actions(out)).toEqual(["mark", "rollback", "mark"]); + }); + test("a settled cycle disarms, so the next cycle's pre-commit retry is inert", () => { const out = mapProductionSequence([ { type: "inference.start" }, @@ -268,7 +295,7 @@ describe("inference.retry", () => { expect(actions(out)).toEqual(["mark", "clear"]); }); - test("any event other than the retry expires the error handoff", () => { + test("any event other than retry or start expires the error handoff", () => { const out = mapProductionSequence([ { type: "inference.start" }, { type: "inference.text.delta", data: { token: "partial" } }, diff --git a/src/tui/stream-event-map.ts b/src/tui/stream-event-map.ts index c8c53e85f..9ff63e9cc 100644 --- a/src/tui/stream-event-map.ts +++ b/src/tui/stream-event-map.ts @@ -109,10 +109,11 @@ export interface StreamMapContext { attemptCallIds: Set; /** * A committed attempt can also end in `inference.error` with no - * `inference.done`, and the reactor's committed-retry follows that error - * immediately. The boundary must not stay armed across a terminal error, so - * the error hands it off here: the very next event either is the retry that - * consumes it, or expires it. + * `inference.done`. Recovery follows that error as either the reactor's + * committed-retry or a same-turn failover `inference.start`. The boundary + * must not stay armed across a terminal error, so the error hands it off + * here: the very next event either consumes it (retry or start) and + * retracts the failed attempt, or expires it and keeps the error row. */ errorRollbackArmed: boolean; /** @@ -151,6 +152,23 @@ function disarmAttempt(ctx: StreamMapContext | undefined): readonly BridgeInboun return [ATTEMPT_CLEAR]; } +/** Drop call bookkeeping and held deltas that belonged only to the failed attempt. */ +function forgetAttemptLocalState(ctx: StreamMapContext): void { + for (const callId of [...ctx.callIdToName.keys()]) { + if (ctx.attemptCallIds.has(callId)) continue; + ctx.callIdToName.delete(callId); + ctx.callIdToArgs.delete(callId); + ctx.emittedToolCalls.delete(callId); + } + ctx.hadTextDelta = false; + ctx.pendingDelta.assistant = ""; + ctx.pendingDelta.thinking = ""; +} + +function recoversErrorHandoff(type: string): boolean { + return type === "inference.retry" || type === "inference.start"; +} + type DeltaChannel = "assistant" | "thinking"; const DELTA_EVENT_TYPE: Record = { @@ -276,10 +294,11 @@ export function mapProductionEvent( flushed.push(...flushDelta(ctx, "thinking")); } // The error handoff only survives to the very next event; consume it here so - // anything other than the retry it was meant for expires the boundary. + // anything other than the retry or failover start it was meant for expires + // the boundary and keeps the error row. const handoff = ctx?.errorRollbackArmed === true; if (ctx) ctx.errorRollbackArmed = false; - const expired = handoff && event.type !== "inference.retry" ? [ATTEMPT_CLEAR] : []; + const expired = handoff && !recoversErrorHandoff(event.type) ? [ATTEMPT_CLEAR] : []; const mapped = mapEvent(event, ctx, handoff); return [...flushed, ...expired, ...mapped]; } @@ -311,13 +330,20 @@ function mapEvent( return [...disarmed, { type: "user", text: full }]; } - case "inference.start": + case "inference.start": { + const recovered = errorRollbackHandoff; if (ctx) { + if (recovered) forgetAttemptLocalState(ctx); ctx.hadTextDelta = false; ctx.attemptArmed = true; ctx.attemptCallIds = new Set(ctx.callIdToName.keys()); } - return [ATTEMPT_MARK, { type: "run", state: "busy" }]; + return [ + ...(recovered ? [ATTEMPT_ROLLBACK] : []), + ATTEMPT_MARK, + { type: "run", state: "busy" }, + ]; + } case "inference.done": // Cycle settled: disarm so a pre-commit retry belonging to the *next* @@ -330,17 +356,7 @@ function mapEvent( // A retry that arrives with nothing armed is the harness's pre-commit // kind: the failed attempt never streamed, so there is nothing to undo. if (!armed && !errorRollbackHandoff) return []; - if (ctx) { - for (const callId of [...ctx.callIdToName.keys()]) { - if (ctx.attemptCallIds.has(callId)) continue; - ctx.callIdToName.delete(callId); - ctx.callIdToArgs.delete(callId); - ctx.emittedToolCalls.delete(callId); - } - ctx.hadTextDelta = false; - ctx.pendingDelta.assistant = ""; - ctx.pendingDelta.thinking = ""; - } + if (ctx) forgetAttemptLocalState(ctx); return [ATTEMPT_ROLLBACK]; } @@ -484,9 +500,9 @@ function mapEvent( ...(typeof err.retryAfterMs === "number" ? { retryAfterMs: err.retryAfterMs } : {}), }) : rawMessage; - // Hand the armed boundary to the next event rather than disarming: the - // reactor's committed-retry follows this error and must still retract - // the failed attempt, including the error row painted here. + // Hand the armed boundary to the next event rather than disarming: a + // committed retry or same-turn failover start must still retract the + // failed attempt, including the error row painted here. if (ctx?.attemptArmed === true) { ctx.attemptArmed = false; ctx.errorRollbackArmed = true;