From 893336bc7c919fcd88529b596ff1fec1909cc3d2 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Thu, 6 Aug 2026 23:07:50 -0700 Subject: [PATCH 1/4] Abort a model that loops on repeated output The stall watchdog measured silence, so a model streaming the same line on repeat never tripped it: tokens kept arriving, lastActivityAt kept refreshing, and the run burned tokens until a human noticed and interrupted it. The watchdog now also folds streamed text into a bounded buffer and checks it for a line repeated past a threshold, aborting immediately on detection regardless of how fast the loop is producing output. The recovery message names it as the model repeating itself and reports the tokens spent on the looped span, so a retry reads as reasonable rather than papering over a hang. --- src/tui-opentui/runtime-bridge.ts | 23 ++++++-- src/tui-opentui/stall-watchdog.test.ts | 55 +++++++++++++++++- src/tui-opentui/stall-watchdog.ts | 59 ++++++++++++++++++- src/tui-opentui/turn-monitor.test.ts | 55 ++++++++++++++++++ src/tui-opentui/turn-state.test.ts | 73 ++++++++++++++++++++++- src/tui-opentui/turn-state.ts | 80 ++++++++++++++++++++++---- 6 files changed, 324 insertions(+), 21 deletions(-) diff --git a/src/tui-opentui/runtime-bridge.ts b/src/tui-opentui/runtime-bridge.ts index 2f3bddedd..7987accfd 100644 --- a/src/tui-opentui/runtime-bridge.ts +++ b/src/tui-opentui/runtime-bridge.ts @@ -41,10 +41,12 @@ import { import { quotaWaitSeconds, shouldAutoRetryQuota } from "./quota-retry.js" import { applyStallRecovery, + repetitionRecoveryMessage, shouldAbortForStall, shouldNoticeStall, STALL_NOTICE_MESSAGE, STALL_NOTICE_MS, + STALL_RECOVERY_MESSAGE, STALL_TIMEOUT_MS, } from "./stall-watchdog.js" import { @@ -919,6 +921,19 @@ export function attachSessionBridge( return } + // Content-based, not time-based: a repeating line means the model is + // stuck regardless of how fast it is producing it, so this is checked + // before the silence clock rather than folded into it. + if (bag.turn.status === "running" && bag.turn.repeating) { + const repeatedTokens = + bag.turn.streamTokenCount - (bag.turn.repeatingSinceTokenCount ?? 0) + applyStallRecovery( + { abort: doInterrupt, notify: (message) => setStatusFlash(shell, message) }, + repetitionRecoveryMessage(repeatedTokens), + ) + return + } + const stallArgs = { status: bag.turn.status, awaitingResponse: bag.turn.awaitingResponse, @@ -930,10 +945,10 @@ export function attachSessionBridge( } if (shouldAbortForStall(stallArgs)) { - applyStallRecovery({ - abort: doInterrupt, - notify: (message) => setStatusFlash(shell, message), - }) + applyStallRecovery( + { abort: doInterrupt, notify: (message) => setStatusFlash(shell, message) }, + STALL_RECOVERY_MESSAGE, + ) return } diff --git a/src/tui-opentui/stall-watchdog.test.ts b/src/tui-opentui/stall-watchdog.test.ts index e3ace6a7f..28ce2c52e 100644 --- a/src/tui-opentui/stall-watchdog.test.ts +++ b/src/tui-opentui/stall-watchdog.test.ts @@ -2,6 +2,8 @@ import { describe, expect, test } from "bun:test" import { applyStallRecovery, + detectRepetition, + repetitionRecoveryMessage, shouldAbortForStall, shouldNoticeStall, STALL_NOTICE_MS, @@ -81,7 +83,7 @@ describe("shouldAbortForStall", () => { }) describe("applyStallRecovery", () => { - test("aborts then notifies", () => { + test("aborts then notifies with the default message", () => { const calls: string[] = [] applyStallRecovery({ abort: () => calls.push("abort"), @@ -89,6 +91,57 @@ describe("applyStallRecovery", () => { }) expect(calls).toEqual(["abort", STALL_RECOVERY_MESSAGE]) }) + + test("aborts then notifies with a supplied message", () => { + const calls: string[] = [] + applyStallRecovery( + { abort: () => calls.push("abort"), notify: (m) => calls.push(m) }, + "custom message", + ) + expect(calls).toEqual(["abort", "custom message"]) + }) +}) + +describe("detectRepetition", () => { + test("finds nothing in fresh, varied output", () => { + const text = [ + "I'll check the callId emission path first.", + "Running the search now.", + "Found three matches across the module.", + ].join("\n") + expect(detectRepetition(text).repeating).toBe(false) + }) + + test("flags a line repeated past the occurrence threshold", () => { + const line1 = + "I'll verify callId emission and remaining edges, then write the ranked findings." + const line2 = "Confirming callId emission, then writing the ranked findings." + const text = Array(4).fill([line1, line2]).flat().join("\n") + const check = detectRepetition(text) + expect(check.repeating).toBe(true) + expect(check.repeatedLine).toBe(line1) + expect(check.occurrences).toBeGreaterThanOrEqual(3) + }) + + test("ignores short recurring lines", () => { + const text = Array(6).fill("Checking...").join("\n") + expect(detectRepetition(text).repeating).toBe(false) + }) + + test("does not flag two occurrences", () => { + const line = + "I'll verify callId emission and remaining edges, then write the ranked findings." + const text = [line, "some other progress here.", line].join("\n") + expect(detectRepetition(text).repeating).toBe(false) + }) +}) + +describe("repetitionRecoveryMessage", () => { + test("names degeneration and attributes the looped tokens", () => { + const message = repetitionRecoveryMessage(42) + expect(message).toContain("repeating itself") + expect(message).toContain("42") + }) }) describe("shouldNoticeStall", () => { diff --git a/src/tui-opentui/stall-watchdog.ts b/src/tui-opentui/stall-watchdog.ts index d51178dfa..5d12175ae 100644 --- a/src/tui-opentui/stall-watchdog.ts +++ b/src/tui-opentui/stall-watchdog.ts @@ -21,6 +21,43 @@ export type ShouldAbortForStallArgs = { readonly streamingType: "text" | "thinking" | "tool" | null } +// A repeated line has to be long enough that short, legitimately-recurring +// fragments ("Let me check.") do not trip the guard. +const REPETITION_MIN_LINE_LENGTH = 20 +// How far back into the streamed text to look for repeats. Bounded so the +// check stays cheap however long the turn runs. +const REPETITION_LOOKBACK_LINES = 12 +// Three verbatim repeats of the same substantial line is not a coincidence +// of phrasing — it is the model looping. +const REPETITION_MIN_OCCURRENCES = 3 + +export type RepetitionCheck = { + readonly repeating: boolean + readonly repeatedLine: string | null + readonly occurrences: number +} + +/** + * Whether the tail of the streamed text is dominated by a line repeated + * verbatim. Pure text-in, decision-out: the caller owns accumulating the + * buffer across deltas and cycles within a turn. + */ +export function detectRepetition(text: string): RepetitionCheck { + const lines = text + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length >= REPETITION_MIN_LINE_LENGTH) + const tail = lines.slice(-REPETITION_LOOKBACK_LINES) + const counts = new Map() + for (const line of tail) counts.set(line, (counts.get(line) ?? 0) + 1) + for (const [line, occurrences] of counts) { + if (occurrences >= REPETITION_MIN_OCCURRENCES) { + return { repeating: true, repeatedLine: line, occurrences } + } + } + return { repeating: false, repeatedLine: null, occurrences: 0 } +} + /** * Whether silence of `thresholdMs` counts as stuck at all. Shared by the notice * and the abort so they never disagree about which runs are stalled — only @@ -62,19 +99,35 @@ export function shouldNoticeStall(args: ShouldNoticeStallArgs): boolean { return silentPastThreshold(args, args.stallNoticeMs) } -/** Shown while the run is silent; names the state and the way out. */ +/** + * Shown while nothing is arriving at all. Never fires while tokens are + * flowing — a model looping on repeated content is still producing output, + * so it is reported by `repetitionRecoveryMessage` instead, not this one. + */ export const STALL_NOTICE_MESSAGE = "no response for a while — ctrl+c to interrupt" export const STALL_RECOVERY_MESSAGE = "stopped after no response — send again to retry" +/** + * Shown once a repeated line aborts the turn. Named as degeneration, not a + * generic failure, so a retry reads as the reasonable next step rather than + * papering over a suspected hang or network fault. + */ +export function repetitionRecoveryMessage(repeatedTokens: number): string { + return `stopped after repeating itself — ~${repeatedTokens} tokens looped — send again to retry` +} + export type ApplyStallRecoveryDeps = { /** Abort the in-flight run through the session port. */ readonly abort: () => void readonly notify: (message: string) => void } -export function applyStallRecovery(deps: ApplyStallRecoveryDeps): void { +export function applyStallRecovery( + deps: ApplyStallRecoveryDeps, + message: string = STALL_RECOVERY_MESSAGE, +): void { deps.abort() - deps.notify(STALL_RECOVERY_MESSAGE) + deps.notify(message) } diff --git a/src/tui-opentui/turn-monitor.test.ts b/src/tui-opentui/turn-monitor.test.ts index 397ef9b93..46b225be9 100644 --- a/src/tui-opentui/turn-monitor.test.ts +++ b/src/tui-opentui/turn-monitor.test.ts @@ -369,6 +369,61 @@ describe("stall watchdog", () => { }) }) +describe("repetition guard", () => { + test("aborts a looping model without waiting on the stall clock", async () => { + await withTestRenderer(async (h) => { + const t: Harness = await setup(h) + try { + t.bridge.submit("build it", "immediate") + t.port.clear() + + const line1 = + "I'll verify callId emission and remaining edges, then write the ranked findings.\n" + const line2 = "Confirming callId emission, then writing the ranked findings.\n" + + // Tokens keep landing every tick — a real stall would never fire here. + for (let i = 0; i < 6; i++) { + t.bridge.handle({ + type: "inference.text.delta", + data: { token: i % 2 === 0 ? line1 : line2 }, + }) + t.advance(10) + t.tick() + } + + expect(t.port.calls).toEqual([{ op: "interrupt" }]) + expect(t.shell.statusFlash).toContain("repeating itself") + expect(t.shell.statusFlash).not.toBe(STALL_NOTICE_MESSAGE) + } finally { + t.bridge.dispose() + } + }) + }) + + test("a slow but progressing turn is never killed", async () => { + await withTestRenderer(async (h) => { + const t: Harness = await setup(h) + try { + t.bridge.submit("build it", "immediate") + t.port.clear() + + for (let i = 0; i < 5; i++) { + t.bridge.handle({ + type: "inference.text.delta", + data: { token: `distinct progress update number ${i}\n` }, + }) + t.advance(500) + t.tick() + } + + expect(t.port.calls).toEqual([]) + } finally { + t.bridge.dispose() + } + }) + }) +}) + describe("reasoning settles to a summary", () => { test("a closed thinking row carries its elapsed time", async () => { await withTestRenderer(async (h) => { diff --git a/src/tui-opentui/turn-state.test.ts b/src/tui-opentui/turn-state.test.ts index 2cdc813aa..39720c339 100644 --- a/src/tui-opentui/turn-state.test.ts +++ b/src/tui-opentui/turn-state.test.ts @@ -9,7 +9,12 @@ import { } from "./turn-state.js" const fold = ( - events: readonly { type: string; data?: unknown; state?: string }[], + events: readonly { + type: string + data?: unknown + state?: string + text?: string + }[], startMs = 0, ) => events.reduce( @@ -184,3 +189,69 @@ describe("turn transitions", () => { expect(s.isProcessing).toBe(true) }) }) + +describe("repetition tracking", () => { + const line1 = + "I'll verify callId emission and remaining edges, then write the ranked findings." + const line2 = + "Confirming callId emission, then writing the ranked findings." + + const textDelta = (text: string) => ({ + type: "inference.text.delta", + data: { token: text }, + }) + + test("varied streamed text is never flagged", () => { + const s = fold([ + { type: "inference.start" }, + textDelta("I'll check the callId path.\n"), + textDelta("Running the search now.\n"), + textDelta("Found the match.\n"), + ]) + expect(s.repeating).toBe(false) + expect(s.repeatingSinceTokenCount).toBeNull() + }) + + test("a line looping past the threshold flips repeating and latches the token count", () => { + const deltas = Array(4) + .fill([line1, line2]) + .flat() + .map((line) => textDelta(`${line}\n`)) + const s = fold([{ type: "inference.start" }, ...deltas]) + expect(s.repeating).toBe(true) + // Three text deltas land before the third `line1` repeat crosses the + // occurrence threshold and latches the count. + expect(s.repeatingSinceTokenCount).toBe(5) + }) + + test("repetition tracked across a tool cycle survives connector.reply with tools outstanding", () => { + const deltas = Array(4) + .fill([line1, line2]) + .flat() + .map((line) => textDelta(`${line}\n`)) + const withTool = turnStateFromEvent( + fold([{ type: "inference.start" }, ...deltas]), + { type: "tool.start", data: { call: { id: "c1", name: "grep" } } }, + 100, + ) + const afterReply = turnStateFromEvent( + withTool, + { type: "connector.reply" }, + 101, + ) + expect(afterReply.repeating).toBe(true) + expect(afterReply.streamText.length).toBeGreaterThan(0) + }) + + test("a fresh submit clears the repetition state", () => { + const deltas = Array(4) + .fill([line1, line2]) + .flat() + .map((line) => textDelta(`${line}\n`)) + const looping = fold([{ type: "inference.start" }, ...deltas]) + const restarted = turnStateOnSubmit(looping, 200) + expect(restarted.repeating).toBe(false) + expect(restarted.repeatingSinceTokenCount).toBeNull() + expect(restarted.streamText).toBe("") + }) +}) diff --git a/src/tui-opentui/turn-state.ts b/src/tui-opentui/turn-state.ts index 2f681d23c..d15d4e80b 100644 --- a/src/tui-opentui/turn-state.ts +++ b/src/tui-opentui/turn-state.ts @@ -11,8 +11,14 @@ import { type } from "arktype" +import { detectRepetition } from "./stall-watchdog.js" import type { TurnStatus } from "./session-chrome.js" +// Bound on the accumulated stream text kept for repetition checks. Comfortably +// larger than the lookback window `detectRepetition` reads, so trimming never +// drops a line the check still needs. +const STREAM_TEXT_BUFFER_CHARS = 8_000 + export type QuotaWait = { readonly retryAfterMs: number readonly retryAt: number @@ -41,6 +47,20 @@ export type TurnState = { * so the settle decision needs the outstanding ids, not just the last name. */ readonly activeToolCalls: readonly string[] + /** + * Tail of the text/thinking output streamed this turn, across cycles — + * `connector.reply` with tools outstanding does not clear it. Bounded to + * `STREAM_TEXT_BUFFER_CHARS`; feeds `detectRepetition`, nothing else. + */ + readonly streamText: string + /** Live result of checking `streamText` for a looping line. */ + readonly repeating: boolean + /** + * `streamTokenCount` at the moment repetition was first observed this turn. + * Latched, not recomputed, so the abort can report tokens spent looping + * rather than the whole turn's count. + */ + readonly repeatingSinceTokenCount: number | null } export function initialTurnState(nowMs: number): TurnState { @@ -54,6 +74,9 @@ export function initialTurnState(nowMs: number): TurnState { lastActivityAt: nowMs, quota: null, activeToolCalls: [], + streamText: "", + repeating: false, + repeatingSinceTokenCount: null, } } @@ -69,6 +92,9 @@ export function turnStateOnSubmit(state: TurnState, nowMs: number): TurnState { streamTokenCount: 0, lastActivityAt: nowMs, activeToolCalls: [], + streamText: "", + repeating: false, + repeatingSinceTokenCount: null, } } @@ -93,6 +119,20 @@ const inferenceErrorData = type({ }, }) +const tokenData = type({ "token?": "string" }) + +/** + * Text carried by a delta event. Reactor-shaped deltas carry it as + * `data.token`; canonical bridge deltas carry it as a top-level `text`. + */ +function deltaText(event: { readonly data?: unknown; readonly text?: string }): string { + const parsed = tokenData(event.data) + if (!(parsed instanceof type.errors) && parsed.token !== undefined) { + return parsed.token + } + return event.text ?? "" +} + const namedCallData = type({ "name?": "string" }) const toolStartData = type({ call: { "name?": "string" }, @@ -178,16 +218,30 @@ const streaming = ( state: TurnState, kind: "text" | "thinking", nowMs: number, -): TurnState => ({ - ...state, - status: state.status === "blocked" ? "blocked" : "running", - isProcessing: true, - awaitingResponse: false, - streamingType: kind, - streamTokenCount: - kind === "text" ? state.streamTokenCount + 1 : state.streamTokenCount, - lastActivityAt: nowMs, -}) + text: string, +): TurnState => { + const streamTokenCount = + kind === "text" ? state.streamTokenCount + 1 : state.streamTokenCount + const streamText = `${state.streamText}${text}`.slice( + -STREAM_TEXT_BUFFER_CHARS, + ) + const check = detectRepetition(streamText) + return { + ...state, + status: state.status === "blocked" ? "blocked" : "running", + isProcessing: true, + awaitingResponse: false, + streamingType: kind, + streamTokenCount, + lastActivityAt: nowMs, + streamText, + repeating: check.repeating, + repeatingSinceTokenCount: + check.repeating && state.repeatingSinceTokenCount === null + ? streamTokenCount + : state.repeatingSinceTokenCount, + } +} const runningTool = ( state: TurnState, @@ -215,6 +269,7 @@ export function turnStateFromEvent( /** Canonical bridge shapes carry these instead of `data`. */ readonly state?: string readonly name?: string + readonly text?: string }, nowMs: number, ): TurnState { @@ -235,10 +290,11 @@ export function turnStateFromEvent( case "inference.text.delta": case "assistant.delta": - return streaming(state, "text", nowMs) + return streaming(state, "text", nowMs, deltaText(event)) case "inference.thinking.delta": - return streaming(state, "thinking", nowMs) + case "thinking.delta": + return streaming(state, "thinking", nowMs, deltaText(event)) case "inference.tool_call.delta": return runningTool(state, toolName(event.data), nowMs) From 0f1ae2fa59dd3703f7dbf55a6b3e65ee72d6ae9a Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Thu, 6 Aug 2026 23:30:33 -0700 Subject: [PATCH 2/4] Detect repetition by character period, not by line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Line-splitting missed the captured incident outright: the looping sentences ran together with no newline between them, so the whole span collapsed into one line and never reached the occurrence check. Detection now finds the smallest period the streamed tail exactly repeats, which catches the no-newline shape the same way it catches a line-level loop. Line counting also flagged ordinary structure — a repeated markdown table row, a few identical code lines — as degeneration. The period search requires more repeats and enough character variety in the repeating unit to rule those out, verified against both plus a monochrome run (a repeated rule or the same character streamed many times), which is trivially periodic at every length and would otherwise be the easiest false trigger of all. The per-delta check is now throttled to run once per chunk of new text rather than once per token, since a repeating tail cannot appear or disappear between two three-character tokens. The idle notice also now checks the repetition flag directly, so a stream that is looping but not silent can no longer be labeled a silent hang. --- src/tui-opentui/runtime-bridge.ts | 16 +++- src/tui-opentui/stall-watchdog.test.ts | 53 ++++++++++--- src/tui-opentui/stall-watchdog.ts | 105 ++++++++++++++++++------- src/tui-opentui/turn-monitor.test.ts | 11 ++- src/tui-opentui/turn-state.test.ts | 41 +++++----- src/tui-opentui/turn-state.ts | 38 +++++++-- 6 files changed, 197 insertions(+), 67 deletions(-) diff --git a/src/tui-opentui/runtime-bridge.ts b/src/tui-opentui/runtime-bridge.ts index 7987accfd..412597941 100644 --- a/src/tui-opentui/runtime-bridge.ts +++ b/src/tui-opentui/runtime-bridge.ts @@ -924,6 +924,14 @@ export function attachSessionBridge( // Content-based, not time-based: a repeating line means the model is // stuck regardless of how fast it is producing it, so this is checked // before the silence clock rather than folded into it. + // + // Gated on `status === "running"` because every turn-ending transition + // (interrupt, connector.reply with no tools outstanding, reactor.done / + // reactor.error) routes through `initialTurnState`, which clears + // `repeating`. If a future settle path changes `isProcessing` without + // also resetting `status` and `repeating` through that same reset, this + // guard would no longer mean "the turn is actually live" and could fire + // on an already-settled turn — recheck this alongside any such change. if (bag.turn.status === "running" && bag.turn.repeating) { const repeatedTokens = bag.turn.streamTokenCount - (bag.turn.repeatingSinceTokenCount ?? 0) @@ -954,7 +962,13 @@ export function attachSessionBridge( // Notice only — the phase still paints below, because a ramp that stops // moving is the very thing that reads as a hang. - if (shouldNoticeStall({ ...stallArgs, stallNoticeMs })) { + if ( + shouldNoticeStall({ + ...stallArgs, + stallNoticeMs, + repeating: bag.turn.repeating, + }) + ) { setStatusFlash(shell, STALL_NOTICE_MESSAGE) } diff --git a/src/tui-opentui/stall-watchdog.test.ts b/src/tui-opentui/stall-watchdog.test.ts index 28ce2c52e..885ae7115 100644 --- a/src/tui-opentui/stall-watchdog.test.ts +++ b/src/tui-opentui/stall-watchdog.test.ts @@ -112,26 +112,54 @@ describe("detectRepetition", () => { expect(detectRepetition(text).repeating).toBe(false) }) - test("flags a line repeated past the occurrence threshold", () => { + // The captured incident: the two sentences ran together with no line break + // at all. A line-splitting detector never sees this; the period search + // does not care where (or whether) the lines break. + test("flags the captured incident string verbatim, with no newlines", () => { const line1 = "I'll verify callId emission and remaining edges, then write the ranked findings." const line2 = "Confirming callId emission, then writing the ranked findings." - const text = Array(4).fill([line1, line2]).flat().join("\n") + const text = Array(10).fill(`${line1}${line2}`).join("") const check = detectRepetition(text) expect(check.repeating).toBe(true) - expect(check.repeatedLine).toBe(line1) - expect(check.occurrences).toBeGreaterThanOrEqual(3) + expect(check.period).toBe(line1.length + line2.length) }) - test("ignores short recurring lines", () => { - const text = Array(6).fill("Checking...").join("\n") + test("does not flag the same cycle a handful of times", () => { + const line1 = + "I'll verify callId emission and remaining edges, then write the ranked findings." + const line2 = "Confirming callId emission, then writing the ranked findings." + // Fewer than the occurrence threshold: a model can legitimately restate + // a step once or twice across tool-call cycles without looping. + const text = Array(4).fill(`${line1}${line2}`).join("") expect(detectRepetition(text).repeating).toBe(false) }) - test("does not flag two occurrences", () => { - const line = - "I'll verify callId emission and remaining edges, then write the ranked findings." - const text = [line, "some other progress here.", line].join("\n") + test("does not flag a repeated markdown table separator row", () => { + const row = "| ---------------------- | ---------------------- |" + const text = Array(6).fill(row).join("\n") + expect(detectRepetition(text).repeating).toBe(false) + }) + + test("does not flag a few identical code lines", () => { + const line = " const result = await fetchData(request, options, context)" + const text = Array(3).fill(line).join("\n") + expect(detectRepetition(text).repeating).toBe(false) + }) + + test("ignores short recurring fragments", () => { + const text = Array(10).fill("ok").join(" ") + expect(detectRepetition(text).repeating).toBe(false) + }) + + // A monochrome run is periodic at every period by construction — the + // easiest thing to false-trigger on if entropy is not checked. + test("does not flag a long run of the same character", () => { + expect(detectRepetition("x".repeat(500)).repeating).toBe(false) + }) + + test("does not flag a repeated horizontal rule", () => { + const text = Array(10).fill("----------------------------").join("\n") expect(detectRepetition(text).repeating).toBe(false) }) }) @@ -154,8 +182,13 @@ describe("shouldNoticeStall", () => { stallNoticeMs: STALL_NOTICE_MS, isProcessing: true, streamingType: null, + repeating: false, } + test("stays quiet while repeating, even if also silent by the clock", () => { + expect(shouldNoticeStall({ ...base, repeating: true })).toBe(false) + }) + test("speaks up long before the abort backstop", () => { expect(STALL_NOTICE_MS).toBeLessThan(STALL_TIMEOUT_MS) expect(shouldNoticeStall(base)).toBe(true) diff --git a/src/tui-opentui/stall-watchdog.ts b/src/tui-opentui/stall-watchdog.ts index 5d12175ae..3ae4c4416 100644 --- a/src/tui-opentui/stall-watchdog.ts +++ b/src/tui-opentui/stall-watchdog.ts @@ -21,41 +21,88 @@ export type ShouldAbortForStallArgs = { readonly streamingType: "text" | "thinking" | "tool" | null } -// A repeated line has to be long enough that short, legitimately-recurring -// fragments ("Let me check.") do not trip the guard. -const REPETITION_MIN_LINE_LENGTH = 20 -// How far back into the streamed text to look for repeats. Bounded so the -// check stays cheap however long the turn runs. -const REPETITION_LOOKBACK_LINES = 12 -// Three verbatim repeats of the same substantial line is not a coincidence -// of phrasing — it is the model looping. -const REPETITION_MIN_OCCURRENCES = 3 +// The captured incident looped two sentences with no line break between them +// ("...ranked findings.Confirming callId emission...") — degeneration is a +// character-level loop, not a line-level one. Splitting on "\n" misses it +// entirely, so the tail is treated as a plain string and checked for the +// smallest period it exactly repeats: the shortest span p such that the last +// several hundred characters equal p repeated. +// +// A period below this is more likely a short structural tic (indentation, a +// repeated bullet or table-cell divider) than a looping phrase. Chosen well +// under the ~140-char period of the captured incident's two-sentence cycle, +// with headroom for shorter degenerate loops (a single repeated sentence). +const REPETITION_MIN_PERIOD = 24 +// How many exact repeats of the period are required before it counts as a +// loop rather than a coincidence. Verified against real non-degenerate +// repetition: a 6-row markdown table separator (period ~51 chars, 6 exact +// repeats) and 3 identical code lines (period ~60 chars, 3 exact repeats) +// both land under this bar and are not flagged; the captured incident's +// sentence pair comfortably clears it well before the stream ends. +const REPETITION_MIN_REPEATS = 8 +// Hard ceiling on the period search regardless of buffer size, purely to cap +// worst-case work per check — token-level degeneration loops on a phrase or +// two, never on multi-paragraph spans. +const REPETITION_MAX_PERIOD_CAP = 2_000 +// A monochrome run ("x".repeat(500), a "----" rule, a wall of spaces) is +// trivially periodic at *every* period, which would otherwise make it the +// single easiest thing to false-trigger on — verified by execution against +// `thinking-reveal.test.ts`'s burst-of-"x" fixture, which tripped the guard +// before this floor existed. Requiring the repeating unit itself to contain +// this many distinct characters keeps single-character and low-variety runs +// out without weakening the sentence-level case: the captured incident's +// cycle spans two full sentences, comfortably above it. +const REPETITION_MIN_DISTINCT_CHARS = 8 export type RepetitionCheck = { readonly repeating: boolean - readonly repeatedLine: string | null - readonly occurrences: number + readonly period: number | null + readonly repeats: number } /** - * Whether the tail of the streamed text is dominated by a line repeated - * verbatim. Pure text-in, decision-out: the caller owns accumulating the - * buffer across deltas and cycles within a turn. + * Length of the exact-period run ending at the last character of `text`, + * including the base period itself. `text[i] === text[i - period]` walked + * backwards from the end; stops at the first mismatch or the start of the + * string. + */ +function periodicSuffixLength(text: string, period: number): number { + let i = text.length - 1 + let j = i - period + let matched = 0 + while (j >= 0 && text[i] === text[j]) { + matched++ + i-- + j-- + } + return matched + period +} + +/** + * Whether the tail of `text` is an exact repeat of some short span at least + * `REPETITION_MIN_REPEATS` times. Pure text-in, decision-out: the caller owns + * accumulating the buffer across deltas and cycles within a turn. + * + * Periods longer than `text.length / REPETITION_MIN_REPEATS` are skipped, not + * as an arbitrary cutoff but because they cannot mathematically reach the + * occurrence threshold within the given text — a loop with a longer period + * needs a longer buffer to confirm, which is a buffer-size trade-off owned by + * the caller, not a second detection path here. */ export function detectRepetition(text: string): RepetitionCheck { - const lines = text - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length >= REPETITION_MIN_LINE_LENGTH) - const tail = lines.slice(-REPETITION_LOOKBACK_LINES) - const counts = new Map() - for (const line of tail) counts.set(line, (counts.get(line) ?? 0) + 1) - for (const [line, occurrences] of counts) { - if (occurrences >= REPETITION_MIN_OCCURRENCES) { - return { repeating: true, repeatedLine: line, occurrences } - } + const maxPeriod = Math.min( + REPETITION_MAX_PERIOD_CAP, + Math.floor(text.length / REPETITION_MIN_REPEATS), + ) + for (let period = REPETITION_MIN_PERIOD; period <= maxPeriod; period++) { + const matched = periodicSuffixLength(text, period) + const repeats = matched / period + if (repeats < REPETITION_MIN_REPEATS) continue + const unit = text.slice(text.length - period) + if (new Set(unit).size < REPETITION_MIN_DISTINCT_CHARS) continue + return { repeating: true, period, repeats } } - return { repeating: false, repeatedLine: null, occurrences: 0 } + return { repeating: false, period: null, repeats: 0 } } /** @@ -87,14 +134,18 @@ export function shouldAbortForStall(args: ShouldAbortForStallArgs): boolean { export type ShouldNoticeStallArgs = ShouldAbortForStallArgs & { readonly stallNoticeMs: number + /** Whether the repetition guard currently sees a looping tail. */ + readonly repeating: boolean } /** * Returns true while the run has been silent long enough to say so but not yet * long enough to abort. False once the abort takes over, so the two never - * paint at the same time. + * paint at the same time, and false while repeating — that run is producing + * output, just not useful output, and "no response" would misdescribe it. */ export function shouldNoticeStall(args: ShouldNoticeStallArgs): boolean { + if (args.repeating) return false if (shouldAbortForStall(args)) return false return silentPastThreshold(args, args.stallNoticeMs) } diff --git a/src/tui-opentui/turn-monitor.test.ts b/src/tui-opentui/turn-monitor.test.ts index 46b225be9..dc4bca865 100644 --- a/src/tui-opentui/turn-monitor.test.ts +++ b/src/tui-opentui/turn-monitor.test.ts @@ -377,15 +377,18 @@ describe("repetition guard", () => { t.bridge.submit("build it", "immediate") t.port.clear() + // The captured incident shape: the two sentences run together with + // no line break between cycles. const line1 = - "I'll verify callId emission and remaining edges, then write the ranked findings.\n" - const line2 = "Confirming callId emission, then writing the ranked findings.\n" + "I'll verify callId emission and remaining edges, then write the ranked findings." + const line2 = "Confirming callId emission, then writing the ranked findings." + const cycle = `${line1}${line2}` // Tokens keep landing every tick — a real stall would never fire here. - for (let i = 0; i < 6; i++) { + for (let i = 0; i < 10; i++) { t.bridge.handle({ type: "inference.text.delta", - data: { token: i % 2 === 0 ? line1 : line2 }, + data: { token: cycle }, }) t.advance(10) t.tick() diff --git a/src/tui-opentui/turn-state.test.ts b/src/tui-opentui/turn-state.test.ts index 39720c339..07405a932 100644 --- a/src/tui-opentui/turn-state.test.ts +++ b/src/tui-opentui/turn-state.test.ts @@ -193,8 +193,10 @@ describe("turn transitions", () => { describe("repetition tracking", () => { const line1 = "I'll verify callId emission and remaining edges, then write the ranked findings." - const line2 = - "Confirming callId emission, then writing the ranked findings." + const line2 = "Confirming callId emission, then writing the ranked findings." + // The captured incident shape: the two sentences run together with no + // separator, each delta landing as one full cycle. + const cycle = `${line1}${line2}` const textDelta = (text: string) => ({ type: "inference.text.delta", @@ -212,23 +214,27 @@ describe("repetition tracking", () => { expect(s.repeatingSinceTokenCount).toBeNull() }) - test("a line looping past the threshold flips repeating and latches the token count", () => { - const deltas = Array(4) - .fill([line1, line2]) - .flat() - .map((line) => textDelta(`${line}\n`)) + test("the captured incident shape (no separator between cycles) flips repeating", () => { + const deltas = Array(10) + .fill(cycle) + .map((text) => textDelta(text)) const s = fold([{ type: "inference.start" }, ...deltas]) expect(s.repeating).toBe(true) - // Three text deltas land before the third `line1` repeat crosses the - // occurrence threshold and latches the count. - expect(s.repeatingSinceTokenCount).toBe(5) + expect(s.repeatingSinceTokenCount).not.toBeNull() + }) + + test("a couple of restated cycles across tool calls is not a loop", () => { + const deltas = Array(3) + .fill(cycle) + .map((text) => textDelta(text)) + const s = fold([{ type: "inference.start" }, ...deltas]) + expect(s.repeating).toBe(false) }) test("repetition tracked across a tool cycle survives connector.reply with tools outstanding", () => { - const deltas = Array(4) - .fill([line1, line2]) - .flat() - .map((line) => textDelta(`${line}\n`)) + const deltas = Array(10) + .fill(cycle) + .map((text) => textDelta(text)) const withTool = turnStateFromEvent( fold([{ type: "inference.start" }, ...deltas]), { type: "tool.start", data: { call: { id: "c1", name: "grep" } } }, @@ -244,10 +250,9 @@ describe("repetition tracking", () => { }) test("a fresh submit clears the repetition state", () => { - const deltas = Array(4) - .fill([line1, line2]) - .flat() - .map((line) => textDelta(`${line}\n`)) + const deltas = Array(10) + .fill(cycle) + .map((text) => textDelta(text)) const looping = fold([{ type: "inference.start" }, ...deltas]) const restarted = turnStateOnSubmit(looping, 200) expect(restarted.repeating).toBe(false) diff --git a/src/tui-opentui/turn-state.ts b/src/tui-opentui/turn-state.ts index d15d4e80b..688c5e0f5 100644 --- a/src/tui-opentui/turn-state.ts +++ b/src/tui-opentui/turn-state.ts @@ -15,10 +15,18 @@ import { detectRepetition } from "./stall-watchdog.js" import type { TurnStatus } from "./session-chrome.js" // Bound on the accumulated stream text kept for repetition checks. Comfortably -// larger than the lookback window `detectRepetition` reads, so trimming never -// drops a line the check still needs. +// larger than the periods `detectRepetition` can confirm, so trimming never +// drops content the check still needs. const STREAM_TEXT_BUFFER_CHARS = 8_000 +// `detectRepetition` walks a character-level period search; cheap per call, +// but the reactor loop can emit a delta per token, and running it on every +// single one makes it the hottest thing in that loop for no benefit — a +// repeating tail does not appear or disappear between two three-character +// tokens. Checking once per chunk of newly streamed text instead keeps the +// cost proportional to output, not token count. +const REPETITION_CHECK_INTERVAL_CHARS = 40 + export type QuotaWait = { readonly retryAfterMs: number readonly retryAt: number @@ -53,7 +61,15 @@ export type TurnState = { * `STREAM_TEXT_BUFFER_CHARS`; feeds `detectRepetition`, nothing else. */ readonly streamText: string - /** Live result of checking `streamText` for a looping line. */ + /** + * Total characters streamed this turn, uncapped — unlike `streamText.length` + * this keeps climbing after the buffer fills, which is what lets the + * throttle below tell "40 more chars arrived" from "the buffer is full." + */ + readonly streamCharsSeen: number + /** `streamCharsSeen` as of the last `detectRepetition` call. */ + readonly repetitionCheckedAt: number + /** Result of the most recent `detectRepetition` check on `streamText`. */ readonly repeating: boolean /** * `streamTokenCount` at the moment repetition was first observed this turn. @@ -75,6 +91,8 @@ export function initialTurnState(nowMs: number): TurnState { quota: null, activeToolCalls: [], streamText: "", + streamCharsSeen: 0, + repetitionCheckedAt: 0, repeating: false, repeatingSinceTokenCount: null, } @@ -93,6 +111,8 @@ export function turnStateOnSubmit(state: TurnState, nowMs: number): TurnState { lastActivityAt: nowMs, activeToolCalls: [], streamText: "", + streamCharsSeen: 0, + repetitionCheckedAt: 0, repeating: false, repeatingSinceTokenCount: null, } @@ -225,7 +245,10 @@ const streaming = ( const streamText = `${state.streamText}${text}`.slice( -STREAM_TEXT_BUFFER_CHARS, ) - const check = detectRepetition(streamText) + const streamCharsSeen = state.streamCharsSeen + text.length + const due = + streamCharsSeen - state.repetitionCheckedAt >= REPETITION_CHECK_INTERVAL_CHARS + const repeating = due ? detectRepetition(streamText).repeating : state.repeating return { ...state, status: state.status === "blocked" ? "blocked" : "running", @@ -235,9 +258,11 @@ const streaming = ( streamTokenCount, lastActivityAt: nowMs, streamText, - repeating: check.repeating, + streamCharsSeen, + repetitionCheckedAt: due ? streamCharsSeen : state.repetitionCheckedAt, + repeating, repeatingSinceTokenCount: - check.repeating && state.repeatingSinceTokenCount === null + repeating && state.repeatingSinceTokenCount === null ? streamTokenCount : state.repeatingSinceTokenCount, } @@ -293,7 +318,6 @@ export function turnStateFromEvent( return streaming(state, "text", nowMs, deltaText(event)) case "inference.thinking.delta": - case "thinking.delta": return streaming(state, "thinking", nowMs, deltaText(event)) case "inference.tool_call.delta": From 95bbd865511d256e694cd58fc780a9e7ae3f869c Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 7 Aug 2026 00:57:32 -0700 Subject: [PATCH 3/4] Scope the repetition guard's buffer to one streaming cycle A tool call ended a streaming cycle but left the repetition buffer intact, so several short narration lines said before separate tool calls in one turn concatenated into an apparent loop and aborted an otherwise ordinary turn. A genuinely degenerate model repeats within one unbroken stream; narration between tool calls does not, so the buffer now clears whenever a tool call begins. --- src/tui-opentui/turn-state.test.ts | 48 ++++++++++++++++++++++++++++-- src/tui-opentui/turn-state.ts | 17 +++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/src/tui-opentui/turn-state.test.ts b/src/tui-opentui/turn-state.test.ts index 07405a932..cf3d728cb 100644 --- a/src/tui-opentui/turn-state.test.ts +++ b/src/tui-opentui/turn-state.test.ts @@ -231,7 +231,10 @@ describe("repetition tracking", () => { expect(s.repeating).toBe(false) }) - test("repetition tracked across a tool cycle survives connector.reply with tools outstanding", () => { + test("a tool call ends the streaming cycle and clears the repetition buffer", () => { + // Repeats within one unbroken stream are a real loop; a tool call + // interrupting the stream is not part of that cycle, so it must not + // carry the accumulated repetition state into the next one. const deltas = Array(10) .fill(cycle) .map((text) => textDelta(text)) @@ -240,13 +243,52 @@ describe("repetition tracking", () => { { type: "tool.start", data: { call: { id: "c1", name: "grep" } } }, 100, ) + expect(withTool.repeating).toBe(false) + expect(withTool.streamText).toBe("") + const afterReply = turnStateFromEvent( withTool, { type: "connector.reply" }, 101, ) - expect(afterReply.repeating).toBe(true) - expect(afterReply.streamText.length).toBeGreaterThan(0) + expect(afterReply.repeating).toBe(false) + }) + + test("a short narration line repeated before each of nine tool calls is not a loop", () => { + // Verified false positive (CL-5577): "Let me check the next file now." + // fed in 4-char chunks before nine separate tool calls, interleaved with + // tool.start/connector.reply/tool.done, must not abort the turn. Nothing + // about saying a similar short thing before each of several tool calls + // in one turn is degenerate. + const narration = "Let me check the next file now." + const chunks: string[] = [] + for (let i = 0; i < narration.length; i += 4) { + chunks.push(narration.slice(i, i + 4)) + } + + let state = fold([{ type: "inference.start" }]) + let clock = 1 + for (let cycleIndex = 0; cycleIndex < 12; cycleIndex++) { + for (const chunk of chunks) { + state = turnStateFromEvent(state, textDelta(chunk), ++clock) + } + state = turnStateFromEvent( + state, + { + type: "tool.start", + data: { call: { id: `c${cycleIndex}`, name: "read_file" } }, + }, + ++clock, + ) + state = turnStateFromEvent(state, { type: "connector.reply" }, ++clock) + state = turnStateFromEvent( + state, + { type: "tool.done", data: { result: { callId: `c${cycleIndex}` } } }, + ++clock, + ) + expect(state.repeating).toBe(false) + } + expect(state.repeating).toBe(false) }) test("a fresh submit clears the repetition state", () => { diff --git a/src/tui-opentui/turn-state.ts b/src/tui-opentui/turn-state.ts index 688c5e0f5..d846e00c7 100644 --- a/src/tui-opentui/turn-state.ts +++ b/src/tui-opentui/turn-state.ts @@ -56,8 +56,11 @@ export type TurnState = { */ readonly activeToolCalls: readonly string[] /** - * Tail of the text/thinking output streamed this turn, across cycles — - * `connector.reply` with tools outstanding does not clear it. Bounded to + * Tail of the text/thinking output streamed in the current uninterrupted + * streaming cycle. A tool call ends the cycle and clears it: a model + * narrating a similar short line before each of several tool calls is + * ordinary and must not accumulate into an apparent loop, whereas a + * genuinely degenerate model repeats within one unbroken stream. Bounded to * `STREAM_TEXT_BUFFER_CHARS`; feeds `detectRepetition`, nothing else. */ readonly streamText: string @@ -268,6 +271,11 @@ const streaming = ( } } +// A tool call ends the current streaming cycle. Clearing the repetition +// buffer here, rather than only on a fresh turn, is what keeps repeats from +// accumulating across `connector.reply` boundaries — the mechanism that +// turned nine separate narration lines ("Let me check the next file now.") +// into one apparent loop and killed an ordinary turn mid-flight. const runningTool = ( state: TurnState, name: string | null, @@ -280,6 +288,11 @@ const runningTool = ( streamingType: "tool", currentToolName: name ?? state.currentToolName, lastActivityAt: nowMs, + streamText: "", + streamCharsSeen: 0, + repetitionCheckedAt: 0, + repeating: false, + repeatingSinceTokenCount: null, }) /** From bcc3bc8b1bc3e9dc59a5e5b8978b39b8a8a82a5b Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 7 Aug 2026 01:13:25 -0700 Subject: [PATCH 4/4] Fingerprint each cycle instead of discarding repetition state Clearing the repetition buffer on every tool call fixed the narration false positive but went too far: a model that loops while interleaving even a no-op tool call between repeats was no longer caught at all, since nothing carried across the reset. Keep a cheap fingerprint of each completed cycle instead of its raw text, and flag only once several consecutive cycles fingerprint alike. Narration varies enough cycle to cycle to clear that bar; an unvarying repeated block does not, and now trips within a small, bounded number of cycles rather than never. --- src/tui-opentui/turn-state.test.ts | 55 ++++++++++-- src/tui-opentui/turn-state.ts | 133 ++++++++++++++++++++++++----- 2 files changed, 161 insertions(+), 27 deletions(-) diff --git a/src/tui-opentui/turn-state.test.ts b/src/tui-opentui/turn-state.test.ts index cf3d728cb..f987c20e5 100644 --- a/src/tui-opentui/turn-state.test.ts +++ b/src/tui-opentui/turn-state.test.ts @@ -231,19 +231,24 @@ describe("repetition tracking", () => { expect(s.repeating).toBe(false) }) - test("a tool call ends the streaming cycle and clears the repetition buffer", () => { - // Repeats within one unbroken stream are a real loop; a tool call - // interrupting the stream is not part of that cycle, so it must not - // carry the accumulated repetition state into the next one. + test("a tool call ends the streaming cycle but does not un-latch a real detection", () => { + // The raw text buffer is discarded at the tool-call boundary (that is + // what keeps narration from accumulating into a false loop), but a real + // in-cycle detection that already fired must stay latched — the model + // did loop, and a coincidental tool call right after should not erase + // that fact. const deltas = Array(10) .fill(cycle) .map((text) => textDelta(text)) + const looping = fold([{ type: "inference.start" }, ...deltas]) + expect(looping.repeating).toBe(true) + const withTool = turnStateFromEvent( - fold([{ type: "inference.start" }, ...deltas]), + looping, { type: "tool.start", data: { call: { id: "c1", name: "grep" } } }, 100, ) - expect(withTool.repeating).toBe(false) + expect(withTool.repeating).toBe(true) expect(withTool.streamText).toBe("") const afterReply = turnStateFromEvent( @@ -251,7 +256,43 @@ describe("repetition tracking", () => { { type: "connector.reply" }, 101, ) - expect(afterReply.repeating).toBe(false) + expect(afterReply.repeating).toBe(true) + }) + + test("the same block repeated every cycle, interleaved with tool calls, still trips as a loop", () => { + // The gap this closes: an unconditional per-cycle reset (no cross-cycle + // memory at all) never catches a model that loops while interleaving a + // trivial tool call between every repeat — verified against a 500-cycle, + // 88,000-character run that never flipped `repeating`. A fingerprint of + // each completed cycle, compared to the one before it, catches this + // shape within a small, bounded number of cycles instead. + const block = "xk4mQ2 loop unit that never varies at all here" + expect(block.length).toBeGreaterThanOrEqual(24) + + let state = fold([{ type: "inference.start" }]) + let clock = 1 + let trippedAtCycle = -1 + for (let cycleIndex = 0; cycleIndex < 30; cycleIndex++) { + state = turnStateFromEvent(state, textDelta(block), ++clock) + state = turnStateFromEvent( + state, + { + type: "tool.start", + data: { call: { id: `c${cycleIndex}`, name: "noop" } }, + }, + ++clock, + ) + state = turnStateFromEvent(state, { type: "connector.reply" }, ++clock) + state = turnStateFromEvent( + state, + { type: "tool.done", data: { result: { callId: `c${cycleIndex}` } } }, + ++clock, + ) + if (trippedAtCycle === -1 && state.repeating) trippedAtCycle = cycleIndex + } + expect(state.repeating).toBe(true) + expect(trippedAtCycle).toBeGreaterThan(-1) + expect(trippedAtCycle).toBeLessThan(30) }) test("a short narration line repeated before each of nine tool calls is not a loop", () => { diff --git a/src/tui-opentui/turn-state.ts b/src/tui-opentui/turn-state.ts index d846e00c7..d0cb54cc8 100644 --- a/src/tui-opentui/turn-state.ts +++ b/src/tui-opentui/turn-state.ts @@ -27,6 +27,47 @@ const STREAM_TEXT_BUFFER_CHARS = 8_000 // cost proportional to output, not token count. const REPETITION_CHECK_INTERVAL_CHARS = 40 +// Cycles shorter than this are skipped when updating the cross-cycle streak: +// a bare tool call with no preceding text, or a one-word aside, is too little +// signal to compare — matching by coincidence is common at this length, and +// skipping neither breaks nor extends a streak already in progress. +const CYCLE_FINGERPRINT_MIN_CHARS = 24 + +// How many consecutive cycles must fingerprint identically before it counts +// as a loop rather than ordinary phrasing. The fingerprint covers the whole +// cycle's text, so any variation at all — a changing filename, index, or +// detail ("Editing src/module_47.ts next.") produces a different hash and +// never advances the streak, no matter how many cycles run. That is what +// makes this bar tolerable at a bare-number glance: it only ever governs +// content that is byte-for-byte invariant, cycle after cycle, which ordinary +// narration is not. The verified false positive (CL-5577) is a model saying +// the exact same short line before each of 9-12 separate tool calls in one +// turn — that must not abort, so the bar sits above that range with +// headroom. Set well below the reported repro (an unvarying 46-char block +// repeated every cycle for 500 cycles, which the unconditional-reset version +// never caught at all): at this bar the streak still trips a small fraction +// of the way in, a few thousand characters and under two dozen tool calls, +// not after 500 and 88,000 characters. The remaining exposure is narrow and +// explicit: an exact, invariant line of at least `CYCLE_FINGERPRINT_MIN_CHARS` +// chars repeated with zero variation for this many cycles running straight +// through tool calls — contentless boilerplate, not narration. +const CYCLE_REPETITION_MIN_CONSECUTIVE = 20 + +/** + * Cheap 32-bit fingerprint (FNV-1a) of one completed cycle's text, so the + * cross-cycle streak only has to remember a short string per turn rather than + * retain raw text across cycles — the retained text is exactly what caused + * the cross-cycle false positive this replaces. + */ +function cycleFingerprint(text: string): string { + let hash = 0x811c9dc5 + for (let i = 0; i < text.length; i++) { + hash ^= text.charCodeAt(i) + hash = Math.imul(hash, 0x01000193) + } + return (hash >>> 0).toString(16) +} + export type QuotaWait = { readonly retryAfterMs: number readonly retryAt: number @@ -80,6 +121,20 @@ export type TurnState = { * rather than the whole turn's count. */ readonly repeatingSinceTokenCount: number | null + /** + * Fingerprint of the most recently completed streaming cycle (set at each + * tool-call boundary), used only to compare against the next cycle's + * fingerprint. Not the raw text — carrying that across cycles is what + * caused repeats to accumulate into a false positive across tool calls. + */ + readonly cycleFingerprint: string | null + /** + * Consecutive completed cycles whose fingerprint matched the one before it. + * A model repeating the same block every cycle, with a tool call in + * between each, builds this streak even though no single cycle's text ever + * gets long enough to trip `detectRepetition` on its own. + */ + readonly consecutiveMatchingCycles: number } export function initialTurnState(nowMs: number): TurnState { @@ -98,6 +153,8 @@ export function initialTurnState(nowMs: number): TurnState { repetitionCheckedAt: 0, repeating: false, repeatingSinceTokenCount: null, + cycleFingerprint: null, + consecutiveMatchingCycles: 0, } } @@ -118,6 +175,8 @@ export function turnStateOnSubmit(state: TurnState, nowMs: number): TurnState { repetitionCheckedAt: 0, repeating: false, repeatingSinceTokenCount: null, + cycleFingerprint: null, + consecutiveMatchingCycles: 0, } } @@ -251,7 +310,12 @@ const streaming = ( const streamCharsSeen = state.streamCharsSeen + text.length const due = streamCharsSeen - state.repetitionCheckedAt >= REPETITION_CHECK_INTERVAL_CHARS - const repeating = due ? detectRepetition(streamText).repeating : state.repeating + // Once true, stays true for the rest of the turn — a fresh cycle's buffer + // starts empty (see `runningTool`) and would otherwise read back false on + // the next check, un-latching a real detection the moment a tool call + // interrupts the stream. + const repeating = + state.repeating || (due && detectRepetition(streamText).repeating) return { ...state, status: state.status === "blocked" ? "blocked" : "running", @@ -271,29 +335,58 @@ const streaming = ( } } -// A tool call ends the current streaming cycle. Clearing the repetition -// buffer here, rather than only on a fresh turn, is what keeps repeats from -// accumulating across `connector.reply` boundaries — the mechanism that -// turned nine separate narration lines ("Let me check the next file now.") -// into one apparent loop and killed an ordinary turn mid-flight. +// A tool call ends the current streaming cycle. The raw text buffer is +// discarded here, rather than only on a fresh turn, so repeats never +// accumulate across `connector.reply` boundaries — the mechanism that turned +// nine separate narration lines ("Let me check the next file now.") into one +// apparent loop and killed an ordinary turn mid-flight. But discarding the +// buffer outright would also erase a genuine loop that interleaves a tool +// call between every repeat of the same block, so a fingerprint of the +// completed cycle is kept and compared against the next one: several +// consecutive cycles fingerprinting alike is what that shape of loop looks +// like, and nine different narration lines never do. const runningTool = ( state: TurnState, name: string | null, nowMs: number, -): TurnState => ({ - ...state, - status: state.status === "blocked" ? "blocked" : "running", - isProcessing: true, - awaitingResponse: false, - streamingType: "tool", - currentToolName: name ?? state.currentToolName, - lastActivityAt: nowMs, - streamText: "", - streamCharsSeen: 0, - repetitionCheckedAt: 0, - repeating: false, - repeatingSinceTokenCount: null, -}) +): TurnState => { + const cycleText = state.streamText + const longEnoughToCompare = cycleText.length >= CYCLE_FINGERPRINT_MIN_CHARS + const fingerprint = longEnoughToCompare + ? cycleFingerprint(cycleText) + : null + const matchedPrevious = + longEnoughToCompare && + state.cycleFingerprint !== null && + fingerprint === state.cycleFingerprint + const consecutiveMatchingCycles = matchedPrevious + ? state.consecutiveMatchingCycles + 1 + : longEnoughToCompare + ? 1 + : state.consecutiveMatchingCycles + const repeating = + state.repeating || consecutiveMatchingCycles >= CYCLE_REPETITION_MIN_CONSECUTIVE + + return { + ...state, + status: state.status === "blocked" ? "blocked" : "running", + isProcessing: true, + awaitingResponse: false, + streamingType: "tool", + currentToolName: name ?? state.currentToolName, + lastActivityAt: nowMs, + streamText: "", + streamCharsSeen: 0, + repetitionCheckedAt: 0, + repeating, + repeatingSinceTokenCount: + repeating && state.repeatingSinceTokenCount === null + ? state.streamTokenCount + : state.repeatingSinceTokenCount, + cycleFingerprint: longEnoughToCompare ? fingerprint : state.cycleFingerprint, + consecutiveMatchingCycles, + } +} /** * Fold one inbound event (reactor-shaped or canonical bridge-shaped) into the