Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/agent/retry-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,23 @@ describe("createCorbitsRetryPolicy", () => {
raw: { error: { message: "Too Many Requests" } },
},
});
// Remapped to retryable → default backoff, not abort on moderate Retry-After.
// Remapped to retryable -> default backoff, not abort on moderate Retry-After.
expect(decision).toEqual({ kind: "retry", delayMs: 500 });
});

test("stamped Codex usage-limit 429 retries as retryable, not long-quota abort", async () => {
const policy = createCorbitsRetryPolicy({ providerId: "codex/abk-labs" });
const decision = await policy({
attempt: 1,
elapsedMs: 0,
error: {
category: "quota_exhausted",
message: "You have hit your ChatGPT usage limit",
statusCode: 429,
retryAfterMs: 45_000,
raw: "You have hit your ChatGPT usage limit",
},
});
expect(decision).toEqual({ kind: "retry", delayMs: 500 });
});

Expand Down
23 changes: 23 additions & 0 deletions src/inference-error-message.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,27 @@ describe("inferenceErrorMessage", () => {
});
expect(line).toBe("Quota exhausted — usage limit reached.");
});

test("known-Codex short 429 shows rate-limit line, not usage-limit copy", () => {
const line = inferenceErrorMessage({
category: "quota_exhausted",
message: "You have hit your ChatGPT usage limit",
statusCode: 429,
providerId: "codex/abk-labs",
raw: "You have hit your ChatGPT usage limit",
});
expect(line.toLowerCase()).toMatch(/rate limit/);
expect(line).not.toContain("Quota exhausted");
expect(line.toLowerCase()).not.toContain("the usage limit has been reached");
expect(line.toLowerCase()).not.toContain("usage limit reached");
});

test("credential_failure tells the user to log in again", () => {
const line = inferenceErrorMessage({
category: "credential_failure",
message: '{"error":{"code":401}}',
});
expect(line.toLowerCase()).not.toContain("re-authenticating");
expect(line.toLowerCase()).toMatch(/log in again|sign in again/);
});
});
14 changes: 8 additions & 6 deletions src/inference-error-message.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ import {
import { codexProfileFromProviderName, isCodexProviderName } from "./config/codex-providers.js";
import {
gatewayOverloadUserMessage,
isCodexShortRateLimitInferenceError,
isGatewayOverloadInferenceError,
isXaiShortRateLimitInferenceError,
XAI_RATE_LIMIT_USER_MESSAGE,
RATE_LIMIT_USER_MESSAGE,
type InferenceErrorLike,
} from "./inference-gateway-error.js";

const FRIENDLY_BY_CATEGORY: Record<string, string> = {
// Re-authentication runs on its own; keep the transcript line short and free
// of the provider's raw 401 JSON.
credential_failure: "Session expired — re-authenticating…",
// Committed auth death — do not claim a refresh is in flight.
credential_failure: "Authentication failed — log in again.",
quota_exhausted: "Quota exhausted — usage limit reached.",
context_overflow:
"Context window full — compaction could not keep up. Try /clear to start fresh.",
Expand Down Expand Up @@ -93,8 +93,10 @@ function codexUsageLimitLine(error: InferenceErrorLike): string | undefined {
export function inferenceErrorMessage(error: InferenceErrorLike): string {
if (isGatewayOverloadInferenceError(error)) return gatewayOverloadUserMessage(error);
// Dual-path: harness may still emit intx's quota_exhausted for a known-xAI
// short 429; FRIENDLY_BY_CATEGORY would otherwise say "Quota exhausted".
if (isXaiShortRateLimitInferenceError(error)) return XAI_RATE_LIMIT_USER_MESSAGE;
// or known-Codex short 429; FRIENDLY_BY_CATEGORY would otherwise say
// "Quota exhausted".
if (isXaiShortRateLimitInferenceError(error) || isCodexShortRateLimitInferenceError(error))
return RATE_LIMIT_USER_MESSAGE;

const category = classifyInferenceErrorCategory(error);
if (category === "quota_exhausted") {
Expand Down
66 changes: 66 additions & 0 deletions src/inference-gateway-error.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -318,4 +318,70 @@ describe("normalizeInferenceErrorForRetry", () => {
};
expect(normalizeInferenceErrorForRetry(err)).toEqual(err);
});

test("known-Codex bare 429 without usage_limit_reached remaps to retryable", () => {
const bare = {
category: "quota_exhausted" as const,
message: "Too Many Requests",
statusCode: 429,
retryAfterMs: 5_000,
raw: { error: { message: "Too Many Requests" } },
};

expect(normalizeInferenceErrorForRetry(bare)).toEqual(bare);

const viaProviderId = normalizeInferenceErrorForRetry({
...bare,
providerId: "codex/abk-labs",
});
expect(viaProviderId.category).toBe("retryable");
expect(viaProviderId.retryAfterMs).toBe(5_000);
expect(viaProviderId.message.toLowerCase()).toMatch(/rate limit/);
expect(viaProviderId.message.toLowerCase()).not.toMatch(/quota exhausted|usage limit reached/);
});

test("known-Codex 429 with ChatGPT usage-limit prose remaps to retryable", () => {
const normalized = normalizeInferenceErrorForRetry({
category: "quota_exhausted",
message: "You have hit your ChatGPT usage limit",
statusCode: 429,
providerId: "codex/abk-labs",
raw: "You have hit your ChatGPT usage limit",
});
expect(normalized.category).toBe("retryable");
expect(normalized.message.toLowerCase()).toMatch(/rate limit/);
expect(normalized.message.toLowerCase()).not.toMatch(/quota exhausted|usage limit reached/);
});

test("known-Codex 429 with empty body remaps to retryable", () => {
const normalized = normalizeInferenceErrorForRetry({
category: "quota_exhausted",
message: "Too Many Requests",
statusCode: 429,
providerId: "codex/abk-labs",
});
expect(normalized.category).toBe("retryable");
});

test("known-Codex usage_limit_reached 429 stays quota_exhausted", () => {
const normalized = normalizeInferenceErrorForRetry({
category: "quota_exhausted",
message: "Too Many Requests",
statusCode: 429,
providerId: "codex/abk-labs",
raw: {
detail: {
error: {
code: "usage_limit_reached",
message: "You have reached your usage limit. Try again later.",
plan_type: "workspace_member",
resets_in_seconds: 3435,
},
},
},
});
expect(normalized.category).toBe("quota_exhausted");
expect(normalized.retryAfterMs).toBe(3_435_000);
expect(normalized.message).toContain('Codex profile "abk-labs"');
});
});
84 changes: 69 additions & 15 deletions src/inference-gateway-error.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ const GATEWAY_OVERLOAD_TEXT_MARKERS = [
/** User-visible line while the harness retries a transient gateway overload. */
export const GATEWAY_OVERLOAD_USER_MESSAGE = "Inference gateway overloaded — retrying…";

/** User-visible line while the harness retries a short known-xAI HTTP 429. */
export const XAI_RATE_LIMIT_USER_MESSAGE = "Rate limited — retrying…";
/** User-visible line while the harness retries a short known-provider HTTP 429. */
export const RATE_LIMIT_USER_MESSAGE = "Rate limited — retrying…";

/** Body markers that mean a real usage/quota window, not a short rate limit. */
const XAI_QUOTA_BODY_MARKERS = [
Expand Down Expand Up @@ -246,7 +246,67 @@ export function normalizeXaiRateLimitError(error: InferenceErrorWithGoContext):

return {
category: "retryable",
message: XAI_RATE_LIMIT_USER_MESSAGE,
message: RATE_LIMIT_USER_MESSAGE,
statusCode: 429,
...(error.raw !== undefined ? { raw: error.raw } : {}),
...(error.retryAfterMs !== undefined ? { retryAfterMs: error.retryAfterMs } : {}),
};
}

function parseCodexUsageLimitFromError(
error: InferenceErrorLike,
): ReturnType<typeof parseCodexUsageLimitError> {
const candidates: unknown[] = [];
if (error.raw !== undefined) candidates.push(error.raw);
if (typeof error.message === "string" && error.message.trim().startsWith("{")) {
candidates.push(error.message);
}

for (const candidate of candidates) {
const parsed = parseCodexUsageLimitError(candidate);
if (parsed !== undefined) return parsed;
}
return undefined;
}

function isKnownCodexProviderId(providerId: string | undefined): boolean {
return providerId !== undefined && isCodexProviderName(providerId);
}

/**
* True when a known-Codex HTTP 429 looks like a short rate limit rather than a
* `usage_limit_reached` window. Used by both retry normalization and transcript
* copy — FRIENDLY_BY_CATEGORY would otherwise paint every quota_exhausted 429 as
* "Quota exhausted" even when the policy remaps it to retryable.
*
* Discrimination is the existing Codex usage-limit parser, not Retry-After length
* and not ChatGPT usage-limit prose without `usage_limit_reached`.
*/
export function isCodexShortRateLimitInferenceError(error: InferenceErrorLike): boolean {
if (!isKnownCodexProviderId(error.providerId)) return false;
if (error.statusCode !== 429) return false;
if (error.category !== "quota_exhausted" && error.category !== "retryable") return false;
if (parseCodexUsageLimitFromError(error) !== undefined) return false;
return true;
}

/**
* intx defaults bare 429 → quota_exhausted. For known-Codex contexts a bare 429
* (or usage-limit prose without `usage_limit_reached`) reclassifies as retryable
* so short ChatGPT 429s are not painted as a committed usage-limit window.
*
* Nested `detail.error.code === usage_limit_reached` stays quota_exhausted via
* `normalizeCodexUsageLimitError`. Unknown / non-Codex providers are never remapped.
*/
export function normalizeCodexRateLimitError(error: InferenceErrorWithGoContext): InferenceError {
if (error.statusCode !== 429) return error;
if (error.category !== "quota_exhausted") return error;
if (!isKnownCodexProviderId(error.providerId)) return error;
if (parseCodexUsageLimitFromError(error) !== undefined) return error;

return {
category: "retryable",
message: RATE_LIMIT_USER_MESSAGE,
statusCode: 429,
...(error.raw !== undefined ? { raw: error.raw } : {}),
...(error.retryAfterMs !== undefined ? { retryAfterMs: error.retryAfterMs } : {}),
Expand All @@ -266,17 +326,7 @@ function normalizeCodexUsageLimitError(error: InferenceErrorWithGoContext): Infe
return error;
}

const candidates: unknown[] = [];
if (error.raw !== undefined) candidates.push(error.raw);
if (typeof error.message === "string" && error.message.trim().startsWith("{")) {
candidates.push(error.message);
}

let parsed = undefined as ReturnType<typeof parseCodexUsageLimitError>;
for (const candidate of candidates) {
parsed = parseCodexUsageLimitError(candidate);
if (parsed !== undefined) break;
}
const parsed = parseCodexUsageLimitFromError(error);
if (parsed === undefined) return error;

const profile =
Expand All @@ -298,7 +348,8 @@ function normalizeCodexUsageLimitError(error: InferenceErrorWithGoContext): Infe
* Reclassify gateway overload errors so the default retry policy treats them as
* transient instead of aborting on protocol_mismatch. Also normalizes OpenCode
* Go quota/rate-limit shapes (including HTTP 400 mis-status), known-xAI short
* 429s, and Codex usage limits (nested detail.error with resets_in_seconds).
* 429s, Codex usage limits (nested detail.error with resets_in_seconds), and
* known-Codex short 429s that are not usage_limit_reached.
*/
export function normalizeInferenceErrorForRetry(
error: InferenceErrorWithGoContext,
Expand All @@ -312,6 +363,9 @@ export function normalizeInferenceErrorForRetry(
const codexNormalized = normalizeCodexUsageLimitError(error);
if (codexNormalized !== error) return codexNormalized;

const codexRateLimit = normalizeCodexRateLimitError(error);
if (codexRateLimit !== error) return codexRateLimit;

if (!isGatewayOverloadInferenceError(error)) return error;
if (error.category === "retryable" || error.category === "timeout") return error;

Expand Down
25 changes: 24 additions & 1 deletion src/tui/stream-event-map.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ describe("inference.error text", () => {

test("a classified failure gets its written line, not the provider body", () => {
expect(message({ category: "credential_failure", message: '{"error":{"code":401}}' })).toBe(
"Session expiredre-authenticating…",
"Authentication failedlog in again.",
);
expect(message({ category: "quota_exhausted", message: "429" })).toBe(
"Quota exhausted — usage limit reached.",
Expand Down Expand Up @@ -371,6 +371,29 @@ describe("inference.error text", () => {
expect(event.message).not.toContain("Quota exhausted");
});

test("ctx.providerId Codex + ChatGPT usage-limit 429 shows rate-limit copy", () => {
const ctx = createStreamMapContext({ providerId: "codex/abk-labs" });
const [event] = mapProductionEvent(
{
type: "inference.error",
data: {
error: {
category: "quota_exhausted",
message: "You have hit your ChatGPT usage limit",
statusCode: 429,
raw: "You have hit your ChatGPT usage limit",
},
},
},
ctx,
);
expect(event?.type).toBe("error");
if (event?.type !== "error") return;
expect(event.message.toLowerCase()).toMatch(/rate limit/);
expect(event.message).not.toContain("Quota exhausted");
expect(event.message.toLowerCase()).not.toContain("usage limit reached");
});

test("bare quota_exhausted 429 without ctx/provider still shows Quota exhausted", () => {
expect(
message({
Expand Down
Loading