From 140324d7a7425ba6d0c90b43d02347cbb215c6e8 Mon Sep 17 00:00:00 2001 From: Benjamin Bachmann Date: Mon, 21 Sep 2026 23:52:35 +0200 Subject: [PATCH] feat: add Jev turn routing --- apps/server/src/cli.ts | 40 ++++++ apps/server/src/config.ts | 3 + apps/server/src/jevTurnRouter.test.ts | 95 ++++++++++++++ apps/server/src/jevTurnRouter.ts | 176 ++++++++++++++++++++++++++ apps/server/src/ws.ts | 80 +++++++++--- docs/jev-turn-routing.md | 41 ++++++ 6 files changed, 416 insertions(+), 19 deletions(-) create mode 100644 apps/server/src/jevTurnRouter.test.ts create mode 100644 apps/server/src/jevTurnRouter.ts create mode 100644 docs/jev-turn-routing.md diff --git a/apps/server/src/cli.ts b/apps/server/src/cli.ts index 9ece02a0d3f3..10209fe5b544 100644 --- a/apps/server/src/cli.ts +++ b/apps/server/src/cli.ts @@ -15,6 +15,7 @@ import { import { readBootstrapEnvelope } from "./bootstrap"; import { expandHomePath, resolveBaseDir } from "./os-jank"; import { runServer } from "./server"; +import type { JevTurnRouterConfig, JevTurnRouterMode } from "./jevTurnRouter"; const PortSchema = Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65535 })); @@ -133,6 +134,30 @@ const EnvServerConfig = Config.all({ Config.option, Config.map(Option.getOrUndefined), ), + jevApiKey: Config.string("T3CODE_JEV_API_KEY").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + jevMode: Config.string("T3CODE_JEV_MODE").pipe(Config.withDefault("off")), + jevApiUrl: Config.string("T3CODE_JEV_API_URL").pipe( + Config.withDefault("https://api.typesafe.ai/v1/systemone"), + ), + jevTimeoutMs: Config.int("T3CODE_JEV_TIMEOUT_MS").pipe(Config.withDefault(1500)), + jevConfidenceThreshold: Config.number("T3CODE_JEV_CONFIDENCE_THRESHOLD").pipe( + Config.withDefault(0.8), + ), + jevModelSmall: Config.string("T3CODE_JEV_MODEL_SMALL").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + jevModelNormal: Config.string("T3CODE_JEV_MODEL_NORMAL").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), + jevModelExpert: Config.string("T3CODE_JEV_MODEL_EXPERT").pipe( + Config.option, + Config.map(Option.getOrUndefined), + ), }); interface CliServerFlags { @@ -291,6 +316,20 @@ export const resolveServerConfig = ( () => (mode === "desktop" ? "127.0.0.1" : undefined), ); const logLevel = Option.getOrElse(cliLogLevel, () => env.logLevel); + const jevMode: JevTurnRouterMode = + env.jevMode === "shadow" || env.jevMode === "apply" ? env.jevMode : "off"; + const jevTurnRouter: JevTurnRouterConfig = { + mode: jevMode, + ...(env.jevApiKey ? { apiKey: env.jevApiKey } : {}), + apiUrl: env.jevApiUrl, + timeoutMs: Math.max(100, env.jevTimeoutMs), + confidenceThreshold: Math.min(1, Math.max(0, env.jevConfidenceThreshold)), + models: { + ...(env.jevModelSmall ? { small: env.jevModelSmall } : {}), + ...(env.jevModelNormal ? { normal: env.jevModelNormal } : {}), + ...(env.jevModelExpert ? { expert: env.jevModelExpert } : {}), + }, + }; const config: ServerConfigShape = { logLevel, @@ -330,6 +369,7 @@ export const resolveServerConfig = ( authToken, autoBootstrapProjectFromCwd, logWebSocketEvents, + jevTurnRouter, }; return config; diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index 9ceea4c13cd1..be0c2c116147 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -8,6 +8,8 @@ */ import { Effect, FileSystem, Layer, LogLevel, Path, Schema, ServiceMap } from "effect"; +import type { JevTurnRouterConfig } from "./jevTurnRouter"; + export const DEFAULT_PORT = 3773; export const RuntimeMode = Schema.Literals(["web", "desktop"]); @@ -57,6 +59,7 @@ export interface ServerConfigShape extends ServerDerivedPaths { readonly authToken: string | undefined; readonly autoBootstrapProjectFromCwd: boolean; readonly logWebSocketEvents: boolean; + readonly jevTurnRouter?: JevTurnRouterConfig; } export const deriveServerPaths = Effect.fn(function* ( diff --git a/apps/server/src/jevTurnRouter.test.ts b/apps/server/src/jevTurnRouter.test.ts new file mode 100644 index 000000000000..d357ef95786c --- /dev/null +++ b/apps/server/src/jevTurnRouter.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import { CommandId, MessageId, ThreadId, type OrchestrationCommand } from "@t3tools/contracts"; + +import { routeTurnWithJev, type JevTurnRouterConfig } from "./jevTurnRouter"; + +const command = { + type: "thread.turn.start", + commandId: CommandId.makeUnsafe("command:test"), + threadId: ThreadId.makeUnsafe("thread:test"), + message: { + messageId: MessageId.makeUnsafe("message:test"), + role: "user", + text: "Diagnose the failing test and implement the fix.", + attachments: [], + }, + modelSelection: { provider: "codex", model: "gpt-5.6-sol" }, + runtimeMode: "full-access", + interactionMode: "default", + createdAt: "2026-09-21T10:00:00.000Z", +} as Extract; + +const config: JevTurnRouterConfig = { + mode: "shadow", + apiKey: "test-key", + apiUrl: "https://api.typesafe.ai/v1/systemone", + timeoutMs: 100, + confidenceThreshold: 0.8, + models: { + small: "gpt-5.6-luna", + normal: "gpt-5.6-terra", + expert: "gpt-5.6-sol", + }, +}; + +const response = (tier: "small" | "normal" | "expert", confidence: number) => + new Response( + JSON.stringify({ + answers: { model_tier: { type: "choice", choice: tier, confidence } }, + usage: { input_tokens: 120, output_tokens: 20 }, + }), + { status: 200 }, + ); + +describe("routeTurnWithJev", () => { + it("reports a shadow recommendation without changing the command", async () => { + let capturedRequest: RequestInit | undefined; + const fetchImplementation = async (_input: string | URL | Request, init?: RequestInit) => { + capturedRequest = init; + return response("normal", 0.91); + }; + const result = await routeTurnWithJev(command, config, fetchImplementation); + + expect(result.command.modelSelection?.model).toBe("gpt-5.6-sol"); + expect(result.recommendation).toMatchObject({ + status: "recommended", + tier: "normal", + confidence: 0.91, + recommendedModel: "gpt-5.6-terra", + usage: { inputTokens: 120, outputTokens: 20 }, + }); + expect(capturedRequest?.headers).toMatchObject({ Authorization: "Bearer test-key" }); + expect(capturedRequest?.body).not.toContain("test-key"); + }); + + it("applies a confident recommendation for only this turn", async () => { + const result = await routeTurnWithJev(command, { ...config, mode: "apply" }, async () => + response("small", 0.95), + ); + + expect(result.recommendation.status).toBe("applied"); + expect(result.command.modelSelection?.model).toBe("gpt-5.6-luna"); + expect(command.modelSelection?.model).toBe("gpt-5.6-sol"); + }); + + it("keeps the selected model below the confidence threshold", async () => { + const result = await routeTurnWithJev(command, { ...config, mode: "apply" }, async () => + response("small", 0.7), + ); + + expect(result.recommendation).toMatchObject({ + status: "fallback", + reason: "confidence below threshold", + }); + expect(result.command.modelSelection?.model).toBe("gpt-5.6-sol"); + }); + + it("fails open when Jev is unavailable", async () => { + const result = await routeTurnWithJev(command, config, async () => { + throw new Error("offline"); + }); + + expect(result.recommendation).toMatchObject({ status: "fallback", reason: "Jev unavailable" }); + expect(result.command.modelSelection?.model).toBe("gpt-5.6-sol"); + }); +}); diff --git a/apps/server/src/jevTurnRouter.ts b/apps/server/src/jevTurnRouter.ts new file mode 100644 index 000000000000..730b2400194a --- /dev/null +++ b/apps/server/src/jevTurnRouter.ts @@ -0,0 +1,176 @@ +import type { ModelSelection, OrchestrationCommand } from "@t3tools/contracts"; + +export type JevTurnRouterMode = "off" | "shadow" | "apply"; +export type JevModelTier = "small" | "normal" | "expert"; + +export interface JevTurnRouterConfig { + readonly mode: JevTurnRouterMode; + readonly apiKey?: string; + readonly apiUrl: string; + readonly timeoutMs: number; + readonly confidenceThreshold: number; + readonly models: Partial>; +} + +export interface JevTurnRouteResult { + readonly command: Extract; + readonly recommendation: { + readonly status: "disabled" | "recommended" | "applied" | "fallback"; + readonly tier?: JevModelTier; + readonly confidence?: number; + readonly originalModel?: string; + readonly recommendedModel?: string; + readonly reason?: string; + readonly latencyMs: number; + readonly usage?: { readonly inputTokens?: number; readonly outputTokens?: number }; + }; +} + +type FetchImplementation = (input: string | URL | Request, init?: RequestInit) => Promise; + +const MODEL_TIER_CRITERIA: Record = { + small: "A bounded, low-risk request answerable with light reasoning and few tool steps", + normal: "A typical coding or investigation task requiring repository context and tools", + expert: "A complex, ambiguous, high-risk, or cross-system task requiring deep reasoning", +}; +const MAX_ROUTING_TEXT_CHARS = 12_000; + +const isModelTier = (value: unknown): value is JevModelTier => + value === "small" || value === "normal" || value === "expert"; + +const readUsage = (value: unknown) => { + if (typeof value !== "object" || value === null) return undefined; + const usage = value as { input_tokens?: unknown; output_tokens?: unknown }; + const inputTokens = typeof usage.input_tokens === "number" ? usage.input_tokens : undefined; + const outputTokens = typeof usage.output_tokens === "number" ? usage.output_tokens : undefined; + return inputTokens === undefined && outputTokens === undefined + ? undefined + : { + ...(inputTokens !== undefined ? { inputTokens } : {}), + ...(outputTokens !== undefined ? { outputTokens } : {}), + }; +}; + +const readRecommendation = (value: unknown) => { + if (typeof value !== "object" || value === null) return null; + const response = value as { answers?: unknown; usage?: unknown }; + if (typeof response.answers !== "object" || response.answers === null) return null; + const answer = (response.answers as { model_tier?: unknown }).model_tier; + if (typeof answer !== "object" || answer === null) return null; + const choice = (answer as { choice?: unknown }).choice; + const confidence = (answer as { confidence?: unknown }).confidence; + if (!isModelTier(choice) || typeof confidence !== "number") return null; + return { tier: choice, confidence, usage: readUsage(response.usage) }; +}; + +export async function routeTurnWithJev( + command: Extract, + config: JevTurnRouterConfig, + fetchImplementation: FetchImplementation = fetch, +): Promise { + const startedAt = performance.now(); + const originalModel = command.modelSelection?.model; + const finish = ( + recommendation: Omit, + routedCommand = command, + ): JevTurnRouteResult => ({ + command: routedCommand, + recommendation: { + ...recommendation, + latencyMs: Math.max(0, Math.round(performance.now() - startedAt)), + }, + }); + + const originalModelDetails = originalModel ? { originalModel } : {}; + if (config.mode === "off") return finish({ status: "disabled", ...originalModelDetails }); + if (!config.apiKey) { + return finish({ + status: "fallback", + ...originalModelDetails, + reason: "missing API key", + }); + } + if (!command.modelSelection) { + return finish({ status: "fallback", reason: "turn has no model selection" }); + } + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), config.timeoutMs); + try { + const response = await fetchImplementation(config.apiUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${config.apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: "jev-latest", + state: { + source: "t3", + request: command.message.text.slice(0, MAX_ROUTING_TEXT_CHARS), + request_truncated: command.message.text.length > MAX_ROUTING_TEXT_CHARS, + provider: command.modelSelection.provider, + current_model: command.modelSelection.model, + interaction_mode: command.interactionMode, + }, + questions: { + model_tier: { + type: "choice", + instructions: "Choose the lowest model tier likely to complete `request` correctly.", + criteria: MODEL_TIER_CRITERIA, + }, + }, + }), + signal: controller.signal, + }); + if (!response.ok) { + return finish({ + status: "fallback", + ...originalModelDetails, + reason: `Jev returned HTTP ${response.status}`, + }); + } + const parsed = readRecommendation(await response.json()); + if (!parsed) { + return finish({ + status: "fallback", + ...originalModelDetails, + reason: "invalid Jev response", + }); + } + + const recommendedModel = config.models[parsed.tier]; + const base = { + tier: parsed.tier, + confidence: parsed.confidence, + ...originalModelDetails, + ...(recommendedModel ? { recommendedModel } : {}), + ...(parsed.usage ? { usage: parsed.usage } : {}), + }; + if (config.mode !== "apply") return finish({ status: "recommended", ...base }); + if (!recommendedModel) { + return finish({ status: "fallback", ...base, reason: "tier has no model mapping" }); + } + if (parsed.confidence < config.confidenceThreshold) { + return finish({ status: "fallback", ...base, reason: "confidence below threshold" }); + } + + const modelSelection: ModelSelection = { + ...command.modelSelection, + model: recommendedModel, + }; + return finish( + { status: "applied", ...base }, + { + ...command, + modelSelection, + }, + ); + } catch (error) { + const reason = + error instanceof Error && error.name === "AbortError" ? "timeout" : "Jev unavailable"; + return finish({ status: "fallback", ...originalModelDetails, reason }); + } finally { + clearTimeout(timeout); + } +} diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 1ad16548a9c2..8779e5d3a447 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -53,6 +53,7 @@ import { ProjectSetupScriptRunner } from "./project/Services/ProjectSetupScriptR import { ReviewCommentRepository } from "./persistence/Services/ReviewCommentRepository"; import { ReviewRequestRepository } from "./persistence/Services/ReviewRequestRepository"; import { GitHubCli } from "./git/Services/GitHubCli"; +import { routeTurnWithJev, type JevTurnRouteResult } from "./jevTurnRouter"; const WsRpcLayer = WsRpcGroup.toLayer( Effect.gen(function* () { @@ -100,6 +101,33 @@ const WsRpcLayer = WsRpcGroup.toLayer( createdAt: input.createdAt, }); + const appendJevRoutingActivity = ( + command: Extract, + recommendation: JevTurnRouteResult["recommendation"], + ) => + orchestrationEngine.dispatch({ + type: "thread.activity.append", + commandId: serverCommandId("jev-routing-activity"), + threadId: command.threadId, + activity: { + id: EventId.makeUnsafe(crypto.randomUUID()), + tone: recommendation.status === "fallback" ? "error" : "info", + kind: "jev.turn-routing", + summary: + recommendation.status === "applied" + ? `Jev selected ${recommendation.recommendedModel ?? recommendation.tier ?? "a model"}` + : recommendation.status === "recommended" + ? `Jev recommends ${recommendation.recommendedModel ?? recommendation.tier ?? "a model"}` + : recommendation.status === "fallback" + ? `Jev fallback: ${recommendation.reason ?? "unknown reason"}` + : "Jev routing disabled", + payload: recommendation, + turnId: null, + createdAt: command.createdAt, + }, + createdAt: command.createdAt, + }); + const toDispatchCommandError = (cause: unknown, fallbackMessage: string) => Schema.is(OrchestrationDispatchCommandError)(cause) ? cause @@ -310,26 +338,40 @@ const WsRpcLayer = WsRpcGroup.toLayer( const dispatchNormalizedCommand = ( normalizedCommand: OrchestrationCommand, - ): Effect.Effect<{ readonly sequence: number }, OrchestrationDispatchCommandError> => { - const dispatchEffect = - normalizedCommand.type === "thread.turn.start" && normalizedCommand.bootstrap - ? dispatchBootstrapTurnStart(normalizedCommand) - : orchestrationEngine - .dispatch(normalizedCommand) - .pipe( - Effect.mapError((cause) => - toDispatchCommandError(cause, "Failed to dispatch orchestration command"), - ), - ); + ): Effect.Effect<{ readonly sequence: number }, OrchestrationDispatchCommandError> => + Effect.gen(function* () { + const routing = + normalizedCommand.type === "thread.turn.start" && config.jevTurnRouter + ? yield* Effect.promise(() => + routeTurnWithJev(normalizedCommand, config.jevTurnRouter!), + ) + : null; + const command = routing?.command ?? normalizedCommand; + const dispatchEffect = + command.type === "thread.turn.start" && command.bootstrap + ? dispatchBootstrapTurnStart(command) + : orchestrationEngine + .dispatch(command) + .pipe( + Effect.mapError((cause) => + toDispatchCommandError(cause, "Failed to dispatch orchestration command"), + ), + ); - return startup - .enqueueCommand(dispatchEffect) - .pipe( - Effect.mapError((cause) => - toDispatchCommandError(cause, "Failed to dispatch orchestration command"), - ), - ); - }; + const result = yield* startup + .enqueueCommand(dispatchEffect) + .pipe( + Effect.mapError((cause) => + toDispatchCommandError(cause, "Failed to dispatch orchestration command"), + ), + ); + if (routing && routing.recommendation.status !== "disabled") { + yield* appendJevRoutingActivity(routing.command, routing.recommendation).pipe( + Effect.ignoreCause({ log: true }), + ); + } + return result; + }); const reviewCommentRepo = yield* ReviewCommentRepository; const reviewRequestRepo = yield* ReviewRequestRepository; const gitHubCli = yield* GitHubCli; diff --git a/docs/jev-turn-routing.md b/docs/jev-turn-routing.md new file mode 100644 index 000000000000..e7f8ae92dbdb --- /dev/null +++ b/docs/jev-turn-routing.md @@ -0,0 +1,41 @@ +# Jev turn routing + +T3 can optionally ask TypeSafe Jev for a model tier before dispatching a +`thread.turn.start` command. Routing is disabled by default and always fails +open to the model selected in the composer. + +The server sends the first 12,000 characters of the user request, the current +provider/model, and the interaction mode. API keys stay server-side and are +never included in activities or request bodies. + +## Configuration + +| Environment variable | Default | Purpose | +| --------------------------------- | ---------------------------- | ---------------------------------- | +| `T3CODE_JEV_MODE` | `off` | `off`, `shadow`, or `apply` | +| `T3CODE_JEV_API_KEY` | unset | TypeSafe API key | +| `T3CODE_JEV_API_URL` | TypeSafe System One endpoint | API override | +| `T3CODE_JEV_TIMEOUT_MS` | `1500` | Fail-open request timeout | +| `T3CODE_JEV_CONFIDENCE_THRESHOLD` | `0.8` | Minimum confidence in `apply` mode | +| `T3CODE_JEV_MODEL_SMALL` | unset | Model slug for the `small` tier | +| `T3CODE_JEV_MODEL_NORMAL` | unset | Model slug for the `normal` tier | +| `T3CODE_JEV_MODEL_EXPERT` | unset | Model slug for the `expert` tier | + +Start with `shadow`. Each routed turn gets a `jev.turn-routing` activity with +the recommendation, confidence, latency, token usage, and chosen mapping. +The selected model is unchanged. In `apply`, T3 changes only the current turn +when the tier has a configured model and confidence meets the threshold. + +Example: + +```bash +T3CODE_JEV_MODE=shadow \ +T3CODE_JEV_API_KEY="$TYPESAFE_API_KEY" \ +T3CODE_JEV_MODEL_SMALL=gpt-5.6-luna \ +T3CODE_JEV_MODEL_NORMAL=gpt-5.6-terra \ +T3CODE_JEV_MODEL_EXPERT=gpt-5.6-sol \ +bun run --cwd apps/server dev +``` + +Use a service manager or secret store for production configuration. Do not put +the API key in a repository, command-line argument, or browser-side setting.