From 7765f12639702d12070a19861944d1edf43ea9de Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 1 Sep 2026 15:03:21 +0200 Subject: [PATCH] fix(mount): classify transport rejections from the launch and daemon-start execs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `startMount` wrapped its mkdir and status-probe execs so a rejected transport becomes a classified error, but left the initial-sync launch and the daemon start unwrapped. Every throw in this function that embeds sandbox output is one callers recognise by message prefix, so those two were the only paths that could produce an error matching no prefix at all — and callers bucket that as "invocation unknown" and discard the cause. That is not hypothetical. Cloud is currently failing 100% of JIT provisions with `relayfile_mount_invocation_unknown` after ~122s, which matches Daytona's ~120s proxy read timeout, and the underlying error was unrecoverable from either the logs or the HTTP body. See #45. Wrap both in the same shape the two already-wrapped execs use, preserving the original error as `cause`. A rejection now lands on `Failed to launch relayfile initial sync:` or `Failed to start relayfile mount:` respectively, which callers already map to named phases. This makes the failure legible; it does not by itself stop the exec from hanging. #45 stays open for the hang. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AGB949mfHcP68whBJebzEM Session-Id: 1fa09ce6-c8d4-4c4a-ad2b-0f1999cb237d --- src/orchestrator.start-mount.test.ts | 66 ++++++++++++++++++++++++++++ src/orchestrator.ts | 60 ++++++++++++++++--------- 2 files changed, 105 insertions(+), 21 deletions(-) diff --git a/src/orchestrator.start-mount.test.ts b/src/orchestrator.start-mount.test.ts index b7f6fb1..53c7fbb 100644 --- a/src/orchestrator.start-mount.test.ts +++ b/src/orchestrator.start-mount.test.ts @@ -165,6 +165,72 @@ describe("startMount initial-sync idle budget", () => { ); }); + it("classifies a rejected initial-sync launch transport", async () => { + // The launch exec is the first long-lived call in startMount. Daytona's + // proxy read timeout is ~120s, so a rejection here is exactly what a + // wedged mount looks like from the outside — and while this call was + // unwrapped, the raw transport error matched none of the message prefixes + // callers classify on, so it fell through to their "unknown" bucket and + // the real cause was discarded. See AgentWorkforce/sandbox#45. + const cause = new Error("read ETIMEDOUT after 120000ms"); + let calls = 0; + const orchestrator = new SandboxOrchestrator<{ id: string }>({ + provision: async () => ({ id: "sbx" }), + uploadBundle: async () => {}, + runScript: async () => { + calls += 1; + if (calls === 2) throw cause; + return { output: "ok", exitCode: 0 }; + }, + teardown: async () => {}, + }); + + await assert.rejects( + orchestrator.startMount({ id: "sbx" }, MOUNT), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match( + error.message, + /Failed to launch relayfile initial sync: read ETIMEDOUT after 120000ms/u, + ); + assert.equal(error.cause, cause); + return true; + }, + ); + assert.equal(calls, 2); + }); + + it("classifies a rejected daemon-start transport", async () => { + // The other formerly-unwrapped exec. Reached only after the initial sync + // reports a clean exit, so it needs the full happy path in front of it. + const cause = new Error("read ETIMEDOUT after 120000ms"); + const orchestrator = new SandboxOrchestrator<{ id: string }>({ + provision: async () => ({ id: "sbx" }), + uploadBundle: async () => {}, + runScript: async (_handle, options) => { + if (options.command.includes("nohup relayfile-mount")) throw cause; + if (options.command.includes("relayfile-initial-sync-exit:")) { + return { output: "relayfile-initial-sync-exit:0", exitCode: 0 }; + } + return { output: "ok", exitCode: 0 }; + }, + teardown: async () => {}, + }); + + await assert.rejects( + orchestrator.startMount({ id: "sbx" }, MOUNT), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match( + error.message, + /Failed to start relayfile mount: read ETIMEDOUT after 120000ms/u, + ); + assert.equal(error.cause, cause); + return true; + }, + ); + }); + it("requests a complete readiness traversal with bounded foreground concurrency", async () => { const { orchestrator, commands } = recordingRuntime(); await orchestrator.startMount({ id: "sbx" }, MOUNT); diff --git a/src/orchestrator.ts b/src/orchestrator.ts index 76159a1..28a04ea 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -252,21 +252,30 @@ export class SandboxOrchestrator { // mount lease, and the exit sentinel is written only after the one-shot // supervisor has exited and released that lease. const initialSyncRun = { runId: relayfileInitialSyncRunId() }; - const launch = await this.runtime.runScript(handle, { - command: withRelayfileInitialSyncEnvironment( - buildRelayfileMountInitialSyncBackgroundShell( - { - ...config, - idleTimeoutSeconds: initialSyncIdleTimeoutSeconds, - }, - initialSyncRun, + let launch: SandboxCommandResult; + try { + launch = await this.runtime.runScript(handle, { + command: withRelayfileInitialSyncEnvironment( + buildRelayfileMountInitialSyncBackgroundShell( + { + ...config, + idleTimeoutSeconds: initialSyncIdleTimeoutSeconds, + }, + initialSyncRun, + ), + initialSyncIdleTimeoutSeconds, + initialSyncReadConcurrency, + initialSyncMaxFilesPerCycle, ), - initialSyncIdleTimeoutSeconds, - initialSyncReadConcurrency, - initialSyncMaxFilesPerCycle, - ), - cwd, - }); + cwd, + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error( + `Failed to launch relayfile initial sync: ${detail}`, + { cause: error }, + ); + } if (launch.exitCode !== 0) { throw new Error(`Failed to launch relayfile initial sync: ${launch.output}`); } @@ -338,13 +347,22 @@ export class SandboxOrchestrator { await sleepMs(pollIntervalMs); } - const start = await this.runtime.runScript(handle, { - command: withRelayfileBootstrapIdleTimeout( - buildRelayfileMountStartShell(config), - initialSyncIdleTimeoutSeconds, - ), - cwd, - }); + let start: SandboxCommandResult; + try { + start = await this.runtime.runScript(handle, { + command: withRelayfileBootstrapIdleTimeout( + buildRelayfileMountStartShell(config), + initialSyncIdleTimeoutSeconds, + ), + cwd, + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error( + `Failed to start relayfile mount: ${detail}`, + { cause: error }, + ); + } if (start.exitCode !== 0) { throw new Error(`Failed to start relayfile mount: ${start.output}`); }