From bc509b754bf9d2aa229c02c466e6e0fbcf83f80d Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Sat, 5 Sep 2026 07:13:27 +0800 Subject: [PATCH] test(runtime-host): bound owned-Host lifecycle tests to kernel contracts Fixes the intermittent failures reported in #4776. The test-side analysis and fix shape are @UncertaintyDeterminesYou4ndMe's (proposed in #4784 and donated in the issue after that PR was withdrawn); this commit implements it so the findings are not lost. - The launch-owner exit test gated its assertion on the Client's own `connection.closed`, which the Client aborts after a 2 s unanswered liveness probe, so a merely busy Host resolved it while still running. The assertion now waits on the process itself with a 20 s bound derived from the kernel's `shutdownGraceMs` contract, below the launcher's new 60 s idle grace. - The owned launch fixture's `idleGraceMs` (10 s) could expire mid-test and let an idle exit masquerade as an owner-loss exit; it now sits at 60 s with an explicit `initialConnectionTimeoutMs`. - "Exits promptly" now asserts shutdown start (the kernel's published `draining` registration) separately from shutdown completion, and the settle bound (15 s) sits above the kernel's own 10 s grace instead of inside it, so a starved-but-clean Host no longer reports an unclean exit. Fixes #4776 Generated-by: GLM-5.3-Flash (ZCode) --- .../fixtures/owned-authority-launcher.ts | 8 ++- .../src/__tests__/host-kernel.test.ts | 18 ++++++- .../src/__tests__/owned-candidate.test.ts | 52 +++++++++++++++++-- 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts index 6e67dcd363..04799e3c96 100644 --- a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts +++ b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts @@ -32,7 +32,13 @@ const attempt = await launchOwnedRuntimeHostCandidate({ rootPath, expectedRootId, entrypoint: new URL('../../execution-candidate-main.js', import.meta.url), - idleGraceMs: 10_000, + // The idle grace only has to outlast the test, and it has to stay clear of + // any bound a test puts on an owner-loss exit: a Candidate that exits + // because it went idle must never be mistaken for one that exited because + // its launch owner died. The first-connection deadline stays short so a + // Candidate no Client ever reaches still exits on its own. + idleGraceMs: 60_000, + initialConnectionTimeoutMs: 10_000, inheritableAuthorityLeaseFd: leaseFd, launchOwnerClientInstanceId: clientInstanceId, }).spawned; diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 2e330f305a..dace5c2fbb 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -1916,12 +1916,26 @@ describe('non-serving Runtime Host kernel', () => { launcher.kill('SIGKILL'); await waitForExit(launcher); + // The process is the only thing that reports the claim. A Client's + // `connection.closed` does not: it is that Client's own transport, and + // the Client aborts it after its liveness probe goes unanswered for two + // seconds. A Host that is merely busy therefore resolves it while still + // running, and gating the exit assertion on it starts the exit budget at + // a moment that has nothing to do with the Host's shutdown. + // + // The bound comes from the kernel's contract rather than from an + // interval this test could predict. Owner loss cannot close a + // composition before its startup settles, and the shutdown that follows + // is bounded by `shutdownGraceMs` (10 s), after which the kernel + // force-terminates. Twenty seconds therefore sits above every + // legitimate exit and below the launcher's 60 s idle grace, so it cannot + // be satisfied by a Candidate that merely went idle. + await waitForProcessExit(launchedPid, 20_000); await withTimeout( connected.connection.closed, 5_000, - 'authority-supervised Candidate survived its launch owner', + 'authority-supervised Candidate exited without closing its Client connection', ); - await waitForProcessExit(launchedPid); paths.resources.forgetPid(launchedPid); }); }); diff --git a/packages/runtime-host/src/__tests__/owned-candidate.test.ts b/packages/runtime-host/src/__tests__/owned-candidate.test.ts index 3c264f145d..e260349ccb 100644 --- a/packages/runtime-host/src/__tests__/owned-candidate.test.ts +++ b/packages/runtime-host/src/__tests__/owned-candidate.test.ts @@ -37,6 +37,11 @@ import { type CandidateExitDetails, type OwnedCandidateAttempt, } from '../client/launcher.js'; +import { + resolveExistingStorageRoot, + resolveExistingStorageRootControlDirectory, +} from '@maka/storage/root-authority'; +import { readHostRegistration } from '../control/registration.js'; test('owned connection keeps a fresh Host alive for its full election window', async () => { const rootPath = await mkdtemp(join(tmpdir(), 'maka-owned-first-connection-')); @@ -257,12 +262,20 @@ test('owned Host exits promptly after its first connection closes', async () => assert.equal(result.kind, 'connected', connectFailure(result)); if (result.kind !== 'connected') return; + const controlDirectory = await resolveHostControlDirectory(rootPath, result.connection.rootId); await result.connection.close(); - // Prompt means the owned launch's idleGraceMs of 0, as opposed to the 30 s - // default grace, so the bound only has to sit well below that. Shutdown takes - // about 30 ms on an idle machine and stretches past 500 ms under a full CI - // suite while still exiting cleanly: the Host is starved, not stuck. - assert.equal(await result.host.settle(5_000), true); + // Promptness is when the Host starts shutting down, not how long shutting + // down takes: the owned launch's idleGraceMs is 0 against a 30 s default. + // The kernel publishes its draining registration as the first step of + // shutdown, so the registration reports the idle grace directly. Reading it + // from `settle` alone could not separate the two, which is why a loaded + // machine that only made the shutdown itself slow failed this assertion. + await waitForHostShutdownStart(controlDirectory, 10_000); + // The exit is a second claim with a bound of its own, and the kernel sets + // it: shutdown gets `shutdownGraceMs` (10 s) to close every resource before + // the kernel force-terminates the process. Anything below that fails a Host + // that is starved rather than stuck. + assert.equal(await result.host.settle(15_000), true); }); test('an exited owned Candidate permits one real successor in the same election', { @@ -436,6 +449,35 @@ test('pre-cancelled hosted execution does not start a Runtime Host', async () => assert.deepEqual(await readdir(rootPath), []); }); +async function resolveHostControlDirectory(rootPath: string, rootId: string): Promise { + const capability = await resolveExistingStorageRoot({ + path: rootPath, + kind: 'interactive', + expectedRootId: rootId, + }); + const { controlDirectory } = await resolveExistingStorageRootControlDirectory(capability); + return controlDirectory; +} + +/** + * Resolves once the Host has begun shutting down. `draining` is the state the + * kernel publishes before it does any shutdown work, and the registration is + * removed near the end of that work, so either observation proves shutdown + * started; the Host was serving this Client, so its registration existed. + */ +async function waitForHostShutdownStart( + controlDirectory: string, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const registration = await readHostRegistration(controlDirectory).catch(() => undefined); + if (!registration || registration.state === 'draining') return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error('owned Host did not begin shutting down after its first connection closed'); +} + function connectFailure( result: | Awaited>