Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,13 @@ const attempt = await launchOwnedRuntimeHostCandidate({
rootPath,
expectedRootId,
entrypoint: new URL('../../execution-candidate-main.js', import.meta.url),
idleGraceMs: 10_000,
// The idle grace only has to outlast the test, and it has to stay clear of
// any bound a test puts on an owner-loss exit: a Candidate that exits
// because it went idle must never be mistaken for one that exited because
// its launch owner died. The first-connection deadline stays short so a
// Candidate no Client ever reaches still exits on its own.
idleGraceMs: 60_000,
initialConnectionTimeoutMs: 10_000,
inheritableAuthorityLeaseFd: leaseFd,
launchOwnerClientInstanceId: clientInstanceId,
}).spawned;
Expand Down
18 changes: 16 additions & 2 deletions packages/runtime-host/src/__tests__/host-kernel.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1996,12 +1996,26 @@ describe('non-serving Runtime Host kernel', () => {

launcher.kill('SIGKILL');
await waitForExit(launcher);
// The process is the only thing that reports the claim. A Client's
// `connection.closed` does not: it is that Client's own transport, and
// the Client aborts it after its liveness probe goes unanswered for two
// seconds. A Host that is merely busy therefore resolves it while still
// running, and gating the exit assertion on it starts the exit budget at
// a moment that has nothing to do with the Host's shutdown.
//
// The bound comes from the kernel's contract rather than from an
// interval this test could predict. Owner loss cannot close a
// composition before its startup settles, and the shutdown that follows
// is bounded by `shutdownGraceMs` (10 s), after which the kernel
// force-terminates. Twenty seconds therefore sits above every
// legitimate exit and below the launcher's 60 s idle grace, so it cannot
// be satisfied by a Candidate that merely went idle.
await waitForProcessExit(launchedPid, 20_000);
await withTimeout(
connected.connection.closed,
5_000,
'authority-supervised Candidate survived its launch owner',
'authority-supervised Candidate exited without closing its Client connection',
);
await waitForProcessExit(launchedPid);
paths.resources.forgetPid(launchedPid);
});
});
Expand Down
52 changes: 47 additions & 5 deletions packages/runtime-host/src/__tests__/owned-candidate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ import {
type CandidateExitDetails,
type OwnedCandidateAttempt,
} from '../client/launcher.js';
import {
resolveExistingStorageRoot,
resolveExistingStorageRootControlDirectory,
} from '@maka/storage/root-authority';
import { readHostRegistration } from '../control/registration.js';

test('owned connection keeps a fresh Host alive for its full election window', async () => {
const rootPath = await mkdtemp(join(tmpdir(), 'maka-owned-first-connection-'));
Expand Down Expand Up @@ -257,12 +262,20 @@ test('owned Host exits promptly after its first connection closes', async () =>

assert.equal(result.kind, 'connected', connectFailure(result));
if (result.kind !== 'connected') return;
const controlDirectory = await resolveHostControlDirectory(rootPath, result.connection.rootId);
await result.connection.close();
// Prompt means the owned launch's idleGraceMs of 0, as opposed to the 30 s
// default grace, so the bound only has to sit well below that. Shutdown takes
// about 30 ms on an idle machine and stretches past 500 ms under a full CI
// suite while still exiting cleanly: the Host is starved, not stuck.
assert.equal(await result.host.settle(5_000), true);
// Promptness is when the Host starts shutting down, not how long shutting
// down takes: the owned launch's idleGraceMs is 0 against a 30 s default.
// The kernel publishes its draining registration as the first step of
// shutdown, so the registration reports the idle grace directly. Reading it
// from `settle` alone could not separate the two, which is why a loaded
// machine that only made the shutdown itself slow failed this assertion.
await waitForHostShutdownStart(controlDirectory, 10_000);
// The exit is a second claim with a bound of its own, and the kernel sets
// it: shutdown gets `shutdownGraceMs` (10 s) to close every resource before
// the kernel force-terminates the process. Anything below that fails a Host
// that is starved rather than stuck.
assert.equal(await result.host.settle(15_000), true);
});

test('an exited owned Candidate permits one real successor in the same election', {
Expand Down Expand Up @@ -436,6 +449,35 @@ test('pre-cancelled hosted execution does not start a Runtime Host', async () =>
assert.deepEqual(await readdir(rootPath), []);
});

async function resolveHostControlDirectory(rootPath: string, rootId: string): Promise<string> {
const capability = await resolveExistingStorageRoot({
path: rootPath,
kind: 'interactive',
expectedRootId: rootId,
});
const { controlDirectory } = await resolveExistingStorageRootControlDirectory(capability);
return controlDirectory;
}

/**
* Resolves once the Host has begun shutting down. `draining` is the state the
* kernel publishes before it does any shutdown work, and the registration is
* removed near the end of that work, so either observation proves shutdown
* started; the Host was serving this Client, so its registration existed.
*/
async function waitForHostShutdownStart(
controlDirectory: string,
timeoutMs: number,
): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const registration = await readHostRegistration(controlDirectory).catch(() => undefined);
if (!registration || registration.state === 'draining') return;
await new Promise((resolve) => setTimeout(resolve, 10));
}
throw new Error('owned Host did not begin shutting down after its first connection closed');
}

function connectFailure(
result:
| Awaited<ReturnType<typeof connectOwnedRuntimeHostWithDependencies>>
Expand Down