Have /health report the services behind the site

projects/website-angular/src/scripts/serve-prod.js

@@ -100,7 +100,7 @@ for (const [context, options] of Object.entries(proxyConfig)) {
  * confidently describe a build that has since been replaced. That is the very
  * failure this exists to catch, and it would be reporting it about itself.
  */
-app.get('/health', (_req, res) => {
+app.get('/health', async (_req, res) => {
   res.setHeader('Cache-Control', 'no-store');
   let html = '';
   try {
@@ -117,9 +117,79 @@ app.get('/health', (_req, res) => {
   } catch {
     // Not worth failing the check over.
   }
-  res.json({ ok: true, bundle, built, dist: DIST });
+  const services = await surveyServices();
+  // `ok` stays the *site's* answer, not the estate's. A monitor pointed here to
+  // ask "is the website up" must not be told no because a sibling it does not
+  // serve is down -- the site renders without the render service, and saying
+  // otherwise would page somebody for the wrong thing. What is wrong is in
+  // `services`, named.
+  res.json({ ok: true, bundle, built, dist: DIST, services });
 });
 
+/**
+ * The services behind this one, asked rather than assumed.
+ *
+ * Every deployment fault on this box has had the same shape: something merged,
+ * something else kept running the version from before it, and nothing said so.
+ * A route was added to nginx pointing at content-node while content-node still
+ * ran the image built before the endpoints existed, so a path Java used to
+ * answer returned 404 until somebody happened to try it.
+ *
+ * So this reports what each one *is*, not merely that it answered: the build it
+ * is running, the release it holds, how many endpoints it serves. Two services
+ * disagreeing about the release is the thing to notice, and it is invisible if
+ * all you have is a green tick each.
+ *
+ * Asked in parallel with a short timeout, because this is polled and must not
+ * become the slow thing. A service that is down is reported as down; it is not
+ * an error here, and it never makes the site's own answer false.
+ */
+async function surveyServices() {
+  // Addresses, not assumptions. They default to where these run beside the site
+  // today and are overridable, because a deployment that moves one should not
+  // have to edit this file -- and because a test can then point them at a dead
+  // port and assert what "down" looks like, which is the case that matters and
+  // the one that cannot be observed on a box where they are all running.
+  const targets = [
+    {
+      name: 'content-node',
+      url: process.env.HEALTH_CONTENT_NODE || 'http://127.0.0.1:4400/health',
+    },
+    { name: 'render', url: process.env.HEALTH_RENDER || 'http://127.0.0.1:4310/health' },
+    { name: 'mcp', url: process.env.HEALTH_MCP || 'http://127.0.0.1:4320/health' },
+  ];
+  const entries = await Promise.all(
+    targets.map(async ({ name, url }) => {
+      try {
+        const response = await fetch(url, { signal: AbortSignal.timeout(2000) });
+        if (!response.ok) return [name, { up: false, status: response.status }];
+        const body = await response.json();
+        return [
+          name,
+          {
+            up: true,
+            // Each service names these differently and there is no value in
+            // pretending otherwise -- what matters is that the fields exist to
+            // compare between them, not that they share a schema.
+            build: body.build ?? body.version ?? null,
+            release: body.release ?? null,
+            endpoints: Array.isArray(body.endpoints) ? body.endpoints.length : undefined,
+          },
+        ];
+      } catch (error) {
+        // Down, unreachable, or slower than the budget. All three are the same
+        // answer to whoever is reading this, and none of them is this route's
+        // failure.
+        return [
+          name,
+          { up: false, reason: error?.name === 'TimeoutError' ? 'timeout' : 'unreachable' },
+        ];
+      }
+    })
+  );
+  return Object.fromEntries(entries);
+}
+
 // Hashed build artefacts are immutable; index.html must never be cached or a
 // redeploy leaves browsers pinned to chunks that no longer exist.
 app.use(

projects/website-angular/src/scripts/serve-prod.spec.ts

@@ -59,7 +59,19 @@ describe('serve-prod', () => {
     const port = await freePort();
     base = `http://127.0.0.1:${port}`;
     server = spawn('node', [SCRIPT], {
-      env: { ...process.env, DIST_DIR: dist, PORT: String(port), HOST: '127.0.0.1' },
+      env: {
+        ...process.env,
+        DIST_DIR: dist,
+        PORT: String(port),
+        HOST: '127.0.0.1',
+        // Pointed at a closed port on purpose. On a developer's box these
+        // services are genuinely running, so without this the suite would
+        // assert "down" and pass only where nothing is deployed — green in CI,
+        // red locally, for a reason that has nothing to do with the code.
+        HEALTH_CONTENT_NODE: 'http://127.0.0.1:9/health',
+        HEALTH_RENDER: 'http://127.0.0.1:9/health',
+        HEALTH_MCP: 'http://127.0.0.1:9/health',
+      },
       stdio: 'ignore',
     });
 
@@ -71,6 +83,34 @@ describe('serve-prod', () => {
     if (dist) await rm(dist, { recursive: true, force: true });
   });
 
+  /**
+   * The health route reports the services behind this one, and this suite runs
+   * with none of them up — which is the case worth pinning. Every deployment
+   * fault on this box has been something running a version from before the
+   * merge with nothing saying so, and a health check that fell over when a
+   * sibling was down would be reporting the estate under the name of the site.
+   */
+  it('reports each service, and says which are down', async () => {
+    const body = await (await fetch(base + '/health')).json();
+    expect(Object.keys(body.services)).toEqual(
+      expect.arrayContaining(['content-node', 'render', 'mcp'])
+    );
+    // Nothing is running in this suite, so every one of them is down and each
+    // says so on its own rather than collapsing into a single failure.
+    for (const [name, state] of Object.entries(body.services as Record<string, { up: boolean }>)) {
+      expect(state.up, `${name} should report its own state`).toBe(false);
+    }
+  });
+
+  it('stays ok when a service behind it is down', async () => {
+    // `ok` is the site's answer, not the estate's. The site renders without the
+    // render service; telling a monitor otherwise pages somebody for the wrong
+    // thing, and what is actually wrong is named in `services`.
+    const body = await (await fetch(base + '/health')).json();
+    expect(body.ok).toBe(true);
+    expect(body.bundle === null || typeof body.bundle === 'string').toBe(true);
+  });
+
   it('serves the build at the root', async () => {
     const response = await fetch(base + '/');
     expect(response.status).toBe(200);