From a1d78d6222d5521c89304e63b71945a4803653f1 Mon Sep 17 00:00:00 2001
From: Evan Nemerson <evan.nemerson@cloudzero.com>
Date: Fri, 4 Sep 2026 15:33:25 -0400
Subject: [PATCH] CP-47363: Make the clustered kuttl install diagnosable

The clustered suite fails with a bare "context deadline exceeded", and
kuttl deletes the namespace immediately afterwards, so nothing survives
to say which container never became ready. That is why this suite went
undiagnosed for months: the evidence is discarded before it can be read.

Dump pod phases, container states, events, and Alloy logs when the
install does not become ready. That dump is what identified the actual
cause -- the Alloy container crash-looping on conflicting OpenTelemetry
schema URLs, fixed separately in the Alloy fork -- rather than the
timeout it presented as.

Also shrink the release. The suite installs a second full-size stack
beside the default one already on the node, and clustered mode fans the
server out to three replicas. That is not what was failing here, since
the suite passes without this change once Alloy starts, but the same
overlay already exists for the webhookServer suites (CP-43292) and
leaves headroom rather than relying on there being enough.

The wait goes from three minutes to eight for the same reason as the
dump: at three, a slow start and a broken install look identical.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../steps/01-install-clustered-chart.yaml     | 38 +++++++++++++++++--
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/tests/kuttl/alloy-clustered-test/steps/01-install-clustered-chart.yaml b/tests/kuttl/alloy-clustered-test/steps/01-install-clustered-chart.yaml
index 2c8095ae0..abb8f501a 100644
--- a/tests/kuttl/alloy-clustered-test/steps/01-install-clustered-chart.yaml
+++ b/tests/kuttl/alloy-clustered-test/steps/01-install-clustered-chart.yaml
@@ -17,17 +17,49 @@ commands:
   # per-command timeout here (this suite's kuttl-test.yaml sets `spec.timeout`, which
   # kuttl ignores — the timeout field is top-level — so the 30s default applies), and
   # that would kill the install mid-wait. The explicit per-command `timeout:` overrides it.
-  - timeout: 240
+  # On failure, dump why. kuttl deletes the namespace as soon as a step fails,
+  # so anything not captured here is gone: every past failure of this suite has
+  # produced a bare "context deadline exceeded" with no indication of which
+  # container never became ready, which is why it could not be diagnosed from
+  # CI logs.
+  #
+  # The wait is 8m rather than 3m. Clustered mode starts an Alloy container that
+  # must load the freshly built image, form its gossip ring, and pass a readiness
+  # probe on a cold KIND node; at three minutes a slow start and a broken install
+  # are indistinguishable.
+  - timeout: 600
     script: |
       set -e
       ROOT="$PWD"
       while [ "$ROOT" != "/" ] && [ ! -f "$ROOT/helm/Chart.yaml" ]; do ROOT=$(dirname "$ROOT"); done
       cd "$ROOT"
       DEV_TAG="dev-$(git rev-parse HEAD)"
+      NS=cz-alloy-clustered-test
+
+      dump_state() {
+        echo "=== clustered install did not become ready; dumping state ==="
+        kubectl -n "$NS" get pods -o wide || true
+        echo "--- container states ---"
+        kubectl -n "$NS" get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{range .status.containerStatuses[*]}    {.name} ready={.ready} restarts={.restartCount} {.state}{"\n"}{end}{end}' 2>/dev/null || true
+        echo ""
+        echo "--- recent events ---"
+        kubectl -n "$NS" get events --sort-by=.lastTimestamp 2>/dev/null | tail -30 || true
+        echo "--- alloy logs ---"
+        for P in $(kubectl -n "$NS" get pods -l app.kubernetes.io/name=server -o name 2>/dev/null); do
+          echo "  ---- $P"
+          kubectl -n "$NS" logs "$P" -c cloudzero-agent-alloy --tail=80 2>&1 || true
+        done
+      }
+      trap dump_state EXIT
+
       .tools/bin/helm upgrade --install cz-alloy-ct ./helm \
-        --namespace cz-alloy-clustered-test \
+        --namespace "$NS" \
         --create-namespace \
         --values clusters/kind-overrides.yaml \
+        --values tests/kuttl/webhookserver-min-footprint.yaml \
         --set components.agent.mode=clustered \
         --set components.agent.image.tag="$DEV_TAG" \
-        --wait --timeout=3m
+        --wait --timeout=8m
+
+      trap - EXIT
+      echo "clustered release installed and ready"