From 55af1934d84b9e803a252e7d3272af1f4040cfcf Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 14:48:37 -0700
Subject: [PATCH 01/14] Run e2e tests from the runner via Holodeck remoteAccess

Enable kubernetes.remoteAccess in tests/holodeck.yaml so Holodeck hands the
GitHub Actions runner a kubeconfig for the test cluster, and rework the e2e
workflow to use it. The case scripts now run on the runner instead of being
rsynced to the EC2 instance and driven over SSH, which removes the scp of the
values override file, the ci-run-e2e.sh/local.sh/push.sh/pull.sh chain and the
key.pem written into the workspace.

SSH is still needed for the two host-mutating operations, so both jobs write
the key under RUNNER_TEMP, export NODE_SSH_HOST/NODE_SSH_KEY/
NODE_SSH_KNOWN_HOSTS for tests/scripts/node-exec.sh, and delete the key
directory at the end of the job.

helm, kubectl and jq are now installed on the runner at pinned versions. helm
was previously installed on the instance by tests/scripts/prerequisites.sh
from the get-helm-3 master script, so it was whatever release happened to be
current; pinning it is a deliberate change. kubectl is pinned to the
Kubernetes version in tests/holodeck.yaml.

Also add a preflight step that fails the job early if the kubeconfig is
unusable, and an always() step that dumps nodes, pods, events and helm
releases into the log directory so the existing artifacts are more useful.

tests/local.sh and friends are unchanged and remain the documented developer
path.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml | 164 ++++++++++++++++++++++++-------
 tests/README.md                  |  17 ++++
 tests/holodeck.yaml              |   1 +
 3 files changed, 148 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 298b16414c..7f07c54b45 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -55,6 +55,16 @@ on:
         type: boolean
         default: false
 
+env:
+  # Pinned versions of the tooling the test scripts drive from the runner.
+  # helm used to be installed on the test node by tests/scripts/prerequisites.sh
+  # straight from the helm get-helm-3 master script, i.e. whatever release was
+  # current at the time the job ran. Pinning it here is a deliberate change.
+  HELM_VERSION: v3.19.0
+  # Kept in sync with spec.kubernetes.version in tests/holodeck.yaml.
+  KUBECTL_VERSION: v1.35.4
+  JQ_VERSION: "1.7.1"
+
 jobs:
   variables:
     uses: ./.github/workflows/variables.yaml
@@ -79,6 +89,11 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      # Holodeck is configured with kubernetes.remoteAccess, so it drops a
+      # kubeconfig pointing at the node's public API server endpoint here.
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         name: Check out code
@@ -92,11 +107,28 @@ jobs:
           path: ${{ github.workspace }}
       - name: Set up Helm
         uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+        with:
+          version: ${{ env.HELM_VERSION }}
       - name: Verify the published Helm chart is available
         env:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
         run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}"
+      - name: Install kubectl
+        uses: azure/setup-kubectl@c0c8b32d33a5244f1e5947304550403b63930415 # v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7
         with:
@@ -109,24 +141,36 @@ jobs:
         uses: mikefarah/yq@c14f446382944492701b16c1ddb48bb9dbe683e3 # v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}" >> "$GITHUB_ENV"
+          echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa" >> "$GITHUB_ENV"
+          echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts" >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
@@ -134,13 +178,19 @@ jobs:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/defaults.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          mkdir -p "${GITHUB_WORKSPACE}/logs"
+          ./tests/cases/defaults.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
+          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@@ -148,6 +198,9 @@ jobs:
           name: containerd-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove node SSH credentials
+        if: always()
+        run: rm -rf "${RUNNER_TEMP}/holodeck-ssh"
 
   e2e-tests-nvidiadriver:
     needs: [variables, publish-helm-oci-chart]
@@ -156,6 +209,11 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      # Holodeck is configured with kubernetes.remoteAccess, so it drops a
+      # kubeconfig pointing at the node's public API server endpoint here.
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         name: Check out code
@@ -169,11 +227,28 @@ jobs:
           path: ${{ github.workspace }}
       - name: Set up Helm
         uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+        with:
+          version: ${{ env.HELM_VERSION }}
       - name: Verify the published Helm chart is available
         env:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
         run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}"
+      - name: Install kubectl
+        uses: azure/setup-kubectl@c0c8b32d33a5244f1e5947304550403b63930415 # v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7
         with:
@@ -186,24 +261,36 @@ jobs:
         uses: mikefarah/yq@c14f446382944492701b16c1ddb48bb9dbe683e3 # v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}" >> "$GITHUB_ENV"
+          echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa" >> "$GITHUB_ENV"
+          echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts" >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
@@ -211,13 +298,19 @@ jobs:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/nvidia-driver.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          mkdir -p "${GITHUB_WORKSPACE}/logs"
+          ./tests/cases/nvidia-driver.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
+          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@@ -225,3 +318,6 @@ jobs:
           name: nvidiadriver-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove node SSH credentials
+        if: always()
+        run: rm -rf "${RUNNER_TEMP}/holodeck-ssh"
diff --git a/tests/README.md b/tests/README.md
index 9b14de2462..3a4096ce5c 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,5 +1,22 @@
 # GPU operator test utilities
 
+## Testing in CI
+CI no longer uses `local.sh` or `ci-run-e2e.sh`, and no longer syncs the project
+folder to the test instance. Those remain the developer path described below.
+
+Instead, the e2e workflow provisions a Holodeck environment with
+`kubernetes.remoteAccess` enabled, which gives the GitHub Actions runner a
+kubeconfig for the cluster. The case scripts (`cases/defaults.sh`,
+`cases/nvidia-driver.sh`) then run on the runner itself, and everything they do
+-- helm, kubectl, log collection -- goes over that kubeconfig.
+
+Only two operations still need a shell on the instance, and both go through
+`scripts/node-exec.sh`, which dispatches `scripts/node-operations.sh` over SSH:
+loading the `i2c_core` and `ipmi_msghandler` kernel modules, and killing the
+gpu-operator container for the operator restart test. With `NODE_SSH_HOST`
+unset, `node-exec.sh` runs the operation locally, so the developer path below is
+unaffected.
+
 ## Testing locally
 The `local.sh` script allows for triggering basic end-to-end testing of the GPU
 operator from a local machine.
diff --git a/tests/holodeck.yaml b/tests/holodeck.yaml
index 3050eaf680..880aa417a6 100644
--- a/tests/holodeck.yaml
+++ b/tests/holodeck.yaml
@@ -24,3 +24,4 @@ spec:
     version: v1.35.4
     crictlVersion: v1.35.0
     calicoVersion: v3.31.5
+    remoteAccess: true

From 21c363a65de932700410442feda1da8077756044 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 14:49:06 -0700
Subject: [PATCH 02/14] feat(ci): add node-operation dispatch for e2e tests

The e2e tests are moving to run against the cluster from the GitHub Actions
runner instead of over SSH on the node. Two operations still have to run on
the node itself: loading the i2c_core and ipmi_msghandler kernel modules, and
killing the gpu-operator container in the restart test.

Add node-operations.sh, which implements both operations and is self-contained
so that it can be streamed to the node over SSH stdin, and node-exec.sh, which
dispatches an operation over SSH when NODE_SSH_HOST is set and runs it locally
otherwise so that the existing developer workflow keeps working.

test_restart_operator now calls node-exec.sh instead of running crictl or
docker inline. The container selection logic is unchanged, except that an
empty container ID is now reported as an error instead of being handed to the
removal command.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/checks.sh          | 13 ++---
 tests/scripts/node-exec.sh       | 91 +++++++++++++++++++++++++++++
 tests/scripts/node-operations.sh | 99 ++++++++++++++++++++++++++++++++
 3 files changed, 195 insertions(+), 8 deletions(-)
 create mode 100755 tests/scripts/node-exec.sh
 create mode 100755 tests/scripts/node-operations.sh

diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 5056d48b2f..1577a4ae18 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -85,14 +85,11 @@ test_restart_operator() {
 	local ns=${1}
 	local runtime=${2}
 
-	if [[ x"${runtime}" == x"containerd" ]]; then
-		# The operator is the only container that has the string '"gpu-operator"'
-		# TODO: This requires permissions on containerd.sock
-		sudo crictl rm --force "$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')"
-	else
-		# The operator is the only container that has the string '"gpu-operator"'
-	    docker kill "$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')"
-	fi
+	# Killing the operator container mutates the node, so it is dispatched to the
+	# node itself. node-exec.sh runs it either over SSH or locally.
+	local checks_script_dir
+	checks_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+	"${checks_script_dir}"/node-exec.sh restart-operator-container "${runtime}"
 
 	for i in $(seq 1 10); do
 		# Sleep a reasonable amount of time for k8s to update the container status to crashing
diff --git a/tests/scripts/node-exec.sh b/tests/scripts/node-exec.sh
new file mode 100755
index 0000000000..9f4b351e4d
--- /dev/null
+++ b/tests/scripts/node-exec.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# Copyright NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# This script dispatches a host-mutating operation to the node hosting the
+# cluster. When NODE_SSH_HOST is set the operation is streamed to the node over
+# SSH; otherwise it is executed locally, which is the developer path where the
+# tests already run on the node itself.
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
+
+usage() {
+	cat <<'EOF'
+Usage: node-exec.sh <operation> [args...]
+
+Runs one of the node-operations.sh operations on the node hosting the cluster.
+
+Environment:
+  NODE_SSH_HOST         user@host of the node, e.g. ubuntu@ec2-1-2-3-4.compute.amazonaws.com.
+                        If unset or empty the operation is executed locally.
+  NODE_SSH_KEY          Path to the private key. Required when NODE_SSH_HOST is set.
+  NODE_SSH_KNOWN_HOSTS  Path to the known_hosts file. Required when NODE_SSH_HOST is set.
+
+Operations:
+  load-modules
+  restart-operator-container <runtime>
+EOF
+}
+
+require_readable_file() {
+	local name="${1}"
+	local value="${2}"
+
+	if [[ -z "${value}" ]]; then
+		echo "Error: ${name} must be set when NODE_SSH_HOST is set" >&2
+		exit 1
+	fi
+	if [[ ! -r "${value}" ]]; then
+		echo "Error: ${name} '${value}' does not exist or is not readable" >&2
+		exit 1
+	fi
+}
+
+if [[ $# -lt 1 ]]; then
+	usage >&2
+	exit 2
+fi
+
+if [[ ! -r "${NODE_OPERATIONS}" ]]; then
+	echo "Error: ${NODE_OPERATIONS} does not exist or is not readable" >&2
+	exit 1
+fi
+
+if [[ -z "${NODE_SSH_HOST:-}" ]]; then
+	echo "Running '$*' locally"
+	bash "${NODE_OPERATIONS}" "$@"
+	exit $?
+fi
+
+require_readable_file "NODE_SSH_KEY" "${NODE_SSH_KEY:-}"
+require_readable_file "NODE_SSH_KNOWN_HOSTS" "${NODE_SSH_KNOWN_HOSTS:-}"
+
+# Quote each argument so that it survives the remote shell.
+REMOTE_COMMAND="bash -s --"
+for arg in "$@"; do
+	REMOTE_COMMAND+=" $(printf '%q' "${arg}")"
+done
+
+echo "Running '$*' on ${NODE_SSH_HOST}"
+ssh -i "${NODE_SSH_KEY}" \
+	-o BatchMode=yes \
+	-o ConnectTimeout=30 \
+	-o StrictHostKeyChecking=accept-new \
+	-o UserKnownHostsFile="${NODE_SSH_KNOWN_HOSTS}" \
+	"${NODE_SSH_HOST}" \
+	"${REMOTE_COMMAND}" < "${NODE_OPERATIONS}"
diff --git a/tests/scripts/node-operations.sh b/tests/scripts/node-operations.sh
new file mode 100755
index 0000000000..fd46b1e80b
--- /dev/null
+++ b/tests/scripts/node-operations.sh
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# Copyright NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# This script runs ON the node hosting the cluster. It is either executed
+# locally or streamed over SSH stdin via `bash -s --`, which means it MUST stay
+# fully self-contained: it must not source sibling scripts such as
+# .definitions.sh and must not refer to any path relative to the repository,
+# since the repository is not guaranteed to exist on the node.
+
+usage() {
+	cat <<'EOF'
+Usage: node-operations.sh <operation> [args...]
+
+Operations:
+  load-modules
+      Load the kernel modules required by the GPU Operator.
+
+  restart-operator-container <runtime>
+      Kill the running gpu-operator container so that kubernetes restarts it.
+      Supported runtimes: containerd, docker.
+EOF
+}
+
+load_modules() {
+	echo "Load kernel modules i2c_core and ipmi_msghandler"
+	sudo modprobe -a i2c_core ipmi_msghandler
+}
+
+# The x-prefixed comparisons and the container selection pipelines below are
+# kept as they were in tests/scripts/checks.sh so that the behaviour of the
+# restart test does not change.
+# shellcheck disable=SC2268
+restart_operator_container() {
+	local runtime="${1:-}"
+	local container_id=""
+
+	if [[ x"${runtime}" == x"containerd" ]]; then
+		# The operator is the only container that has the string '"gpu-operator"'
+		# TODO: This requires permissions on containerd.sock
+		container_id="$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')" || true
+		if [[ -z "${container_id}" ]]; then
+			echo "Error: no running gpu-operator container found via crictl" >&2
+			return 1
+		fi
+		sudo crictl rm --force "${container_id}"
+	elif [[ x"${runtime}" == x"docker" ]]; then
+		# The operator is the only container that has the string '"gpu-operator"'
+		container_id="$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')" || true
+		if [[ -z "${container_id}" ]]; then
+			echo "Error: no running gpu-operator container found via docker" >&2
+			return 1
+		fi
+		docker kill "${container_id}"
+	else
+		echo "Error: unknown runtime '${runtime}'. Supported runtimes: containerd, docker" >&2
+		return 1
+	fi
+}
+
+main() {
+	if [[ $# -lt 1 ]]; then
+		usage >&2
+		exit 2
+	fi
+
+	local operation="${1}"
+	shift
+
+	case "${operation}" in
+		load-modules)
+			load_modules "$@"
+			;;
+		restart-operator-container)
+			restart_operator_container "$@"
+			;;
+		*)
+			echo "Error: unknown operation '${operation}'" >&2
+			usage >&2
+			exit 2
+			;;
+	esac
+}
+
+main "$@"

From 94362736e157b52f529228b72f43687d59fec305 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 15:07:36 -0700
Subject: [PATCH 03/14] Harden the e2e workflow after review

node-exec.sh deliberately falls back to running the operation locally when
NODE_SSH_HOST is empty, which is what the developer path relies on. In CI that
fallback would run modprobe and crictl against the shared self-hosted runner
instead of the test instance, so guard both call sites with a non-empty check
on NODE_SSH_HOST and fail the job loudly.

Also from review:

- verify the downloaded jq binary against the sha256 published in the jq 1.7.1
  release checksum file before putting it on PATH
- fail early if the public-dns-name lookup came back empty, rather than
  building NODE_SSH_HOST=ubuntu@ and getting an opaque ssh error later
- add || true to the mkdir in the diagnostics step so a green job cannot be
  turned red by diagnostics
- remove the kubeconfig alongside the SSH key in the always() cleanup, since
  it holds cluster-admin credentials
- use GITHUB_WORKSPACE instead of interpolating github.workspace into a run
  block, matching how the rest of the job passes values through env

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml | 44 ++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 7f07c54b45..1f018469c5 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -64,6 +64,8 @@ env:
   # Kept in sync with spec.kubernetes.version in tests/holodeck.yaml.
   KUBECTL_VERSION: v1.35.4
   JQ_VERSION: "1.7.1"
+  # From https://github.com/jqlang/jq/releases/download/jq-1.7.1/sha256sum.txt
+  JQ_SHA256: "5942c9b0934e510ee61eb3e30273f1b3fe2590df93933a93d7c58b81d19c8ff5"
 
 jobs:
   variables:
@@ -124,6 +126,7 @@ jobs:
           mkdir -p "${RUNNER_TEMP}/bin"
           curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
             "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
           chmod +x "${RUNNER_TEMP}/bin/jq"
           echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
           helm version
@@ -147,6 +150,7 @@ jobs:
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
           set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
           SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
           mkdir -p "${SSH_DIR}"
           chmod 700 "${SSH_DIR}"
@@ -166,10 +170,15 @@ jobs:
         if: ${{ inputs.use_values_override }}
         run: |
           set -euo pipefail
-          echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> "$GITHUB_ENV"
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
       - name: Load kernel modules on the node
         run: |
           set -euo pipefail
+          # node-exec.sh falls back to running the operation locally when
+          # NODE_SSH_HOST is empty, which is the developer path. In CI that
+          # would run modprobe against the shared runner, so fail loudly
+          # instead.
+          test -n "${NODE_SSH_HOST}"
           ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
@@ -181,12 +190,16 @@ jobs:
           CONTAINER_RUNTIME: "containerd"
         run: |
           set -euo pipefail
+          # The restart-operator test shells out to node-exec.sh, which runs
+          # locally when NODE_SSH_HOST is empty. Refuse to start rather than
+          # let crictl run against the shared runner.
+          test -n "${NODE_SSH_HOST}"
           mkdir -p "${GITHUB_WORKSPACE}/logs"
           ./tests/cases/defaults.sh
       - name: Collect cluster diagnostics
         if: always()
         run: |
-          mkdir -p "${LOG_DIR}"
+          mkdir -p "${LOG_DIR}" || true
           kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
           kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
@@ -198,9 +211,11 @@ jobs:
           name: containerd-e2e-test-logs
           path: ./logs/
           retention-days: 15
-      - name: Remove node SSH credentials
+      - name: Remove credentials from the runner
         if: always()
-        run: rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh" || true
+          rm -f "${KUBECONFIG}" || true
 
   e2e-tests-nvidiadriver:
     needs: [variables, publish-helm-oci-chart]
@@ -244,6 +259,7 @@ jobs:
           mkdir -p "${RUNNER_TEMP}/bin"
           curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
             "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
           chmod +x "${RUNNER_TEMP}/bin/jq"
           echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
           helm version
@@ -267,6 +283,7 @@ jobs:
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
           set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
           SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
           mkdir -p "${SSH_DIR}"
           chmod 700 "${SSH_DIR}"
@@ -286,10 +303,15 @@ jobs:
         if: ${{ inputs.use_values_override }}
         run: |
           set -euo pipefail
-          echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> "$GITHUB_ENV"
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
       - name: Load kernel modules on the node
         run: |
           set -euo pipefail
+          # node-exec.sh falls back to running the operation locally when
+          # NODE_SSH_HOST is empty, which is the developer path. In CI that
+          # would run modprobe against the shared runner, so fail loudly
+          # instead.
+          test -n "${NODE_SSH_HOST}"
           ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
@@ -301,12 +323,16 @@ jobs:
           CONTAINER_RUNTIME: "containerd"
         run: |
           set -euo pipefail
+          # The restart-operator test shells out to node-exec.sh, which runs
+          # locally when NODE_SSH_HOST is empty. Refuse to start rather than
+          # let crictl run against the shared runner.
+          test -n "${NODE_SSH_HOST}"
           mkdir -p "${GITHUB_WORKSPACE}/logs"
           ./tests/cases/nvidia-driver.sh
       - name: Collect cluster diagnostics
         if: always()
         run: |
-          mkdir -p "${LOG_DIR}"
+          mkdir -p "${LOG_DIR}" || true
           kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
           kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
@@ -318,6 +344,8 @@ jobs:
           name: nvidiadriver-e2e-test-logs
           path: ./logs/
           retention-days: 15
-      - name: Remove node SSH credentials
+      - name: Remove credentials from the runner
         if: always()
-        run: rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh" || true
+          rm -f "${KUBECONFIG}" || true

From 0a6449cb0ffb58d1e95d03cc442efa8c65ab2344 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 16:24:31 -0700
Subject: [PATCH 04/14] fix(ci): bound the e2e polling loops by wall clock

The polling loops in checks.sh bounded themselves with a counter that was
incremented by 5 on every iteration, on the assumption that an iteration costs
only the 5 second sleep. That has been close enough while the tests ran on the
node itself, but each iteration also issues a number of kubectl calls, and once
those calls cross a network the iteration takes considerably longer than 5
seconds. The counter then runs slower than the clock and the nominal 45 minute
bound stretches to several hours, which is long enough for the job timeout to
cancel the run before any of the loops give up on their own.

Measure elapsed time with the SECONDS builtin against a baseline taken when the
loop starts, so the bound means what it says regardless of how long an
iteration takes. The 45 minute budget itself is unchanged.

wait_for_driver_upgrade_done printed its debug dump when the counter was
divisible by 30. Elapsed time no longer advances in fixed steps, so that test
can step over every multiple and the dump would never be printed. Track the
time at which the next dump is due instead.

Also pass --tail to the per-pod log fetch in check_gpu_pod_ready. It runs
inside the poll loop for every pod in every namespace and refetches each
complete log every five seconds, which is a lot of traffic to repeat for up to
45 minutes. The log collection on failure is left untouched.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/checks.sh | 42 ++++++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 1577a4ae18..20449ef6d3 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -2,7 +2,8 @@
 
 check_pod_ready() {
 	local pod_label=$1
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
 	while :; do
 		echo "Checking $pod_label pod"
 		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
@@ -21,7 +22,7 @@ check_pod_ready() {
 			fi
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			exit 1;
 		fi
@@ -30,14 +31,14 @@ check_pod_ready() {
 		kubectl get pods -n ${TEST_NAMESPACE}
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
 
 check_pod_deleted() {
 	local pod_label=$1
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
 	while :; do
 		echo "Checking $pod_label pod"
 		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
@@ -53,7 +54,7 @@ check_pod_deleted() {
 			echo "Pod $pod_label has not been deleted"
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			exit 1;
 		fi
@@ -62,7 +63,6 @@ check_pod_deleted() {
 		kubectl get pods -n ${TEST_NAMESPACE}
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
@@ -111,7 +111,8 @@ test_restart_operator() {
 
 check_gpu_pod_ready() {
 	local log_dir=$1
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
 
 	# Ensure the log directory exists
 	mkdir -p ${log_dir}
@@ -125,7 +126,7 @@ check_gpu_pod_ready() {
 			break;
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			exit 1
 		fi
@@ -138,7 +139,8 @@ check_gpu_pod_ready() {
 			echo "Generating logs for pod: ${pod} ns: ${ns}"
 			echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
 			kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
-			kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
+			# This runs on every poll iteration, so bound the volume we re-fetch each time.
+			kubectl -n "${ns}" logs "${pod}" --all-containers=true --tail=200 > "${log_dir}/${pod}.logs" || true
 		done
 
 		echo "Generating cluster logs"
@@ -146,14 +148,14 @@ check_gpu_pod_ready() {
 		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs"
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5;
 	done
 }
 
 # TODO: deduplicate the logic found in this file by moving the duplicate to a common method and parameterizing the labels to select on
 check_nvidia_driver_pods_ready() {
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
 	while :; do
 		echo "Checking nvidia driver pod"
 		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE}
@@ -172,7 +174,7 @@ check_nvidia_driver_pods_ready() {
 			fi
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			exit 1;
 		fi
@@ -181,7 +183,6 @@ check_nvidia_driver_pods_ready() {
 		kubectl get pods -n ${TEST_NAMESPACE}
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
@@ -239,7 +240,13 @@ print_driver_upgrade_debug() {
 
 wait_for_driver_upgrade_done() {
 	gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present --no-headers | wc -l)
-	local current_time=0
+	# SECONDS counts from shell start, so record a baseline and measure against it.
+	local start_time=${SECONDS}
+	local elapsed=0
+	# Next elapsed time at which the full debug dump is due. Iterations can take
+	# much longer than the nominal sleep, so track a due time instead of testing
+	# the elapsed time for divisibility, which would skip dumps entirely.
+	local next_debug=0
 	echo "waiting for the gpu driver upgrade to complete"
 	while :; do
 		local upgraded_count=0
@@ -256,21 +263,22 @@ wait_for_driver_upgrade_done() {
 			echo "gpu driver still in progress. $upgraded_count/$gpu_node_count node(s) upgraded"
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		elapsed=$((SECONDS - start_time))
+		if [[ "${elapsed}" -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
 			print_driver_upgrade_debug
 			exit 1;
 		fi
 
-		if [[ $((current_time % 30)) -eq 0 ]]; then
+		if [[ "${elapsed}" -ge "${next_debug}" ]]; then
 			print_driver_upgrade_debug
+			next_debug=$((elapsed + 30))
 		else
 			kubectl get node -l nvidia.com/gpu.present \
 				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
 		fi
 		
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }

From 71d4d0b5225e0fc1ae5ca888203891ba41b6931c Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 16:50:03 -0700
Subject: [PATCH 05/14] fix(ci): collect e2e pod logs on a slower cadence
 instead of truncating them

check_gpu_pod_ready regenerates a describe and a log file for every pod in
every namespace on each pass of its five second poll loop. Passing --tail to
bound that traffic was the wrong call: the log file is overwritten rather than
appended, so the file left behind when the loop gives up is the one that gets
uploaded as the failure artifact, and truncating it drops exactly the output
that explains a failed driver build.

Fetch the whole log again and instead regenerate on a thirty second cadence,
which cuts the traffic by the same order without shortening anything. The
readiness check keeps running every five seconds so success is still noticed
promptly. Both the timeout and the success path collect once more on the way
out so the files on disk are current rather than up to thirty seconds old.

update-nvidiadriver.sh has seven loops with the same counter-based timeout that
checks.sh had, guarding the nvidiadriver test that runs in the same job under
the same job timeout. Convert them the same way. wait_for_nvidiadriver_owner is
the worst of them: its counter needs 181 iterations to reach a fifteen minute
bound, which is half an hour of wall clock once each iteration waits on a
round trip.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/checks.sh              | 42 ++++++++++++++++++++++------
 tests/scripts/update-nvidiadriver.sh | 42 ++++++++++++++--------------
 2 files changed, 54 insertions(+), 30 deletions(-)

diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 20449ef6d3..437b6a299d 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -109,10 +109,33 @@ test_restart_operator() {
 	exit 1
 }
 
+# Regenerate the describe and log files for every pod passed in. The log files
+# are rewritten in full rather than tailed, so that the artifact left behind
+# holds the complete output of every container.
+collect_pod_logs() {
+	local log_dir=$1
+	local pods=$2
+
+	for pod in $(echo "$pods" | jq -r .[].name); do
+		ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
+		echo "Generating logs for pod: ${pod} ns: ${ns}"
+		echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
+		kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
+		kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
+	done
+}
+
 check_gpu_pod_ready() {
 	local log_dir=$1
 	# SECONDS counts from shell start, so record a baseline and measure against it.
 	local start_time=${SECONDS}
+	local elapsed=0
+	# Regenerating every pod's describe and log files is the expensive part of
+	# this loop, so it runs on its own slower cadence while the readiness check
+	# below keeps polling every 5 seconds. Track the time at which the next
+	# regeneration is due rather than testing the elapsed time for divisibility,
+	# which would skip regenerations when an iteration runs long.
+	local next_collection=0
 
 	# Ensure the log directory exists
 	mkdir -p ${log_dir}
@@ -123,25 +146,26 @@ check_gpu_pod_ready() {
 		if [ "${status}" = "Succeeded" ]; then
 			echo "GPU pod terminated successfully"
 			rc=0
+			collect_pod_logs "${log_dir}" "${pods}"
 			break;
 		fi
 
-		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
+		elapsed=$((SECONDS - start_time))
+		if [[ "${elapsed}" -gt $((60 * 45)) ]]; then
 			echo "timeout reached"
+			# Collect once more so that the artifact reflects the state at the
+			# timeout rather than the state at the last scheduled collection.
+			collect_pod_logs "${log_dir}" "${pods}"
 			exit 1
 		fi
 
 		# Echo useful information on stdout
 		kubectl get pods --all-namespaces
 
-		for pod in $(echo "$pods" | jq -r .[].name); do
-			ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
-			echo "Generating logs for pod: ${pod} ns: ${ns}"
-			echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
-			kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
-			# This runs on every poll iteration, so bound the volume we re-fetch each time.
-			kubectl -n "${ns}" logs "${pod}" --all-containers=true --tail=200 > "${log_dir}/${pod}.logs" || true
-		done
+		if [[ "${elapsed}" -ge "${next_collection}" ]]; then
+			collect_pod_logs "${log_dir}" "${pods}"
+			next_collection=$((elapsed + 30))
+		fi
 
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index d104673654..efab216afe 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -56,7 +56,8 @@ set_default_driver() {
 
 wait_for_default_nvidiadriver() {
     local expected_name=$1
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
 
     echo "Waiting for NVIDIADriver/${expected_name} to be the only default"
     while :; do
@@ -67,14 +68,13 @@ wait_for_default_nvidiadriver() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${expected_name} to be the only default"
             kubectl get nvidiadriver
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
@@ -120,7 +120,8 @@ create_nvidiadriver() {
 
 wait_for_nvidiadriver_owner() {
     local driver_name=$1
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
     local gpu_node_count
 
     gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
@@ -133,7 +134,7 @@ wait_for_nvidiadriver_owner() {
             break
         fi
 
-        if [[ "${current_time}" -gt $((60 * 15)) ]]; then
+        if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} ownership"
             kubectl get nodes -l nvidia.com/gpu.present=true -o json |
                 jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
@@ -142,7 +143,6 @@ wait_for_nvidiadriver_owner() {
 
         echo "NVIDIADriver/${driver_name} owns ${owned_count}/${gpu_node_count} GPU node(s)"
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
@@ -154,7 +154,8 @@ get_nvidiadriver_daemonsets() {
 
 wait_for_nvidiadriver_daemonsets() {
     local driver_name=$1
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
 
     echo "Waiting for daemonsets owned by NVIDIADriver/${driver_name}"
     while :; do
@@ -164,14 +165,13 @@ wait_for_nvidiadriver_daemonsets() {
             break
         fi
 
-        if [[ "${current_time}" -gt $((60 * 15)) ]]; then
+        if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
             echo "timeout reached waiting for daemonsets owned by NVIDIADriver/${driver_name}"
             kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
@@ -184,20 +184,20 @@ test_driver_image_updates() {
     fi
 
     # Verify update is applied to Driver Daemonset
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
     while :; do
         if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e --arg version "${TARGET_DRIVER_VERSION}" 'length > 0 and all(.[]; .spec.template.spec.containers[0].image | contains($version))' >/dev/null; then
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "Image update failed for driver daemonset to version $TARGET_DRIVER_VERSION"
             get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
     echo "driver daemonset image updated successfully to version $TARGET_DRIVER_VERSION"
 
@@ -223,20 +223,20 @@ test_custom_labels_override() {
 
   # Wait for the operator to update the pod template with new labels
   echo "Waiting for DaemonSet pod template to be updated with new labels..."
-  local current_time=0
+  # SECONDS counts from shell start, so record a baseline and measure against it.
+  local start_time=${SECONDS}
   while :; do
     if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e 'length > 0 and all(.[]; .spec.template.metadata.labels.cloudprovider == "aws" and .spec.template.metadata.labels.platform == "kubernetes")' >/dev/null; then
       break
     fi
 
-    if [[ "${current_time}" -gt 120 ]]; then
+    if [[ $((SECONDS - start_time)) -gt 120 ]]; then
       echo "timeout reached waiting for DaemonSet pod template labels"
       get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
       exit 1
     fi
 
     sleep 5
-    current_time=$((${current_time} + 5))
   done
 
   # Delete driver pod to force recreation with updated labels. Existing pods are not automatically restarted due to the DaemonSet's 'OnDelete` updateStrategy.
@@ -276,7 +276,8 @@ assert_nvidiadriver_owner_count() {
 wait_for_nvidiadriver_condition_message() {
     local driver_name=$1
     local message=$2
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
 
     echo "Waiting for NVIDIADriver/${driver_name} status message to contain: ${message}"
     while :; do
@@ -287,20 +288,20 @@ wait_for_nvidiadriver_condition_message() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} status message"
             kubectl get nvidiadriver/"${driver_name}" -o yaml
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_ready() {
     local driver_name=$1
-    local current_time=0
+    # SECONDS counts from shell start, so record a baseline and measure against it.
+    local start_time=${SECONDS}
 
     echo "Waiting for NVIDIADriver/${driver_name} to report Ready"
     while :; do
@@ -312,14 +313,13 @@ wait_for_nvidiadriver_ready() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} to report Ready"
             kubectl get nvidiadriver/"${driver_name}" -o yaml
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 

From 688e155ddf8ba22bc71d44dba396d72fac9d0f25 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 17:19:06 -0700
Subject: [PATCH 06/14] fix(ci): upload e2e logs on cancelled runs too

The archive step was gated on failure(), but a job stopped by
timeout-minutes is cancelled rather than failed, so failure() evaluates
false and the upload is skipped. That loses the logs on exactly the runs
that are hardest to diagnose.

Now that the e2e tests run from the runner, the polling loops in
tests/scripts talk to the API server over the internet rather than over
loopback, so a job is more likely to reach the 90 minute cap than it was
when everything ran on the node. Switch both jobs to always() so a
cancelled run still produces artifacts.

The step still runs before the credential cleanup and still uploads only
./logs/, so neither the kubeconfig nor the SSH key can end up in the
artifact.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 1f018469c5..c43393d161 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -205,7 +205,9 @@ jobs:
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
           helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        # A job stopped by timeout-minutes is cancelled, not failed, so
+        # failure() would skip the upload on the runs we most need logs from.
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: containerd-e2e-test-logs
@@ -338,7 +340,9 @@ jobs:
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
           helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        # A job stopped by timeout-minutes is cancelled, not failed, so
+        # failure() would skip the upload on the runs we most need logs from.
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: nvidiadriver-e2e-test-logs

From 7190e20469dadca26f388d7159a6b0b36a4c8d45 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 18:58:47 -0700
Subject: [PATCH 07/14] fix(ci): survive a transient API outage during the
 driver upgrade

Both e2e jobs died in wait_for_driver_upgrade_done. The driver upgrade
restarts the container runtime on the node, and now that the tests drive the
cluster from the runner rather than from the node itself, the API server is
briefly unreachable across the public address instead of on localhost. The
first casualty was the opening kubectl in print_driver_upgrade_debug, which
unlike its three siblings had no guard, so a debug dump ended the run.

Guard it, and make the wait itself tolerate the outage: the node count and the
per-node upgrade state are now read through checks that treat a failure as
"not upgraded yet" and retry until the existing wall-clock deadline expires.
A count that could not be read stays empty rather than defaulting to zero, so
an unreachable API can never be mistaken for a finished upgrade. Failures are
announced with a timestamp so the next run shows how long such an outage
lasts, which this one died too quickly to reveal.

The calls in these two functions also carry an explicit request timeout, since
the default behaviour was to spend thirty seconds per call discovering that
the address was black-holed.

The same shape exists elsewhere: a kubectl whose output only exists for a
human to read, usually just before exit 1, sitting unguarded next to siblings
that already end in || true. Guard those too, in the readiness and log
collection loops in checks.sh and in the timeout dumps in
update-clusterpolicy.sh, migrate-clusterpolicy-to-nvidiadriver.sh and
update-nvidiadriver.sh. Assertions are left alone: a check that cannot reach
the API still fails the test.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/checks.sh                       | 100 +++++++++++++-----
 .../migrate-clusterpolicy-to-nvidiadriver.sh  |  10 +-
 tests/scripts/update-clusterpolicy.sh         |   2 +-
 tests/scripts/update-nvidiadriver.sh          |  14 +--
 4 files changed, 85 insertions(+), 41 deletions(-)

diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 437b6a299d..073e0c5426 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -6,7 +6,7 @@ check_pod_ready() {
 	local start_time=${SECONDS}
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
+		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} || true
 
 		echo "Checking $pod_label pod readiness"
 		is_pod_ready=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
@@ -28,7 +28,7 @@ check_pod_ready() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n ${TEST_NAMESPACE} || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -41,7 +41,7 @@ check_pod_deleted() {
 	local start_time=${SECONDS}
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
+		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} || true
 
 		echo "Checking if $pod_label pod has been deleted"
 		# note: $(kubectl get pods <options> -o jsonpath='.items' | jq length) does not work for older kubectl clients
@@ -60,7 +60,7 @@ check_pod_deleted() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n ${TEST_NAMESPACE} || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -72,7 +72,7 @@ check_no_restarts() {
 	restartCount=$(kubectl get pod -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "$pod_label restarted multiple times: $restartCount"
-		kubectl logs -p -lapp=$pod_label --all-containers -n ${TEST_NAMESPACE}
+		kubectl logs -p -lapp=$pod_label --all-containers -n ${TEST_NAMESPACE} || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for pod $pod_label"
@@ -120,7 +120,7 @@ collect_pod_logs() {
 		ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
 		echo "Generating logs for pod: ${pod} ns: ${ns}"
 		echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
-		kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
+		kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe" || true
 		kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
 	done
 }
@@ -160,7 +160,7 @@ check_gpu_pod_ready() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods --all-namespaces
+		kubectl get pods --all-namespaces || true
 
 		if [[ "${elapsed}" -ge "${next_collection}" ]]; then
 			collect_pod_logs "${log_dir}" "${pods}"
@@ -169,7 +169,7 @@ check_gpu_pod_ready() {
 
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
-		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs"
+		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs" || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5;
@@ -182,7 +182,7 @@ check_nvidia_driver_pods_ready() {
 	local start_time=${SECONDS}
 	while :; do
 		echo "Checking nvidia driver pod"
-		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE}
+		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} || true
 
 		echo "Checking nvidia driver pod readiness"
 		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
@@ -204,7 +204,7 @@ check_nvidia_driver_pods_ready() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n ${TEST_NAMESPACE} || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -215,34 +215,46 @@ check_no_driver_pod_restarts() {
 	restartCount=$(kubectl get pod -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "nvidia driver pod restarted multiple times: $restartCount"
-		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n ${TEST_NAMESPACE}
+		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n ${TEST_NAMESPACE} || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for the nvidia driver pod"
 	return 0
 }
 
+# Report that the cluster API could not be reached. The tests now run from a
+# runner outside the cluster, so any call can fail while the node restarts its
+# container runtime during a driver upgrade. Callers treat this as "not ready
+# yet" and keep retrying until their own deadline expires. The timestamp is
+# here so that a later run can show whether such an outage is brief or
+# permanent.
+api_unreachable() {
+	echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') WARNING: cluster API unreachable while ${1}; treating as not ready and retrying"
+}
+
+# Purely diagnostic. Every call here is guarded so that a debug dump can never
+# be the thing that ends the run.
 print_driver_upgrade_debug() {
 	echo "current state of driver upgrade"
-	kubectl get node -l nvidia.com/gpu.present \
-		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
+	kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" \
+		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 
 	echo ""
 	echo "driver pods"
-	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
 
 	echo ""
 	echo "gpu operator operands"
-	kubectl get pods -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get pods -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
 
 	echo ""
 	echo "driver daemonsets"
-	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
 
 	echo ""
 	echo "NVIDIADriver status"
 	local nvidiadriver_status
-	if nvidiadriver_status=$(kubectl get nvidiadriver -o json 2>/dev/null); then
+	if nvidiadriver_status=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" 2>/dev/null); then
 		echo "${nvidiadriver_status}" | jq -r '
 			(["NAME", "DEFAULT", "STATE", "REASON", "MESSAGE"] | @tsv),
 			(
@@ -263,7 +275,6 @@ print_driver_upgrade_debug() {
 }
 
 wait_for_driver_upgrade_done() {
-	gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present --no-headers | wc -l)
 	# SECONDS counts from shell start, so record a baseline and measure against it.
 	local start_time=${SECONDS}
 	local elapsed=0
@@ -271,20 +282,53 @@ wait_for_driver_upgrade_done() {
 	# much longer than the nominal sleep, so track a due time instead of testing
 	# the elapsed time for divisibility, which would skip dumps entirely.
 	local next_debug=0
+	local node_list=""
+	local upgraded_count=0
+	local upgrade_state=""
+
+	# The driver upgrade restarts the container runtime on the node, so from a
+	# runner outside the cluster every query below can fail for a while. A failed
+	# query means "not upgraded yet" and is retried until the deadline; it must
+	# never be read as the upgrade having finished.
+	gpu_node_count=""
+
 	echo "waiting for the gpu driver upgrade to complete"
 	while :; do
-		local upgraded_count=0
-		for node in $(kubectl get nodes -o NAME); do
-			upgrade_state=$(kubectl get $node -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}')
-			if [ "${upgrade_state}" = "upgrade-done" ]; then
-				upgraded_count=$((${upgraded_count} + 1))
+		upgraded_count=0
+
+		# Resolve the expected node count once and keep it. Re-reading it every
+		# iteration would risk sampling a transient count while the upgrade churns
+		# node labels, which could make the comparison below succeed early.
+		if [[ -z "${gpu_node_count}" ]]; then
+			if node_list=$(kubectl get node -l nvidia.com/gpu.present --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+				gpu_node_count=$(echo "${node_list}" | grep -c . || true)
+			else
+				api_unreachable "counting the GPU nodes"
 			fi
-		done
-		if [[ $upgraded_count -eq $gpu_node_count ]]; then
+		fi
+
+		if node_list=$(kubectl get nodes -o NAME --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+			for node in ${node_list}; do
+				if upgrade_state=$(kubectl get "$node" -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+					if [ "${upgrade_state}" = "upgrade-done" ]; then
+						upgraded_count=$((upgraded_count + 1))
+					fi
+				else
+					api_unreachable "reading the upgrade state of ${node}"
+				fi
+			done
+		else
+			api_unreachable "listing the nodes"
+		fi
+
+		# The node count guard keeps an unreachable API from being mistaken for a
+		# finished upgrade, which is what comparing 0 against an empty count would
+		# otherwise do.
+		if [[ -n "${gpu_node_count}" ]] && [[ $upgraded_count -eq $gpu_node_count ]]; then
 			echo "gpu driver upgrade completed successfully"
 			break;
 		else
-			echo "gpu driver still in progress. $upgraded_count/$gpu_node_count node(s) upgraded"
+			echo "gpu driver still in progress. $upgraded_count/${gpu_node_count:-unknown} node(s) upgraded"
 		fi
 
 		elapsed=$((SECONDS - start_time))
@@ -298,8 +342,8 @@ wait_for_driver_upgrade_done() {
 			print_driver_upgrade_debug
 			next_debug=$((elapsed + 30))
 		else
-			kubectl get node -l nvidia.com/gpu.present \
-				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
+			kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" \
+				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 		fi
 		
 		echo "Sleeping 5 seconds"
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index 4f56d9f198..b30baf7049 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -34,7 +34,7 @@ wait_for_legacy_driver_daemonset_deleted() {
 
         if [[ "${elapsed_time}" -gt 300 ]]; then
             echo "timeout reached waiting for legacy driver DaemonSet deletion"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide || true
             exit 1
         fi
 
@@ -57,7 +57,7 @@ wait_for_orphaned_legacy_driver_pod() {
 
         if [[ "${elapsed_time}" -gt 300 ]]; then
             echo "timeout reached waiting for legacy driver pod to become orphaned"
-            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml
+            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
             exit 1
         fi
 
@@ -127,7 +127,7 @@ wait_for_nvidiadriver_daemonset() {
 
         if [[ "${elapsed_time}" -gt 300 ]]; then
             echo "timeout reached waiting for NVIDIADriver-owned driver DaemonSet"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml || true
             exit 1
         fi
 
@@ -162,14 +162,14 @@ wait_for_legacy_driver_pod_deleted() {
 legacy_driver_pod=$(kubectl get pod -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}')
 if [[ -z "${legacy_driver_pod}" ]]; then
     echo "legacy ClusterPolicy driver pod not found"
-    kubectl get pods -n "${TEST_NAMESPACE}" -o wide
+    kubectl get pods -n "${TEST_NAMESPACE}" -o wide || true
     exit 1
 fi
 
 operator_name=$(get_helm_release_name)
 if [[ -z "${operator_name}" ]]; then
     echo "GPU Operator Helm release not found in namespace ${TEST_NAMESPACE}"
-    ${HELM} list -n "${TEST_NAMESPACE}"
+    ${HELM} list -n "${TEST_NAMESPACE}" || true
     exit 1
 fi
 
diff --git a/tests/scripts/update-clusterpolicy.sh b/tests/scripts/update-clusterpolicy.sh
index 7a53901c74..6aacbdb107 100755
--- a/tests/scripts/update-clusterpolicy.sh
+++ b/tests/scripts/update-clusterpolicy.sh
@@ -150,7 +150,7 @@ test_gpu_sharing() {
     kubectl wait --for=condition=available --timeout=300s deployment/nvidia-plugin-test -n $TEST_NAMESPACE
     if [ $? -ne 0 ]; then
         echo "cannot run parallel pods with GPU sharing enabled"
-        kubectl get pods -l app=nvidia-plugin-test -n $TEST_NAMESPACE
+        kubectl get pods -l app=nvidia-plugin-test -n $TEST_NAMESPACE || true
         exit 1
     fi
 
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index efab216afe..4682db5835 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -70,7 +70,7 @@ wait_for_default_nvidiadriver() {
 
         if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${expected_name} to be the only default"
-            kubectl get nvidiadriver
+            kubectl get nvidiadriver || true
             exit 1
         fi
 
@@ -83,7 +83,7 @@ test_arbitrary_name_default_nvidiadriver() {
     current_default=$(get_default_nvidiadriver_name)
     if [[ -z "${current_default}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver
+        kubectl get nvidiadriver || true
         exit 1
     fi
 
@@ -110,7 +110,7 @@ create_nvidiadriver() {
     default_name=$(get_default_nvidiadriver_name)
     if [[ -z "${default_name}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver
+        kubectl get nvidiadriver || true
         exit 1
     fi
 
@@ -167,7 +167,7 @@ wait_for_nvidiadriver_daemonsets() {
 
         if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
             echo "timeout reached waiting for daemonsets owned by NVIDIADriver/${driver_name}"
-            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml
+            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml || true
             exit 1
         fi
 
@@ -253,7 +253,7 @@ test_custom_labels_override() {
   gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
   if [[ "${labeled_pod_count}" -ne "${gpu_node_count}" ]]; then
     echo "Custom labels are missing from one or more NVIDIADriver/${NVIDIA_DRIVER_NAME} pods"
-    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels
+    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels || true
     exit 1
   fi
 }
@@ -290,7 +290,7 @@ wait_for_nvidiadriver_condition_message() {
 
         if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} status message"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml
+            kubectl get nvidiadriver/"${driver_name}" -o yaml || true
             exit 1
         fi
 
@@ -315,7 +315,7 @@ wait_for_nvidiadriver_ready() {
 
         if [[ $((SECONDS - start_time)) -gt 120 ]]; then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} to report Ready"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml
+            kubectl get nvidiadriver/"${driver_name}" -o yaml || true
             exit 1
         fi
 

From e5eddb3270d7b6ece80b505b211b3816cbcaa7a6 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Wed, 5 Aug 2026 21:56:05 -0700
Subject: [PATCH 08/14] refactor(ci): tidy the e2e test scripts

Store a deadline rather than a start time in the polling loops. Comparing
SECONDS against a deadline computed once says what the loop means without the
subtraction, and it removes the comment each loop was carrying to explain that
SECONDS counts from shell start rather than from the loop.

migrate-clusterpolicy-to-nvidiadriver.sh had six loops still counting sleeps
instead of measuring time, the same bug already fixed in checks.sh and
update-nvidiadriver.sh. It runs on the containerd path, so convert those too.

check_gpu_pod_ready listed every pod in the cluster as json on every pass of a
five second loop, but only used the result when it regenerated the log files
every thirty seconds. Fetch it where it is used, ask for two custom columns
instead of the whole object, and read it with the shell rather than a jq
invocation per pod. The same loop printed the pod table once for the console
and fetched it again for the log file, which tee does in one call.

The request timeout default now lives in .definitions.sh with the other
defaults instead of being repeated at each use.

Also guard five more diagnostic calls that the earlier pass missed, replace
the runtime comparison in node-operations.sh with a case statement, and drop
the argument and readability checks in node-exec.sh that only repeat what
node-operations.sh and the redirect already report.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml              |  16 +--
 tests/scripts/.definitions.sh                 |   1 +
 tests/scripts/checks.sh                       | 128 +++++++++---------
 .../migrate-clusterpolicy-to-nvidiadriver.sh  |  32 ++---
 tests/scripts/node-exec.sh                    |  50 ++-----
 tests/scripts/node-operations.sh              |  19 +--
 tests/scripts/update-clusterpolicy.sh         |   2 +-
 tests/scripts/update-nvidiadriver.sh          |  43 +++---
 8 files changed, 125 insertions(+), 166 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index c43393d161..00bfa6fcb4 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -194,12 +194,12 @@ jobs:
           # locally when NODE_SSH_HOST is empty. Refuse to start rather than
           # let crictl run against the shared runner.
           test -n "${NODE_SSH_HOST}"
-          mkdir -p "${GITHUB_WORKSPACE}/logs"
+          mkdir -p "${LOG_DIR}"
           ./tests/cases/defaults.sh
       - name: Collect cluster diagnostics
         if: always()
         run: |
-          mkdir -p "${LOG_DIR}" || true
+          mkdir -p "${LOG_DIR}"
           kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
           kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
@@ -216,8 +216,8 @@ jobs:
       - name: Remove credentials from the runner
         if: always()
         run: |
-          rm -rf "${RUNNER_TEMP}/holodeck-ssh" || true
-          rm -f "${KUBECONFIG}" || true
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"
 
   e2e-tests-nvidiadriver:
     needs: [variables, publish-helm-oci-chart]
@@ -329,12 +329,12 @@ jobs:
           # locally when NODE_SSH_HOST is empty. Refuse to start rather than
           # let crictl run against the shared runner.
           test -n "${NODE_SSH_HOST}"
-          mkdir -p "${GITHUB_WORKSPACE}/logs"
+          mkdir -p "${LOG_DIR}"
           ./tests/cases/nvidia-driver.sh
       - name: Collect cluster diagnostics
         if: always()
         run: |
-          mkdir -p "${LOG_DIR}" || true
+          mkdir -p "${LOG_DIR}"
           kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
           kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
@@ -351,5 +351,5 @@ jobs:
       - name: Remove credentials from the runner
         if: always()
         run: |
-          rm -rf "${RUNNER_TEMP}/holodeck-ssh" || true
-          rm -f "${KUBECONFIG}" || true
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"
diff --git a/tests/scripts/.definitions.sh b/tests/scripts/.definitions.sh
index 737a019c6e..1cdd561ecc 100644
--- a/tests/scripts/.definitions.sh
+++ b/tests/scripts/.definitions.sh
@@ -14,6 +14,7 @@ TERRAFORM="terraform -chdir=${TERRAFORM_DIR}"
 
 # Set default values if not defined
 : ${HELM:="helm"}
+: "${KUBECTL_REQUEST_TIMEOUT:="15s"}"
 : ${LOG_DIR:="/tmp/logs"}
 : ${PROJECT:="$(basename "${PROJECT_DIR}")"}
 : ${TEST_NAMESPACE:="test-operator"}
diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 073e0c5426..8f6d571de6 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -2,11 +2,10 @@
 
 check_pod_ready() {
 	local pod_label=$1
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
+	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} || true
+		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" || true
 
 		echo "Checking $pod_label pod readiness"
 		is_pod_ready=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
@@ -22,13 +21,13 @@ check_pod_ready() {
 			fi
 		fi
 
-		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE} || true
+		kubectl get pods -n "${TEST_NAMESPACE}" || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -37,15 +36,20 @@ check_pod_ready() {
 
 check_pod_deleted() {
 	local pod_label=$1
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
+	local deadline=$((SECONDS + 60 * 45))
+	local pod_list
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} || true
+		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" || true
 
 		echo "Checking if $pod_label pod has been deleted"
-		# note: $(kubectl get pods <options> -o jsonpath='.items' | jq length) does not work for older kubectl clients
-		num_pods=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -o json | jq '.items' | jq length)
+		# Leave the count empty when the query itself fails, so that an
+		# unreachable API is never read as the pod having been deleted.
+		if pod_list=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --no-headers); then
+			num_pods=$(echo "${pod_list}" | grep -c . || true)
+		else
+			num_pods=""
+		fi
 
 		if [ "${num_pods}" = 0 ]; then
 			echo "Pod $pod_label has been deleted"
@@ -54,13 +58,13 @@ check_pod_deleted() {
 			echo "Pod $pod_label has not been deleted"
 		fi
 
-		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE} || true
+		kubectl get pods -n "${TEST_NAMESPACE}" || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -72,7 +76,7 @@ check_no_restarts() {
 	restartCount=$(kubectl get pod -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "$pod_label restarted multiple times: $restartCount"
-		kubectl logs -p -lapp=$pod_label --all-containers -n ${TEST_NAMESPACE} || true
+		kubectl logs -p -lapp="${pod_label}" --all-containers -n "${TEST_NAMESPACE}" || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for pod $pod_label"
@@ -109,67 +113,65 @@ test_restart_operator() {
 	exit 1
 }
 
-# Regenerate the describe and log files for every pod passed in. The log files
-# are rewritten in full rather than tailed, so that the artifact left behind
-# holds the complete output of every container.
+# custom-columns keeps this to a few hundred bytes; the equivalent -o json is
+# hundreds of kilobytes, fetched over the WAN on every collection.
+list_all_pods() {
+	kubectl get pods --all-namespaces -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name --no-headers || true
+}
+
+# Logs are rewritten in full rather than tailed: this file is what the job
+# uploads, so truncating it would discard the failure output it exists to keep.
 collect_pod_logs() {
 	local log_dir=$1
 	local pods=$2
+	local ns pod
 
-	for pod in $(echo "$pods" | jq -r .[].name); do
-		ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
+	while read -r ns pod; do
+		[[ -n "${pod}" ]] || continue
 		echo "Generating logs for pod: ${pod} ns: ${ns}"
 		echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
 		kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe" || true
 		kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
-	done
+	done <<< "${pods}"
 }
 
 check_gpu_pod_ready() {
 	local log_dir=$1
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
-	local elapsed=0
+	local deadline=$((SECONDS + 60 * 45))
 	# Regenerating every pod's describe and log files is the expensive part of
 	# this loop, so it runs on its own slower cadence while the readiness check
-	# below keeps polling every 5 seconds. Track the time at which the next
-	# regeneration is due rather than testing the elapsed time for divisibility,
-	# which would skip regenerations when an iteration runs long.
+	# below keeps polling every 5 seconds.
 	local next_collection=0
 
 	# Ensure the log directory exists
 	mkdir -p ${log_dir}
 
 	while :; do
-		pods="$(kubectl get --all-namespaces pods -o json | jq '.items[] | {name: .metadata.name, ns: .metadata.namespace}' | jq -s -c .)"
-		status=$(kubectl get pods gpu-operator-test -o json | jq -r .status.phase)
+		status=$(kubectl get pods gpu-operator-test -o jsonpath='{.status.phase}' || true)
 		if [ "${status}" = "Succeeded" ]; then
 			echo "GPU pod terminated successfully"
 			rc=0
-			collect_pod_logs "${log_dir}" "${pods}"
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			break;
 		fi
 
-		elapsed=$((SECONDS - start_time))
-		if [[ "${elapsed}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			# Collect once more so that the artifact reflects the state at the
 			# timeout rather than the state at the last scheduled collection.
-			collect_pod_logs "${log_dir}" "${pods}"
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			exit 1
 		fi
 
-		# Echo useful information on stdout
-		kubectl get pods --all-namespaces || true
-
-		if [[ "${elapsed}" -ge "${next_collection}" ]]; then
-			collect_pod_logs "${log_dir}" "${pods}"
-			next_collection=$((elapsed + 30))
-		fi
-
+		# Echo useful information on stdout and record it at the same time
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
-		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs" || true
+		kubectl get pods --all-namespaces | tee -a "${log_dir}/cluster.logs" || true
+
+		if (( SECONDS >= next_collection )); then
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
+			next_collection=$((SECONDS + 30))
+		fi
 
 		echo "Sleeping 5 seconds"
 		sleep 5;
@@ -178,11 +180,10 @@ check_gpu_pod_ready() {
 
 # TODO: deduplicate the logic found in this file by moving the duplicate to a common method and parameterizing the labels to select on
 check_nvidia_driver_pods_ready() {
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
+	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking nvidia driver pod"
-		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} || true
+		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" || true
 
 		echo "Checking nvidia driver pod readiness"
 		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
@@ -198,13 +199,13 @@ check_nvidia_driver_pods_ready() {
 			fi
 		fi
 
-		if [[ $((SECONDS - start_time)) -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE} || true
+		kubectl get pods -n "${TEST_NAMESPACE}" || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -215,7 +216,7 @@ check_no_driver_pod_restarts() {
 	restartCount=$(kubectl get pod -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "nvidia driver pod restarted multiple times: $restartCount"
-		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n ${TEST_NAMESPACE} || true
+		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n "${TEST_NAMESPACE}" || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for the nvidia driver pod"
@@ -236,25 +237,25 @@ api_unreachable() {
 # be the thing that ends the run.
 print_driver_upgrade_debug() {
 	echo "current state of driver upgrade"
-	kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" \
+	kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
 		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 
 	echo ""
 	echo "driver pods"
-	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
+	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "gpu operator operands"
-	kubectl get pods -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
+	kubectl get pods -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "driver daemonsets"
-	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" || true
+	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "NVIDIADriver status"
 	local nvidiadriver_status
-	if nvidiadriver_status=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" 2>/dev/null); then
+	if nvidiadriver_status=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" 2>/dev/null); then
 		echo "${nvidiadriver_status}" | jq -r '
 			(["NAME", "DEFAULT", "STATE", "REASON", "MESSAGE"] | @tsv),
 			(
@@ -275,12 +276,10 @@ print_driver_upgrade_debug() {
 }
 
 wait_for_driver_upgrade_done() {
-	# SECONDS counts from shell start, so record a baseline and measure against it.
-	local start_time=${SECONDS}
-	local elapsed=0
-	# Next elapsed time at which the full debug dump is due. Iterations can take
-	# much longer than the nominal sleep, so track a due time instead of testing
-	# the elapsed time for divisibility, which would skip dumps entirely.
+	local deadline=$((SECONDS + 60 * 45))
+	# Time at which the next full debug dump is due. Iterations can take much
+	# longer than the nominal sleep, so track a due time rather than testing the
+	# elapsed time for divisibility, which would skip dumps entirely.
 	local next_debug=0
 	local node_list=""
 	local upgraded_count=0
@@ -300,16 +299,16 @@ wait_for_driver_upgrade_done() {
 		# iteration would risk sampling a transient count while the upgrade churns
 		# node labels, which could make the comparison below succeed early.
 		if [[ -z "${gpu_node_count}" ]]; then
-			if node_list=$(kubectl get node -l nvidia.com/gpu.present --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+			if node_list=$(kubectl get node -l nvidia.com/gpu.present --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
 				gpu_node_count=$(echo "${node_list}" | grep -c . || true)
 			else
 				api_unreachable "counting the GPU nodes"
 			fi
 		fi
 
-		if node_list=$(kubectl get nodes -o NAME --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+		if node_list=$(kubectl get nodes -o NAME --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
 			for node in ${node_list}; do
-				if upgrade_state=$(kubectl get "$node" -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}"); then
+				if upgrade_state=$(kubectl get "$node" -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
 					if [ "${upgrade_state}" = "upgrade-done" ]; then
 						upgraded_count=$((upgraded_count + 1))
 					fi
@@ -331,18 +330,17 @@ wait_for_driver_upgrade_done() {
 			echo "gpu driver still in progress. $upgraded_count/${gpu_node_count:-unknown} node(s) upgraded"
 		fi
 
-		elapsed=$((SECONDS - start_time))
-		if [[ "${elapsed}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			print_driver_upgrade_debug
 			exit 1;
 		fi
 
-		if [[ "${elapsed}" -ge "${next_debug}" ]]; then
+		if (( SECONDS >= next_debug )); then
 			print_driver_upgrade_debug
-			next_debug=$((elapsed + 30))
+			next_debug=$((SECONDS + 30))
 		else
-			kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT:-15s}" \
+			kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
 				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 		fi
 		
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index b30baf7049..8fdf26fecf 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -23,7 +23,7 @@ get_helm_release_name() {
 }
 
 wait_for_legacy_driver_daemonset_deleted() {
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for ClusterPolicy-owned driver DaemonSet to be deleted"
     while :; do
@@ -32,20 +32,19 @@ wait_for_legacy_driver_daemonset_deleted() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver DaemonSet deletion"
             kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_orphaned_legacy_driver_pod() {
     local pod_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for legacy driver pod/${pod_name} to become orphaned"
     while :; do
@@ -55,19 +54,18 @@ wait_for_orphaned_legacy_driver_pod() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver pod to become orphaned"
             kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_default_nvidiadriver() {
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for default NVIDIADriver to be rendered"
     while :; do
@@ -76,20 +74,19 @@ wait_for_default_nvidiadriver() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for default NVIDIADriver"
             kubectl get nvidiadriver || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_owner_labels() {
     local driver_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
     local gpu_node_count
 
     gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
@@ -101,21 +98,20 @@ wait_for_nvidiadriver_owner_labels() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver owner labels"
             kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_daemonset() {
     local driver_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for NVIDIADriver-owned driver DaemonSet"
     while :; do
@@ -125,20 +121,19 @@ wait_for_nvidiadriver_daemonset() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver-owned driver DaemonSet"
             kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_legacy_driver_pod_deleted() {
     local pod_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for orphaned legacy driver pod/${pod_name} to be deleted by the upgrade flow"
     while :; do
@@ -146,7 +141,7 @@ wait_for_legacy_driver_pod_deleted() {
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for orphaned legacy driver pod deletion"
             print_driver_upgrade_debug
             kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
@@ -155,7 +150,6 @@ wait_for_legacy_driver_pod_deleted() {
 
         print_driver_upgrade_debug
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
diff --git a/tests/scripts/node-exec.sh b/tests/scripts/node-exec.sh
index 9f4b351e4d..8c72cfb64e 100755
--- a/tests/scripts/node-exec.sh
+++ b/tests/scripts/node-exec.sh
@@ -16,32 +16,13 @@
 
 set -euo pipefail
 
-# This script dispatches a host-mutating operation to the node hosting the
-# cluster. When NODE_SSH_HOST is set the operation is streamed to the node over
-# SSH; otherwise it is executed locally, which is the developer path where the
-# tests already run on the node itself.
+# With NODE_SSH_HOST unset the operation runs locally, which is the developer
+# path where the tests already run on the node. CI must therefore set it, or
+# node operations would target the runner instead of the cluster node.
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
 
-usage() {
-	cat <<'EOF'
-Usage: node-exec.sh <operation> [args...]
-
-Runs one of the node-operations.sh operations on the node hosting the cluster.
-
-Environment:
-  NODE_SSH_HOST         user@host of the node, e.g. ubuntu@ec2-1-2-3-4.compute.amazonaws.com.
-                        If unset or empty the operation is executed locally.
-  NODE_SSH_KEY          Path to the private key. Required when NODE_SSH_HOST is set.
-  NODE_SSH_KNOWN_HOSTS  Path to the known_hosts file. Required when NODE_SSH_HOST is set.
-
-Operations:
-  load-modules
-  restart-operator-container <runtime>
-EOF
-}
-
 require_readable_file() {
 	local name="${1}"
 	local value="${2}"
@@ -56,30 +37,19 @@ require_readable_file() {
 	fi
 }
 
-if [[ $# -lt 1 ]]; then
-	usage >&2
-	exit 2
-fi
-
-if [[ ! -r "${NODE_OPERATIONS}" ]]; then
-	echo "Error: ${NODE_OPERATIONS} does not exist or is not readable" >&2
-	exit 1
-fi
-
 if [[ -z "${NODE_SSH_HOST:-}" ]]; then
 	echo "Running '$*' locally"
-	bash "${NODE_OPERATIONS}" "$@"
-	exit $?
+	exec bash "${NODE_OPERATIONS}" "$@"
 fi
 
 require_readable_file "NODE_SSH_KEY" "${NODE_SSH_KEY:-}"
 require_readable_file "NODE_SSH_KNOWN_HOSTS" "${NODE_SSH_KNOWN_HOSTS:-}"
 
-# Quote each argument so that it survives the remote shell.
-REMOTE_COMMAND="bash -s --"
-for arg in "$@"; do
-	REMOTE_COMMAND+=" $(printf '%q' "${arg}")"
-done
+# Quote the arguments so that they survive the remote shell.
+REMOTE_ARGS=""
+if (( $# )); then
+	printf -v REMOTE_ARGS ' %q' "$@"
+fi
 
 echo "Running '$*' on ${NODE_SSH_HOST}"
 ssh -i "${NODE_SSH_KEY}" \
@@ -88,4 +58,4 @@ ssh -i "${NODE_SSH_KEY}" \
 	-o StrictHostKeyChecking=accept-new \
 	-o UserKnownHostsFile="${NODE_SSH_KNOWN_HOSTS}" \
 	"${NODE_SSH_HOST}" \
-	"${REMOTE_COMMAND}" < "${NODE_OPERATIONS}"
+	"bash -s --${REMOTE_ARGS}" < "${NODE_OPERATIONS}"
diff --git a/tests/scripts/node-operations.sh b/tests/scripts/node-operations.sh
index fd46b1e80b..58c4e17c97 100755
--- a/tests/scripts/node-operations.sh
+++ b/tests/scripts/node-operations.sh
@@ -41,15 +41,15 @@ load_modules() {
 	sudo modprobe -a i2c_core ipmi_msghandler
 }
 
-# The x-prefixed comparisons and the container selection pipelines below are
-# kept as they were in tests/scripts/checks.sh so that the behaviour of the
-# restart test does not change.
-# shellcheck disable=SC2268
+# The container selection pipelines below are kept as they were in
+# tests/scripts/checks.sh so that the behaviour of the restart test does not
+# change.
 restart_operator_container() {
 	local runtime="${1:-}"
 	local container_id=""
 
-	if [[ x"${runtime}" == x"containerd" ]]; then
+	case "${runtime}" in
+	containerd)
 		# The operator is the only container that has the string '"gpu-operator"'
 		# TODO: This requires permissions on containerd.sock
 		container_id="$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')" || true
@@ -58,7 +58,8 @@ restart_operator_container() {
 			return 1
 		fi
 		sudo crictl rm --force "${container_id}"
-	elif [[ x"${runtime}" == x"docker" ]]; then
+		;;
+	docker)
 		# The operator is the only container that has the string '"gpu-operator"'
 		container_id="$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')" || true
 		if [[ -z "${container_id}" ]]; then
@@ -66,10 +67,12 @@ restart_operator_container() {
 			return 1
 		fi
 		docker kill "${container_id}"
-	else
+		;;
+	*)
 		echo "Error: unknown runtime '${runtime}'. Supported runtimes: containerd, docker" >&2
 		return 1
-	fi
+		;;
+	esac
 }
 
 main() {
diff --git a/tests/scripts/update-clusterpolicy.sh b/tests/scripts/update-clusterpolicy.sh
index 6aacbdb107..66ea4429f9 100755
--- a/tests/scripts/update-clusterpolicy.sh
+++ b/tests/scripts/update-clusterpolicy.sh
@@ -150,7 +150,7 @@ test_gpu_sharing() {
     kubectl wait --for=condition=available --timeout=300s deployment/nvidia-plugin-test -n $TEST_NAMESPACE
     if [ $? -ne 0 ]; then
         echo "cannot run parallel pods with GPU sharing enabled"
-        kubectl get pods -l app=nvidia-plugin-test -n $TEST_NAMESPACE || true
+        kubectl get pods -l app=nvidia-plugin-test -n "${TEST_NAMESPACE}" || true
         exit 1
     fi
 
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index 4682db5835..3816b09312 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -56,8 +56,7 @@ set_default_driver() {
 
 wait_for_default_nvidiadriver() {
     local expected_name=$1
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${expected_name} to be the only default"
     while :; do
@@ -68,7 +67,7 @@ wait_for_default_nvidiadriver() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${expected_name} to be the only default"
             kubectl get nvidiadriver || true
             exit 1
@@ -120,8 +119,7 @@ create_nvidiadriver() {
 
 wait_for_nvidiadriver_owner() {
     local driver_name=$1
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + $((60 * 15))))
     local gpu_node_count
 
     gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
@@ -134,10 +132,10 @@ wait_for_nvidiadriver_owner() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} ownership"
             kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
 
@@ -154,8 +152,7 @@ get_nvidiadriver_daemonsets() {
 
 wait_for_nvidiadriver_daemonsets() {
     local driver_name=$1
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + $((60 * 15))))
 
     echo "Waiting for daemonsets owned by NVIDIADriver/${driver_name}"
     while :; do
@@ -165,7 +162,7 @@ wait_for_nvidiadriver_daemonsets() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt $((60 * 15)) ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for daemonsets owned by NVIDIADriver/${driver_name}"
             kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml || true
             exit 1
@@ -184,16 +181,15 @@ test_driver_image_updates() {
     fi
 
     # Verify update is applied to Driver Daemonset
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + 120))
     while :; do
         if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e --arg version "${TARGET_DRIVER_VERSION}" 'length > 0 and all(.[]; .spec.template.spec.containers[0].image | contains($version))' >/dev/null; then
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "Image update failed for driver daemonset to version $TARGET_DRIVER_VERSION"
-            get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
+            get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" || true
             exit 1
         fi
 
@@ -223,16 +219,15 @@ test_custom_labels_override() {
 
   # Wait for the operator to update the pod template with new labels
   echo "Waiting for DaemonSet pod template to be updated with new labels..."
-  # SECONDS counts from shell start, so record a baseline and measure against it.
-  local start_time=${SECONDS}
+  local deadline=$((SECONDS + 120))
   while :; do
     if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e 'length > 0 and all(.[]; .spec.template.metadata.labels.cloudprovider == "aws" and .spec.template.metadata.labels.platform == "kubernetes")' >/dev/null; then
       break
     fi
 
-    if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+    if (( SECONDS > deadline )); then
       echo "timeout reached waiting for DaemonSet pod template labels"
-      get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
+      get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" || true
       exit 1
     fi
 
@@ -268,7 +263,7 @@ assert_nvidiadriver_owner_count() {
     if [[ "${owned_count}" -ne "${gpu_node_count}" ]]; then
         echo "Expected ${gpu_node_count} GPU node(s) to remain owned by NVIDIADriver/${driver_name}, found ${owned_count}"
         kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-            jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+            jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
         exit 1
     fi
 }
@@ -276,8 +271,7 @@ assert_nvidiadriver_owner_count() {
 wait_for_nvidiadriver_condition_message() {
     local driver_name=$1
     local message=$2
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${driver_name} status message to contain: ${message}"
     while :; do
@@ -288,7 +282,7 @@ wait_for_nvidiadriver_condition_message() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} status message"
             kubectl get nvidiadriver/"${driver_name}" -o yaml || true
             exit 1
@@ -300,8 +294,7 @@ wait_for_nvidiadriver_condition_message() {
 
 wait_for_nvidiadriver_ready() {
     local driver_name=$1
-    # SECONDS counts from shell start, so record a baseline and measure against it.
-    local start_time=${SECONDS}
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${driver_name} to report Ready"
     while :; do
@@ -313,7 +306,7 @@ wait_for_nvidiadriver_ready() {
             break
         fi
 
-        if [[ $((SECONDS - start_time)) -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} to report Ready"
             kubectl get nvidiadriver/"${driver_name}" -o yaml || true
             exit 1

From 748dcd11b55261ea0f444df0fc639f80b07279fa Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Mon, 24 Aug 2026 15:53:49 -0700
Subject: [PATCH 09/14] fix(ci): stop reading a failed query as a satisfied
 condition

Now that the tests drive the cluster from the runner, a kubectl call can fail
at any point, and several checks turned that failure into a value that happens
to satisfy what they were waiting for. Counting through a pipe reports zero
when kubectl writes nothing, so a deletion check saw the resource as gone and
an ownership check saw every node as owned. Iterating over a command
substitution runs the body no times, so the label check in
test_custom_labels_override could report success without reading a single
label. Testing a bare kubectl for non-zero treats unreachable the same as
NotFound.

Ask whether the query succeeded before reading what it said. A query that
failed reports through api_unreachable and falls through to the retry, and the
loops still stop on their existing deadlines. Deletion checks use -o name with
--ignore-not-found so an absent object is a successful query returning
nothing, which is a real answer, rather than an error.

Comparisons against a node count also require the count to be above zero. This
environment always has a GPU node, so zero means the count never arrived, and
without that guard a zero on both sides of the comparison reads as done.

The one-shot assertions fail when their query fails rather than retrying,
since there is no loop around them and a check that could not run has not
passed.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/checks.sh                       | 37 ++++++-----
 .../migrate-clusterpolicy-to-nvidiadriver.sh  | 61 +++++++++++++------
 tests/scripts/update-clusterpolicy.sh         | 12 +++-
 tests/scripts/update-nvidiadriver.sh          | 54 ++++++++++++----
 4 files changed, 117 insertions(+), 47 deletions(-)

diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 8f6d571de6..023759060e 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -43,19 +43,17 @@ check_pod_deleted() {
 		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" || true
 
 		echo "Checking if $pod_label pod has been deleted"
-		# Leave the count empty when the query itself fails, so that an
-		# unreachable API is never read as the pod having been deleted.
-		if pod_list=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --no-headers); then
-			num_pods=$(echo "${pod_list}" | grep -c . || true)
-		else
-			num_pods=""
-		fi
-
-		if [ "${num_pods}" = 0 ]; then
-			echo "Pod $pod_label has been deleted"
-			break;
+		# --ignore-not-found keeps an absent pod from being reported as a failed
+		# query, so empty output here means deleted rather than unreachable.
+		if pod_list=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+			if [ -z "${pod_list}" ]; then
+				echo "Pod $pod_label has been deleted"
+				break;
+			else
+				echo "Pod $pod_label has not been deleted"
+			fi
 		else
-			echo "Pod $pod_label has not been deleted"
+			api_unreachable "checking whether pod $pod_label has been deleted"
 		fi
 
 		if (( SECONDS > deadline )); then
@@ -233,6 +231,14 @@ api_unreachable() {
 	echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') WARNING: cluster API unreachable while ${1}; treating as not ready and retrying"
 }
 
+# Returns non-zero without printing when the query itself failed, so that a
+# genuine zero stays distinguishable from an unreachable API.
+kubectl_count() {
+	local output
+	output=$(kubectl "$@" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}") || return 1
+	echo "${output}" | grep -c . || true
+}
+
 # Purely diagnostic. Every call here is guarded so that a debug dump can never
 # be the thing that ends the run.
 print_driver_upgrade_debug() {
@@ -320,10 +326,9 @@ wait_for_driver_upgrade_done() {
 			api_unreachable "listing the nodes"
 		fi
 
-		# The node count guard keeps an unreachable API from being mistaken for a
-		# finished upgrade, which is what comparing 0 against an empty count would
-		# otherwise do.
-		if [[ -n "${gpu_node_count}" ]] && [[ $upgraded_count -eq $gpu_node_count ]]; then
+		# This environment always has a GPU node, so a zero count means the query
+		# never returned a real answer rather than that there is nothing to upgrade.
+		if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${upgraded_count}" -eq "${gpu_node_count}" ]]; then
 			echo "gpu driver upgrade completed successfully"
 			break;
 		else
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index 8fdf26fecf..4be4e4584e 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -27,9 +27,12 @@ wait_for_legacy_driver_daemonset_deleted() {
 
     echo "Waiting for ClusterPolicy-owned driver DaemonSet to be deleted"
     while :; do
-        daemonset_count=$(kubectl get daemonset -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" --no-headers 2>/dev/null | wc -l)
-        if [[ "${daemonset_count}" -eq 0 ]]; then
-            break
+        if daemonsets=$(kubectl get daemonset -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            if [[ -z "${daemonsets}" ]]; then
+                break
+            fi
+        else
+            api_unreachable "checking for the ClusterPolicy-owned driver DaemonSet"
         fi
 
         if (( SECONDS > deadline )); then
@@ -48,10 +51,14 @@ wait_for_orphaned_legacy_driver_pod() {
 
     echo "Waiting for legacy driver pod/${pod_name} to become orphaned"
     while :; do
-        owner_count=$(kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o json | jq '.metadata.ownerReferences | length')
-        if [[ "${owner_count}" -eq 0 ]]; then
-            echo "legacy driver pod/${pod_name} is orphaned"
-            break
+        if pod_json=$(kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            owner_count=$(echo "${pod_json}" | jq '.metadata.ownerReferences | length')
+            if [[ "${owner_count}" -eq 0 ]]; then
+                echo "legacy driver pod/${pod_name} is orphaned"
+                break
+            fi
+        else
+            api_unreachable "checking whether legacy driver pod/${pod_name} is orphaned"
         fi
 
         if (( SECONDS > deadline )); then
@@ -69,9 +76,13 @@ wait_for_default_nvidiadriver() {
 
     echo "Waiting for default NVIDIADriver to be rendered"
     while :; do
-        default_count=$(kubectl get nvidiadriver -o json 2>/dev/null | jq '[.items[] | select(.spec.default == true)] | length')
-        if [[ "${default_count}" -eq 1 ]]; then
-            break
+        if nvidiadrivers=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            default_count=$(echo "${nvidiadrivers}" | jq '[.items[] | select(.spec.default == true)] | length')
+            if [[ "${default_count}" -eq 1 ]]; then
+                break
+            fi
+        else
+            api_unreachable "counting the default NVIDIADrivers"
         fi
 
         if (( SECONDS > deadline )); then
@@ -87,15 +98,25 @@ wait_for_default_nvidiadriver() {
 wait_for_nvidiadriver_owner_labels() {
     local driver_name=$1
     local deadline=$((SECONDS + 300))
-    local gpu_node_count
+    local gpu_node_count=""
+    local owned_count
 
-    gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
-    echo "Waiting for ${gpu_node_count} GPU node(s) to be owned by NVIDIADriver/${driver_name}"
+    echo "Waiting for the GPU node(s) to be owned by NVIDIADriver/${driver_name}"
 
     while :; do
-        owned_count=$(kubectl get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers | wc -l)
-        if [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
-            break
+        if [[ -z "${gpu_node_count}" ]]; then
+            gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers) \
+                || api_unreachable "counting the GPU nodes"
+        fi
+
+        if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
+            # A zero total means the count never arrived; this environment always
+            # has a GPU node, so it can never be a legitimate match.
+            if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
+                break
+            fi
+        else
+            api_unreachable "counting the nodes owned by NVIDIADriver/${driver_name}"
         fi
 
         if (( SECONDS > deadline )); then
@@ -137,8 +158,12 @@ wait_for_legacy_driver_pod_deleted() {
 
     echo "Waiting for orphaned legacy driver pod/${pod_name} to be deleted by the upgrade flow"
     while :; do
-        if ! kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" >/dev/null 2>&1; then
-            break
+        if legacy_pod=$(kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            if [[ -z "${legacy_pod}" ]]; then
+                break
+            fi
+        else
+            api_unreachable "checking whether legacy driver pod/${pod_name} is deleted"
         fi
 
         if (( SECONDS > deadline )); then
diff --git a/tests/scripts/update-clusterpolicy.sh b/tests/scripts/update-clusterpolicy.sh
index 66ea4429f9..75c156857f 100755
--- a/tests/scripts/update-clusterpolicy.sh
+++ b/tests/scripts/update-clusterpolicy.sh
@@ -243,7 +243,17 @@ test_custom_labels_override() {
   for operand in $operands
   do
     echo "checking $operand labels"
-    for pod in $(kubectl get pods -n "$TEST_NAMESPACE" -l app="$operand" --output=jsonpath={.items..metadata.name})
+    # check_pod_ready has just confirmed these pods, so a failed query or an
+    # empty list is a broken assumption rather than nothing left to check.
+    if ! operand_pods=$(kubectl get pods -n "$TEST_NAMESPACE" -l app="$operand" --output=jsonpath={.items..metadata.name} --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+      echo "cannot list $operand pods to verify the overridden labels"
+      exit 1
+    fi
+    if [ -z "$operand_pods" ]; then
+      echo "no $operand pods found when verifying the overridden labels"
+      exit 1
+    fi
+    for pod in $operand_pods
     do
       cp_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath={.metadata.labels.cloudprovider})
       if [ "$cp_label_value" != "aws" ]; then
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index 3816b09312..f8da5ca7c3 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -120,16 +120,26 @@ create_nvidiadriver() {
 wait_for_nvidiadriver_owner() {
     local driver_name=$1
     local deadline=$((SECONDS + $((60 * 15))))
-    local gpu_node_count
+    local gpu_node_count=""
+    local owned_count
 
-    gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
-    echo "Waiting for ${gpu_node_count} GPU node(s) to be owned by NVIDIADriver/${driver_name}"
+    echo "Waiting for the GPU node(s) to be owned by NVIDIADriver/${driver_name}"
 
     while :; do
-        owned_count=$(kubectl get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers | wc -l)
-        if [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
-            echo "All GPU nodes are owned by NVIDIADriver/${driver_name}"
-            break
+        if [[ -z "${gpu_node_count}" ]]; then
+            gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers) \
+                || api_unreachable "counting the GPU nodes"
+        fi
+
+        if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
+            # A zero total means the count never arrived; this environment always
+            # has a GPU node, so it can never be a legitimate match.
+            if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
+                echo "All GPU nodes are owned by NVIDIADriver/${driver_name}"
+                break
+            fi
+        else
+            api_unreachable "counting the nodes owned by NVIDIADriver/${driver_name}"
         fi
 
         if (( SECONDS > deadline )); then
@@ -139,7 +149,7 @@ wait_for_nvidiadriver_owner() {
             exit 1
         fi
 
-        echo "NVIDIADriver/${driver_name} owns ${owned_count}/${gpu_node_count} GPU node(s)"
+        echo "NVIDIADriver/${driver_name} owns ${owned_count:-unknown}/${gpu_node_count:-unknown} GPU node(s)"
         sleep 5
     done
 }
@@ -244,8 +254,18 @@ test_custom_labels_override() {
   check_nvidia_driver_pods_ready
 
   echo "checking nvidia-driver-daemonset labels"
-  labeled_pod_count=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver,cloudprovider=aws,platform=kubernetes" --no-headers | wc -l)
-  gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
+  if ! labeled_pod_count=$(kubectl_count get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver,cloudprovider=aws,platform=kubernetes" --no-headers); then
+    echo "cannot count the labelled NVIDIADriver/${NVIDIA_DRIVER_NAME} pods"
+    exit 1
+  fi
+  if ! gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+    echo "cannot count the GPU nodes"
+    exit 1
+  fi
+  if [[ "${gpu_node_count}" -eq 0 ]]; then
+    echo "no GPU nodes found while verifying NVIDIADriver/${NVIDIA_DRIVER_NAME} labels"
+    exit 1
+  fi
   if [[ "${labeled_pod_count}" -ne "${gpu_node_count}" ]]; then
     echo "Custom labels are missing from one or more NVIDIADriver/${NVIDIA_DRIVER_NAME} pods"
     kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels || true
@@ -258,8 +278,18 @@ assert_nvidiadriver_owner_count() {
     local gpu_node_count
     local owned_count
 
-    gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
-    owned_count=$(kubectl get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers | wc -l)
+    if ! gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+        echo "cannot count the GPU nodes"
+        exit 1
+    fi
+    if ! owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
+        echo "cannot count the nodes owned by NVIDIADriver/${driver_name}"
+        exit 1
+    fi
+    if [[ "${gpu_node_count}" -eq 0 ]]; then
+        echo "no GPU nodes found while checking NVIDIADriver/${driver_name} ownership"
+        exit 1
+    fi
     if [[ "${owned_count}" -ne "${gpu_node_count}" ]]; then
         echo "Expected ${gpu_node_count} GPU node(s) to remain owned by NVIDIADriver/${driver_name}, found ${owned_count}"
         kubectl get nodes -l nvidia.com/gpu.present=true -o json |

From 21cb5a26e70fff70481483724fb25b6d8217b561 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Mon, 24 Aug 2026 16:13:40 -0700
Subject: [PATCH 10/14] refactor(ci): drop comments that restate the code

Keeps the licence headers and the handful of notes that stop a reader
undoing something deliberate: that node-operations.sh has to stay
self-contained because it is streamed over SSH stdin, that the log files
are the uploaded artifact and so must not be tailed, that the GPU node
count is resolved once because the upgrade churns node labels, and that
node-exec.sh runs locally when NODE_SSH_HOST is empty. The rest described
what the line below it already said, or recorded design rationale that
belongs in the commit history.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml              | 30 +++------
 tests/scripts/checks.sh                       | 65 ++++++-------------
 .../migrate-clusterpolicy-to-nvidiadriver.sh  |  3 +-
 tests/scripts/node-exec.sh                    |  6 +-
 tests/scripts/node-operations.sh              | 11 +---
 tests/scripts/update-clusterpolicy.sh         |  4 +-
 tests/scripts/update-nvidiadriver.sh          |  3 +-
 7 files changed, 36 insertions(+), 86 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 00bfa6fcb4..f2f415a8d9 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -56,10 +56,6 @@ on:
         default: false
 
 env:
-  # Pinned versions of the tooling the test scripts drive from the runner.
-  # helm used to be installed on the test node by tests/scripts/prerequisites.sh
-  # straight from the helm get-helm-3 master script, i.e. whatever release was
-  # current at the time the job ran. Pinning it here is a deliberate change.
   HELM_VERSION: v3.19.0
   # Kept in sync with spec.kubernetes.version in tests/holodeck.yaml.
   KUBECTL_VERSION: v1.35.4
@@ -92,8 +88,6 @@ jobs:
       contents: read
       id-token: write
     env:
-      # Holodeck is configured with kubernetes.remoteAccess, so it drops a
-      # kubeconfig pointing at the node's public API server endpoint here.
       KUBECONFIG: ${{ github.workspace }}/kubeconfig
       LOG_DIR: ${{ github.workspace }}/logs
     steps:
@@ -174,10 +168,8 @@ jobs:
       - name: Load kernel modules on the node
         run: |
           set -euo pipefail
-          # node-exec.sh falls back to running the operation locally when
-          # NODE_SSH_HOST is empty, which is the developer path. In CI that
-          # would run modprobe against the shared runner, so fail loudly
-          # instead.
+          # node-exec.sh runs locally when NODE_SSH_HOST is empty; on the
+          # shared runner that would modprobe the wrong machine.
           test -n "${NODE_SSH_HOST}"
           ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
@@ -190,9 +182,8 @@ jobs:
           CONTAINER_RUNTIME: "containerd"
         run: |
           set -euo pipefail
-          # The restart-operator test shells out to node-exec.sh, which runs
-          # locally when NODE_SSH_HOST is empty. Refuse to start rather than
-          # let crictl run against the shared runner.
+          # The restart test reaches node-exec.sh, which runs locally when
+          # NODE_SSH_HOST is empty; crictl would then hit the shared runner.
           test -n "${NODE_SSH_HOST}"
           mkdir -p "${LOG_DIR}"
           ./tests/cases/defaults.sh
@@ -227,8 +218,6 @@ jobs:
       contents: read
       id-token: write
     env:
-      # Holodeck is configured with kubernetes.remoteAccess, so it drops a
-      # kubeconfig pointing at the node's public API server endpoint here.
       KUBECONFIG: ${{ github.workspace }}/kubeconfig
       LOG_DIR: ${{ github.workspace }}/logs
     steps:
@@ -309,10 +298,8 @@ jobs:
       - name: Load kernel modules on the node
         run: |
           set -euo pipefail
-          # node-exec.sh falls back to running the operation locally when
-          # NODE_SSH_HOST is empty, which is the developer path. In CI that
-          # would run modprobe against the shared runner, so fail loudly
-          # instead.
+          # node-exec.sh runs locally when NODE_SSH_HOST is empty; on the
+          # shared runner that would modprobe the wrong machine.
           test -n "${NODE_SSH_HOST}"
           ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
@@ -325,9 +312,8 @@ jobs:
           CONTAINER_RUNTIME: "containerd"
         run: |
           set -euo pipefail
-          # The restart-operator test shells out to node-exec.sh, which runs
-          # locally when NODE_SSH_HOST is empty. Refuse to start rather than
-          # let crictl run against the shared runner.
+          # The restart test reaches node-exec.sh, which runs locally when
+          # NODE_SSH_HOST is empty; crictl would then hit the shared runner.
           test -n "${NODE_SSH_HOST}"
           mkdir -p "${LOG_DIR}"
           ./tests/cases/nvidia-driver.sh
diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 023759060e..beb60588e1 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -43,8 +43,6 @@ check_pod_deleted() {
 		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" || true
 
 		echo "Checking if $pod_label pod has been deleted"
-		# --ignore-not-found keeps an absent pod from being reported as a failed
-		# query, so empty output here means deleted rather than unreachable.
 		if pod_list=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
 			if [ -z "${pod_list}" ]; then
 				echo "Pod $pod_label has been deleted"
@@ -87,8 +85,6 @@ test_restart_operator() {
 	local ns=${1}
 	local runtime=${2}
 
-	# Killing the operator container mutates the node, so it is dispatched to the
-	# node itself. node-exec.sh runs it either over SSH or locally.
 	local checks_script_dir
 	checks_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 	"${checks_script_dir}"/node-exec.sh restart-operator-container "${runtime}"
@@ -111,34 +107,29 @@ test_restart_operator() {
 	exit 1
 }
 
-# custom-columns keeps this to a few hundred bytes; the equivalent -o json is
-# hundreds of kilobytes, fetched over the WAN on every collection.
 list_all_pods() {
 	kubectl get pods --all-namespaces -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name --no-headers || true
 }
 
-# Logs are rewritten in full rather than tailed: this file is what the job
-# uploads, so truncating it would discard the failure output it exists to keep.
+# Not tailed: this file is the uploaded artifact, so truncation loses the
+# failure output it exists to capture.
 collect_pod_logs() {
 	local log_dir=$1
-	local pods=$2
-	local ns pod
-
-	while read -r ns pod; do
-		[[ -n "${pod}" ]] || continue
-		echo "Generating logs for pod: ${pod} ns: ${ns}"
-		echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
-		kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe" || true
-		kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
-	done <<< "${pods}"
+	local namespaced_pods=$2
+	local namespace pod_name
+
+	while read -r namespace pod_name; do
+		[[ -n "${pod_name}" ]] || continue
+		echo "Generating logs for pod: ${pod_name} ns: ${namespace}"
+		echo "------------------------------------------------" >> "${log_dir}/${pod_name}.describe"
+		kubectl -n "${namespace}" describe pods "${pod_name}" >> "${log_dir}/${pod_name}.describe" || true
+		kubectl -n "${namespace}" logs "${pod_name}" --all-containers=true > "${log_dir}/${pod_name}.logs" || true
+	done <<< "${namespaced_pods}"
 }
 
 check_gpu_pod_ready() {
 	local log_dir=$1
 	local deadline=$((SECONDS + 60 * 45))
-	# Regenerating every pod's describe and log files is the expensive part of
-	# this loop, so it runs on its own slower cadence while the readiness check
-	# below keeps polling every 5 seconds.
 	local next_collection=0
 
 	# Ensure the log directory exists
@@ -155,13 +146,11 @@ check_gpu_pod_ready() {
 
 		if (( SECONDS > deadline )); then
 			echo "timeout reached"
-			# Collect once more so that the artifact reflects the state at the
-			# timeout rather than the state at the last scheduled collection.
+			# Refresh so the artifact reflects the timeout, not the last cadence.
 			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			exit 1
 		fi
 
-		# Echo useful information on stdout and record it at the same time
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
 		kubectl get pods --all-namespaces | tee -a "${log_dir}/cluster.logs" || true
@@ -221,26 +210,18 @@ check_no_driver_pod_restarts() {
 	return 0
 }
 
-# Report that the cluster API could not be reached. The tests now run from a
-# runner outside the cluster, so any call can fail while the node restarts its
-# container runtime during a driver upgrade. Callers treat this as "not ready
-# yet" and keep retrying until their own deadline expires. The timestamp is
-# here so that a later run can show whether such an outage is brief or
-# permanent.
 api_unreachable() {
 	echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') WARNING: cluster API unreachable while ${1}; treating as not ready and retrying"
 }
 
-# Returns non-zero without printing when the query itself failed, so that a
-# genuine zero stays distinguishable from an unreachable API.
+# Non-zero on query failure, so a genuine zero stays distinguishable from an
+# unreachable API.
 kubectl_count() {
 	local output
 	output=$(kubectl "$@" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}") || return 1
 	echo "${output}" | grep -c . || true
 }
 
-# Purely diagnostic. Every call here is guarded so that a debug dump can never
-# be the thing that ends the run.
 print_driver_upgrade_debug() {
 	echo "current state of driver upgrade"
 	kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
@@ -283,27 +264,20 @@ print_driver_upgrade_debug() {
 
 wait_for_driver_upgrade_done() {
 	local deadline=$((SECONDS + 60 * 45))
-	# Time at which the next full debug dump is due. Iterations can take much
-	# longer than the nominal sleep, so track a due time rather than testing the
-	# elapsed time for divisibility, which would skip dumps entirely.
+	# Due time, not modulo: a slow iteration can jump over an exact multiple.
 	local next_debug=0
 	local node_list=""
 	local upgraded_count=0
 	local upgrade_state=""
 
-	# The driver upgrade restarts the container runtime on the node, so from a
-	# runner outside the cluster every query below can fail for a while. A failed
-	# query means "not upgraded yet" and is retried until the deadline; it must
-	# never be read as the upgrade having finished.
 	gpu_node_count=""
 
 	echo "waiting for the gpu driver upgrade to complete"
 	while :; do
 		upgraded_count=0
 
-		# Resolve the expected node count once and keep it. Re-reading it every
-		# iteration would risk sampling a transient count while the upgrade churns
-		# node labels, which could make the comparison below succeed early.
+		# Resolved once: the upgrade churns node labels, so a per-iteration count
+		# could sample a transient value and match early.
 		if [[ -z "${gpu_node_count}" ]]; then
 			if node_list=$(kubectl get node -l nvidia.com/gpu.present --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
 				gpu_node_count=$(echo "${node_list}" | grep -c . || true)
@@ -326,8 +300,7 @@ wait_for_driver_upgrade_done() {
 			api_unreachable "listing the nodes"
 		fi
 
-		# This environment always has a GPU node, so a zero count means the query
-		# never returned a real answer rather than that there is nothing to upgrade.
+		# Zero means the count never arrived; this environment always has a GPU node.
 		if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${upgraded_count}" -eq "${gpu_node_count}" ]]; then
 			echo "gpu driver upgrade completed successfully"
 			break;
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index 4be4e4584e..511c3991cf 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -110,8 +110,7 @@ wait_for_nvidiadriver_owner_labels() {
         fi
 
         if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
-            # A zero total means the count never arrived; this environment always
-            # has a GPU node, so it can never be a legitimate match.
+            # Zero means the count never arrived; there is always a GPU node.
             if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
                 break
             fi
diff --git a/tests/scripts/node-exec.sh b/tests/scripts/node-exec.sh
index 8c72cfb64e..52529bb4f2 100755
--- a/tests/scripts/node-exec.sh
+++ b/tests/scripts/node-exec.sh
@@ -16,9 +16,8 @@
 
 set -euo pipefail
 
-# With NODE_SSH_HOST unset the operation runs locally, which is the developer
-# path where the tests already run on the node. CI must therefore set it, or
-# node operations would target the runner instead of the cluster node.
+# Unset NODE_SSH_HOST runs the operation locally, which is the developer path.
+# CI must set it or node operations would target the runner.
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
@@ -45,7 +44,6 @@ fi
 require_readable_file "NODE_SSH_KEY" "${NODE_SSH_KEY:-}"
 require_readable_file "NODE_SSH_KNOWN_HOSTS" "${NODE_SSH_KNOWN_HOSTS:-}"
 
-# Quote the arguments so that they survive the remote shell.
 REMOTE_ARGS=""
 if (( $# )); then
 	printf -v REMOTE_ARGS ' %q' "$@"
diff --git a/tests/scripts/node-operations.sh b/tests/scripts/node-operations.sh
index 58c4e17c97..2381cd6b37 100755
--- a/tests/scripts/node-operations.sh
+++ b/tests/scripts/node-operations.sh
@@ -16,11 +16,9 @@
 
 set -euo pipefail
 
-# This script runs ON the node hosting the cluster. It is either executed
-# locally or streamed over SSH stdin via `bash -s --`, which means it MUST stay
-# fully self-contained: it must not source sibling scripts such as
-# .definitions.sh and must not refer to any path relative to the repository,
-# since the repository is not guaranteed to exist on the node.
+# Runs on the cluster node, streamed over SSH stdin via `bash -s --`. Must stay
+# self-contained: no sourcing siblings, no repo-relative paths — the repository
+# is not guaranteed to exist there.
 
 usage() {
 	cat <<'EOF'
@@ -41,9 +39,6 @@ load_modules() {
 	sudo modprobe -a i2c_core ipmi_msghandler
 }
 
-# The container selection pipelines below are kept as they were in
-# tests/scripts/checks.sh so that the behaviour of the restart test does not
-# change.
 restart_operator_container() {
 	local runtime="${1:-}"
 	local container_id=""
diff --git a/tests/scripts/update-clusterpolicy.sh b/tests/scripts/update-clusterpolicy.sh
index 75c156857f..247ab7b656 100755
--- a/tests/scripts/update-clusterpolicy.sh
+++ b/tests/scripts/update-clusterpolicy.sh
@@ -243,8 +243,8 @@ test_custom_labels_override() {
   for operand in $operands
   do
     echo "checking $operand labels"
-    # check_pod_ready has just confirmed these pods, so a failed query or an
-    # empty list is a broken assumption rather than nothing left to check.
+    # check_pod_ready just confirmed these pods, so an empty list is a broken
+    # assumption, not nothing left to check.
     if ! operand_pods=$(kubectl get pods -n "$TEST_NAMESPACE" -l app="$operand" --output=jsonpath={.items..metadata.name} --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
       echo "cannot list $operand pods to verify the overridden labels"
       exit 1
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index f8da5ca7c3..788227d0f3 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -132,8 +132,7 @@ wait_for_nvidiadriver_owner() {
         fi
 
         if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
-            # A zero total means the count never arrived; this environment always
-            # has a GPU node, so it can never be a legitimate match.
+            # Zero means the count never arrived; there is always a GPU node.
             if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
                 echo "All GPU nodes are owned by NVIDIADriver/${driver_name}"
                 break

From 43ca922661a4034083db40a2f36c3d4a1e9672e4 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Mon, 24 Aug 2026 16:25:36 -0700
Subject: [PATCH 11/14] refactor(ci): make node-exec.sh constants readonly

SCRIPT_DIR and NODE_OPERATIONS are fixed once at startup and nothing
should reassign them. Declared separately from the assignment because
readonly with a command substitution swallows its exit status.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/node-exec.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/scripts/node-exec.sh b/tests/scripts/node-exec.sh
index 52529bb4f2..185167180e 100755
--- a/tests/scripts/node-exec.sh
+++ b/tests/scripts/node-exec.sh
@@ -20,7 +20,8 @@ set -euo pipefail
 # CI must set it or node operations would target the runner.
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
+readonly SCRIPT_DIR
+readonly NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
 
 require_readable_file() {
 	local name="${1}"

From 51fe9f53ddaa3601bfc6aa5205a2aaa86430a75a Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Mon, 24 Aug 2026 16:33:30 -0700
Subject: [PATCH 12/14] fix(ci): group the GITHUB_ENV writes into one redirect

actionlint's shellcheck pass flagged the three consecutive appends
(SC2129). Grouping them also makes it clear they are one unit of work.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f2f415a8d9..b96b909546 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -151,9 +151,11 @@ jobs:
           install -m 600 /dev/null "${SSH_DIR}/id_rsa"
           printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
           install -m 600 /dev/null "${SSH_DIR}/known_hosts"
-          echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}" >> "$GITHUB_ENV"
-          echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa" >> "$GITHUB_ENV"
-          echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts" >> "$GITHUB_ENV"
+          {
+            echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
+            echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
+            echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+          } >> "$GITHUB_ENV"
       - name: Verify cluster access
         run: |
           set -euo pipefail
@@ -281,9 +283,11 @@ jobs:
           install -m 600 /dev/null "${SSH_DIR}/id_rsa"
           printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
           install -m 600 /dev/null "${SSH_DIR}/known_hosts"
-          echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}" >> "$GITHUB_ENV"
-          echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa" >> "$GITHUB_ENV"
-          echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts" >> "$GITHUB_ENV"
+          {
+            echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
+            echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
+            echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+          } >> "$GITHUB_ENV"
       - name: Verify cluster access
         run: |
           set -euo pipefail

From c78dade400344038b47bc206c618b5657a9ba621 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Mon, 24 Aug 2026 17:08:11 -0700
Subject: [PATCH 13/14] fix(ci): re-query the node count until it is positive

The count of GPU nodes is resolved once and kept, because the upgrade churns
node labels and a per-iteration count could sample a transient value and match
early. The guard for that tested the cached value for emptiness, which only
holds when the query failed. A successful query returning zero, which is what
happens when no node carries the label yet, was cached like any other answer,
and the loop then compared against a frozen zero for the rest of its budget.
Cache only a positive count, so an API error and a zero both retry.

Give every kubectl call that a polling loop depends on an explicit request
timeout. kubectl defaults to no per-request timeout, and a wall clock deadline
is only tested between commands, so one hung request could run past it for as
long as the connection stayed open.

Drop the remaining explanatory comments.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 .github/workflows/e2e-tests.yaml              | 14 ----------
 tests/scripts/checks.sh                       | 28 ++++++++-----------
 .../migrate-clusterpolicy-to-nvidiadriver.sh  | 15 ++++++----
 tests/scripts/node-exec.sh                    |  3 --
 tests/scripts/node-operations.sh              |  4 ---
 tests/scripts/update-clusterpolicy.sh         |  2 --
 tests/scripts/update-nvidiadriver.sh          | 23 +++++++++------
 7 files changed, 35 insertions(+), 54 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index b96b909546..ab86bdddbe 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -57,10 +57,8 @@ on:
 
 env:
   HELM_VERSION: v3.19.0
-  # Kept in sync with spec.kubernetes.version in tests/holodeck.yaml.
   KUBECTL_VERSION: v1.35.4
   JQ_VERSION: "1.7.1"
-  # From https://github.com/jqlang/jq/releases/download/jq-1.7.1/sha256sum.txt
   JQ_SHA256: "5942c9b0934e510ee61eb3e30273f1b3fe2590df93933a93d7c58b81d19c8ff5"
 
 jobs:
@@ -170,8 +168,6 @@ jobs:
       - name: Load kernel modules on the node
         run: |
           set -euo pipefail
-          # node-exec.sh runs locally when NODE_SSH_HOST is empty; on the
-          # shared runner that would modprobe the wrong machine.
           test -n "${NODE_SSH_HOST}"
           ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
@@ -184,8 +180,6 @@ jobs:
           CONTAINER_RUNTIME: "containerd"
         run: |
           set -euo pipefail
-          # The restart test reaches node-exec.sh, which runs locally when
-          # NODE_SSH_HOST is empty; crictl would then hit the shared runner.
           test -n "${NODE_SSH_HOST}"
           mkdir -p "${LOG_DIR}"
           ./tests/cases/defaults.sh
@@ -198,8 +192,6 @@ jobs:
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
           helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        # A job stopped by timeout-minutes is cancelled, not failed, so
-        # failure() would skip the upload on the runs we most need logs from.
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
@@ -302,8 +294,6 @@ jobs:
       - name: Load kernel modules on the node
         run: |
           set -euo pipefail
-          # node-exec.sh runs locally when NODE_SSH_HOST is empty; on the
-          # shared runner that would modprobe the wrong machine.
           test -n "${NODE_SSH_HOST}"
           ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
@@ -316,8 +306,6 @@ jobs:
           CONTAINER_RUNTIME: "containerd"
         run: |
           set -euo pipefail
-          # The restart test reaches node-exec.sh, which runs locally when
-          # NODE_SSH_HOST is empty; crictl would then hit the shared runner.
           test -n "${NODE_SSH_HOST}"
           mkdir -p "${LOG_DIR}"
           ./tests/cases/nvidia-driver.sh
@@ -330,8 +318,6 @@ jobs:
           kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
           helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        # A job stopped by timeout-minutes is cancelled, not failed, so
-        # failure() would skip the upload on the runs we most need logs from.
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index beb60588e1..f9cc3e04dd 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -8,11 +8,11 @@ check_pod_ready() {
 		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" || true
 
 		echo "Checking $pod_label pod readiness"
-		is_pod_ready=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
+		is_pod_ready=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
 
 		if [ "${is_pod_ready}" = "True" ]; then
 			# Check if the pod is not in terminating state
-			is_pod_terminating=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
+			is_pod_terminating=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -o jsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
 			if [ "${is_pod_terminating}" != "" ]; then
 				echo "pod $pod_label is in terminating state..."
 			else
@@ -111,8 +111,6 @@ list_all_pods() {
 	kubectl get pods --all-namespaces -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name --no-headers || true
 }
 
-# Not tailed: this file is the uploaded artifact, so truncation loses the
-# failure output it exists to capture.
 collect_pod_logs() {
 	local log_dir=$1
 	local namespaced_pods=$2
@@ -136,7 +134,7 @@ check_gpu_pod_ready() {
 	mkdir -p ${log_dir}
 
 	while :; do
-		status=$(kubectl get pods gpu-operator-test -o jsonpath='{.status.phase}' || true)
+		status=$(kubectl get pods gpu-operator-test --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -o jsonpath='{.status.phase}' || true)
 		if [ "${status}" = "Succeeded" ]; then
 			echo "GPU pod terminated successfully"
 			rc=0
@@ -146,7 +144,6 @@ check_gpu_pod_ready() {
 
 		if (( SECONDS > deadline )); then
 			echo "timeout reached"
-			# Refresh so the artifact reflects the timeout, not the last cadence.
 			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			exit 1
 		fi
@@ -173,11 +170,11 @@ check_nvidia_driver_pods_ready() {
 		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" || true
 
 		echo "Checking nvidia driver pod readiness"
-		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
+		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
 
 		if [ "${is_pod_ready}" = "True" ]; then
 			# Check if the pod is not in terminating state
-			is_pod_terminating=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -ojsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
+			is_pod_terminating=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
 			if [ "${is_pod_terminating}" != "" ]; then
 				echo "nvidia driver pod is in terminating state..."
 			else
@@ -214,8 +211,6 @@ api_unreachable() {
 	echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') WARNING: cluster API unreachable while ${1}; treating as not ready and retrying"
 }
 
-# Non-zero on query failure, so a genuine zero stays distinguishable from an
-# unreachable API.
 kubectl_count() {
 	local output
 	output=$(kubectl "$@" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}") || return 1
@@ -264,9 +259,9 @@ print_driver_upgrade_debug() {
 
 wait_for_driver_upgrade_done() {
 	local deadline=$((SECONDS + 60 * 45))
-	# Due time, not modulo: a slow iteration can jump over an exact multiple.
 	local next_debug=0
 	local node_list=""
+	local node_count=""
 	local upgraded_count=0
 	local upgrade_state=""
 
@@ -276,11 +271,11 @@ wait_for_driver_upgrade_done() {
 	while :; do
 		upgraded_count=0
 
-		# Resolved once: the upgrade churns node labels, so a per-iteration count
-		# could sample a transient value and match early.
-		if [[ -z "${gpu_node_count}" ]]; then
-			if node_list=$(kubectl get node -l nvidia.com/gpu.present --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
-				gpu_node_count=$(echo "${node_list}" | grep -c . || true)
+		if [[ "${gpu_node_count:-0}" -le 0 ]]; then
+			if node_count=$(kubectl_count get node -l nvidia.com/gpu.present --no-headers); then
+				if (( node_count > 0 )); then
+					gpu_node_count="${node_count}"
+				fi
 			else
 				api_unreachable "counting the GPU nodes"
 			fi
@@ -300,7 +295,6 @@ wait_for_driver_upgrade_done() {
 			api_unreachable "listing the nodes"
 		fi
 
-		# Zero means the count never arrived; this environment always has a GPU node.
 		if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${upgraded_count}" -eq "${gpu_node_count}" ]]; then
 			echo "gpu driver upgrade completed successfully"
 			break;
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index 511c3991cf..f16d1703a0 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -99,18 +99,23 @@ wait_for_nvidiadriver_owner_labels() {
     local driver_name=$1
     local deadline=$((SECONDS + 300))
     local gpu_node_count=""
+    local node_count=""
     local owned_count
 
     echo "Waiting for the GPU node(s) to be owned by NVIDIADriver/${driver_name}"
 
     while :; do
-        if [[ -z "${gpu_node_count}" ]]; then
-            gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers) \
-                || api_unreachable "counting the GPU nodes"
+        if [[ "${gpu_node_count:-0}" -le 0 ]]; then
+            if node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+                if (( node_count > 0 )); then
+                    gpu_node_count="${node_count}"
+                fi
+            else
+                api_unreachable "counting the GPU nodes"
+            fi
         fi
 
         if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
-            # Zero means the count never arrived; there is always a GPU node.
             if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
                 break
             fi
@@ -135,7 +140,7 @@ wait_for_nvidiadriver_daemonset() {
 
     echo "Waiting for NVIDIADriver-owned driver DaemonSet"
     while :; do
-        daemonset_count=$(kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o json |
+        daemonset_count=$(kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
             jq --arg driver_name "${driver_name}" '[.items[] | select(.spec.template.spec.nodeSelector["nvidia.com/gpu-operator.driver.owner"] == $driver_name)] | length')
         if [[ "${daemonset_count}" -gt 0 ]]; then
             break
diff --git a/tests/scripts/node-exec.sh b/tests/scripts/node-exec.sh
index 185167180e..9025dc9d61 100755
--- a/tests/scripts/node-exec.sh
+++ b/tests/scripts/node-exec.sh
@@ -16,9 +16,6 @@
 
 set -euo pipefail
 
-# Unset NODE_SSH_HOST runs the operation locally, which is the developer path.
-# CI must set it or node operations would target the runner.
-
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 readonly SCRIPT_DIR
 readonly NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
diff --git a/tests/scripts/node-operations.sh b/tests/scripts/node-operations.sh
index 2381cd6b37..a37fa5ad49 100755
--- a/tests/scripts/node-operations.sh
+++ b/tests/scripts/node-operations.sh
@@ -16,10 +16,6 @@
 
 set -euo pipefail
 
-# Runs on the cluster node, streamed over SSH stdin via `bash -s --`. Must stay
-# self-contained: no sourcing siblings, no repo-relative paths — the repository
-# is not guaranteed to exist there.
-
 usage() {
 	cat <<'EOF'
 Usage: node-operations.sh <operation> [args...]
diff --git a/tests/scripts/update-clusterpolicy.sh b/tests/scripts/update-clusterpolicy.sh
index 247ab7b656..d055a89411 100755
--- a/tests/scripts/update-clusterpolicy.sh
+++ b/tests/scripts/update-clusterpolicy.sh
@@ -243,8 +243,6 @@ test_custom_labels_override() {
   for operand in $operands
   do
     echo "checking $operand labels"
-    # check_pod_ready just confirmed these pods, so an empty list is a broken
-    # assumption, not nothing left to check.
     if ! operand_pods=$(kubectl get pods -n "$TEST_NAMESPACE" -l app="$operand" --output=jsonpath={.items..metadata.name} --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
       echo "cannot list $operand pods to verify the overridden labels"
       exit 1
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index 788227d0f3..bb50cfcff2 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -16,12 +16,12 @@ DEFAULT_NVIDIA_DRIVER_NAME="${DEFAULT_NVIDIA_DRIVER_NAME:-e2e-default-driver}"
 DUPLICATE_DEFAULT_NVIDIA_DRIVER_NAME="${DUPLICATE_DEFAULT_NVIDIA_DRIVER_NAME:-e2e-duplicate-default-driver}"
 
 get_default_nvidiadriver_name() {
-    kubectl get nvidiadriver -o json |
+    kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
         jq -r '.items[] | select(.spec.default == true) | .metadata.name' | head -n 1
 }
 
 get_default_nvidiadriver_count() {
-    kubectl get nvidiadriver -o json |
+    kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
         jq '[.items[] | select(.spec.default == true)] | length'
 }
 
@@ -121,18 +121,23 @@ wait_for_nvidiadriver_owner() {
     local driver_name=$1
     local deadline=$((SECONDS + $((60 * 15))))
     local gpu_node_count=""
+    local node_count=""
     local owned_count
 
     echo "Waiting for the GPU node(s) to be owned by NVIDIADriver/${driver_name}"
 
     while :; do
-        if [[ -z "${gpu_node_count}" ]]; then
-            gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers) \
-                || api_unreachable "counting the GPU nodes"
+        if [[ "${gpu_node_count:-0}" -le 0 ]]; then
+            if node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+                if (( node_count > 0 )); then
+                    gpu_node_count="${node_count}"
+                fi
+            else
+                api_unreachable "counting the GPU nodes"
+            fi
         fi
 
         if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
-            # Zero means the count never arrived; there is always a GPU node.
             if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
                 echo "All GPU nodes are owned by NVIDIADriver/${driver_name}"
                 break
@@ -155,7 +160,7 @@ wait_for_nvidiadriver_owner() {
 
 get_nvidiadriver_daemonsets() {
     local driver_name=$1
-    kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o json |
+    kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
         jq --arg driver_name "${driver_name}" '.items | map(select(.spec.template.spec.nodeSelector["nvidia.com/gpu-operator.driver.owner"] == $driver_name))'
 }
 
@@ -304,7 +309,7 @@ wait_for_nvidiadriver_condition_message() {
 
     echo "Waiting for NVIDIADriver/${driver_name} status message to contain: ${message}"
     while :; do
-        if kubectl get nvidiadriver/"${driver_name}" -o json | jq -e --arg message "${message}" '
+        if kubectl get nvidiadriver/"${driver_name}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" | jq -e --arg message "${message}" '
             (.status.state // "") == "notReady" and
             ([.status.conditions[]?.message // ""] | any(contains($message)))
         ' >/dev/null; then
@@ -327,7 +332,7 @@ wait_for_nvidiadriver_ready() {
 
     echo "Waiting for NVIDIADriver/${driver_name} to report Ready"
     while :; do
-        if kubectl get nvidiadriver/"${driver_name}" -o json | jq -e '
+        if kubectl get nvidiadriver/"${driver_name}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" | jq -e '
             (.status.state // "") == "ready" and
             ([.status.conditions[]? | select(.type == "Ready" and .status == "True")] | length > 0) and
             ([.status.conditions[]? | select(.type == "Error" and .status == "True")] | length == 0)

From 0bfe2a4d74de69cb5a985547d07b48ddb5644067 Mon Sep 17 00:00:00 2001
From: Abrar Shivani <ashivani@nvidia.com>
Date: Mon, 24 Aug 2026 17:27:04 -0700
Subject: [PATCH 14/14] fix(ci): bound the diagnostic queries in the polling
 loops too

A diagnostic that hangs stalls its loop exactly as a decision-driving query
would, and kubectl has no per-request timeout by default, so the wall clock
deadline never gets a chance to be tested. Bound the progress prints, the log
and describe collection, and the state dumps on the timeout paths. The status
poll in test_restart_operator is bounded for the same reason, even though that
loop counts iterations rather than time; the container selection and kill are
untouched.

The existing 15s default suits a listing or a status field, not a fetch that
carries a whole object or a container log. The driver build log runs to
megabytes and now crosses the WAN, and bounding it at 15s would leave a
truncated artifact, which is the failure the log collection exists to prevent.
Add KUBECTL_LOG_TIMEOUT at 120s and use it for logs, describe, and full object
dumps, keeping the short timeout for the cheap queries.

Signed-off-by: Abrar Shivani <ashivani@nvidia.com>
---
 tests/scripts/.definitions.sh                 |  1 +
 tests/scripts/checks.sh                       | 28 +++++++++----------
 .../migrate-clusterpolicy-to-nvidiadriver.sh  | 14 +++++-----
 tests/scripts/update-clusterpolicy.sh         |  6 ++--
 tests/scripts/update-nvidiadriver.sh          | 18 ++++++------
 5 files changed, 34 insertions(+), 33 deletions(-)

diff --git a/tests/scripts/.definitions.sh b/tests/scripts/.definitions.sh
index 1cdd561ecc..3babf0ff24 100644
--- a/tests/scripts/.definitions.sh
+++ b/tests/scripts/.definitions.sh
@@ -15,6 +15,7 @@ TERRAFORM="terraform -chdir=${TERRAFORM_DIR}"
 # Set default values if not defined
 : ${HELM:="helm"}
 : "${KUBECTL_REQUEST_TIMEOUT:="15s"}"
+: "${KUBECTL_LOG_TIMEOUT:="120s"}"
 : ${LOG_DIR:="/tmp/logs"}
 : ${PROJECT:="$(basename "${PROJECT_DIR}")"}
 : ${TEST_NAMESPACE:="test-operator"}
diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index f9cc3e04dd..9bfecd9c20 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -5,7 +5,7 @@ check_pod_ready() {
 	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" || true
+		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Checking $pod_label pod readiness"
 		is_pod_ready=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
@@ -27,7 +27,7 @@ check_pod_ready() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n "${TEST_NAMESPACE}" || true
+		kubectl get pods -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -40,7 +40,7 @@ check_pod_deleted() {
 	local pod_list
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" || true
+		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Checking if $pod_label pod has been deleted"
 		if pod_list=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
@@ -60,7 +60,7 @@ check_pod_deleted() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n "${TEST_NAMESPACE}" || true
+		kubectl get pods -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -72,7 +72,7 @@ check_no_restarts() {
 	restartCount=$(kubectl get pod -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "$pod_label restarted multiple times: $restartCount"
-		kubectl logs -p -lapp="${pod_label}" --all-containers -n "${TEST_NAMESPACE}" || true
+		kubectl logs -p -lapp="${pod_label}" --all-containers -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for pod $pod_label"
@@ -93,7 +93,7 @@ test_restart_operator() {
 		# Sleep a reasonable amount of time for k8s to update the container status to crashing
 		sleep 10
 
-		state=$(kubectl get pods -n "${ns}" -l "app.kubernetes.io/component=gpu-operator" \
+		state=$(kubectl get pods -n "${ns}" -l "app.kubernetes.io/component=gpu-operator" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
 			-o jsonpath='{.items[0].status.phase}')
 
 		echo "Checking state of the GPU Operator, it is: '$state'"
@@ -103,12 +103,12 @@ test_restart_operator() {
 	done
 
 	echo "Timeout reached, the GPU Operator is still not ready. See below for logs:"
-	kubectl logs -n gpu-operator "$(kubectl get pods -n "${ns}" -o json | jq -r '.items[0].metadata.name')"
+	kubectl logs -n gpu-operator --request-timeout="${KUBECTL_LOG_TIMEOUT}" "$(kubectl get pods -n "${ns}" -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" | jq -r '.items[0].metadata.name')"
 	exit 1
 }
 
 list_all_pods() {
-	kubectl get pods --all-namespaces -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name --no-headers || true
+	kubectl get pods --all-namespaces -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 }
 
 collect_pod_logs() {
@@ -120,8 +120,8 @@ collect_pod_logs() {
 		[[ -n "${pod_name}" ]] || continue
 		echo "Generating logs for pod: ${pod_name} ns: ${namespace}"
 		echo "------------------------------------------------" >> "${log_dir}/${pod_name}.describe"
-		kubectl -n "${namespace}" describe pods "${pod_name}" >> "${log_dir}/${pod_name}.describe" || true
-		kubectl -n "${namespace}" logs "${pod_name}" --all-containers=true > "${log_dir}/${pod_name}.logs" || true
+		kubectl -n "${namespace}" describe pods "${pod_name}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" >> "${log_dir}/${pod_name}.describe" || true
+		kubectl -n "${namespace}" logs "${pod_name}" --all-containers=true --request-timeout="${KUBECTL_LOG_TIMEOUT}" > "${log_dir}/${pod_name}.logs" || true
 	done <<< "${namespaced_pods}"
 }
 
@@ -150,7 +150,7 @@ check_gpu_pod_ready() {
 
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
-		kubectl get pods --all-namespaces | tee -a "${log_dir}/cluster.logs" || true
+		kubectl get pods --all-namespaces --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" | tee -a "${log_dir}/cluster.logs" || true
 
 		if (( SECONDS >= next_collection )); then
 			collect_pod_logs "${log_dir}" "$(list_all_pods)"
@@ -167,7 +167,7 @@ check_nvidia_driver_pods_ready() {
 	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking nvidia driver pod"
-		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" || true
+		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Checking nvidia driver pod readiness"
 		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
@@ -189,7 +189,7 @@ check_nvidia_driver_pods_ready() {
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n "${TEST_NAMESPACE}" || true
+		kubectl get pods -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Sleeping 5 seconds"
 		sleep 5
@@ -200,7 +200,7 @@ check_no_driver_pod_restarts() {
 	restartCount=$(kubectl get pod -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "nvidia driver pod restarted multiple times: $restartCount"
-		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n "${TEST_NAMESPACE}" || true
+		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for the nvidia driver pod"
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index f16d1703a0..4f5e3fc2c9 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -37,7 +37,7 @@ wait_for_legacy_driver_daemonset_deleted() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver DaemonSet deletion"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide || true
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
             exit 1
         fi
 
@@ -63,7 +63,7 @@ wait_for_orphaned_legacy_driver_pod() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver pod to become orphaned"
-            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
+            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
@@ -87,7 +87,7 @@ wait_for_default_nvidiadriver() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for default NVIDIADriver"
-            kubectl get nvidiadriver || true
+            kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
             exit 1
         fi
 
@@ -125,7 +125,7 @@ wait_for_nvidiadriver_owner_labels() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver owner labels"
-            kubectl get nodes -l nvidia.com/gpu.present=true -o json |
+            kubectl get nodes -l nvidia.com/gpu.present=true -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" |
                 jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
@@ -148,7 +148,7 @@ wait_for_nvidiadriver_daemonset() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver-owned driver DaemonSet"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml || true
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
@@ -173,7 +173,7 @@ wait_for_legacy_driver_pod_deleted() {
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for orphaned legacy driver pod deletion"
             print_driver_upgrade_debug
-            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
+            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
@@ -185,7 +185,7 @@ wait_for_legacy_driver_pod_deleted() {
 legacy_driver_pod=$(kubectl get pod -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}')
 if [[ -z "${legacy_driver_pod}" ]]; then
     echo "legacy ClusterPolicy driver pod not found"
-    kubectl get pods -n "${TEST_NAMESPACE}" -o wide || true
+    kubectl get pods -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
     exit 1
 fi
 
diff --git a/tests/scripts/update-clusterpolicy.sh b/tests/scripts/update-clusterpolicy.sh
index d055a89411..232b33dd38 100755
--- a/tests/scripts/update-clusterpolicy.sh
+++ b/tests/scripts/update-clusterpolicy.sh
@@ -150,7 +150,7 @@ test_gpu_sharing() {
     kubectl wait --for=condition=available --timeout=300s deployment/nvidia-plugin-test -n $TEST_NAMESPACE
     if [ $? -ne 0 ]; then
         echo "cannot run parallel pods with GPU sharing enabled"
-        kubectl get pods -l app=nvidia-plugin-test -n "${TEST_NAMESPACE}" || true
+        kubectl get pods -l app=nvidia-plugin-test -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
         exit 1
     fi
 
@@ -253,12 +253,12 @@ test_custom_labels_override() {
     fi
     for pod in $operand_pods
     do
-      cp_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath={.metadata.labels.cloudprovider})
+      cp_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath='{.metadata.labels.cloudprovider}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}")
       if [ "$cp_label_value" != "aws" ]; then
           echo "Custom Label cloudprovider is incorrect when clusterpolicy labels are overridden - $pod"
           exit 1
       fi
-      platform_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath={.metadata.labels.platform})
+      platform_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath='{.metadata.labels.platform}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}")
       if [ "$platform_label_value" != "kubernetes" ]; then
           echo "Custom Label platform is incorrect when clusterpolicy labels are overridden - $pod"
           exit 1
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index bb50cfcff2..a0f812ee7b 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -69,7 +69,7 @@ wait_for_default_nvidiadriver() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${expected_name} to be the only default"
-            kubectl get nvidiadriver || true
+            kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
             exit 1
         fi
 
@@ -82,7 +82,7 @@ test_arbitrary_name_default_nvidiadriver() {
     current_default=$(get_default_nvidiadriver_name)
     if [[ -z "${current_default}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver || true
+        kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
         exit 1
     fi
 
@@ -109,7 +109,7 @@ create_nvidiadriver() {
     default_name=$(get_default_nvidiadriver_name)
     if [[ -z "${default_name}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver || true
+        kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
         exit 1
     fi
 
@@ -148,7 +148,7 @@ wait_for_nvidiadriver_owner() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} ownership"
-            kubectl get nodes -l nvidia.com/gpu.present=true -o json |
+            kubectl get nodes -l nvidia.com/gpu.present=true -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" |
                 jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
@@ -178,7 +178,7 @@ wait_for_nvidiadriver_daemonsets() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for daemonsets owned by NVIDIADriver/${driver_name}"
-            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml || true
+            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
@@ -272,7 +272,7 @@ test_custom_labels_override() {
   fi
   if [[ "${labeled_pod_count}" -ne "${gpu_node_count}" ]]; then
     echo "Custom labels are missing from one or more NVIDIADriver/${NVIDIA_DRIVER_NAME} pods"
-    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels || true
+    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
     exit 1
   fi
 }
@@ -296,7 +296,7 @@ assert_nvidiadriver_owner_count() {
     fi
     if [[ "${owned_count}" -ne "${gpu_node_count}" ]]; then
         echo "Expected ${gpu_node_count} GPU node(s) to remain owned by NVIDIADriver/${driver_name}, found ${owned_count}"
-        kubectl get nodes -l nvidia.com/gpu.present=true -o json |
+        kubectl get nodes -l nvidia.com/gpu.present=true -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" |
             jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
         exit 1
     fi
@@ -318,7 +318,7 @@ wait_for_nvidiadriver_condition_message() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} status message"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml || true
+            kubectl get nvidiadriver/"${driver_name}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
@@ -342,7 +342,7 @@ wait_for_nvidiadriver_ready() {
 
         if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} to report Ready"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml || true
+            kubectl get nvidiadriver/"${driver_name}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi