diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 298b16414..ab86bdddb 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -55,6 +55,12 @@ on:
         type: boolean
         default: false
 
+env:
+  HELM_VERSION: v3.19.0
+  KUBECTL_VERSION: v1.35.4
+  JQ_VERSION: "1.7.1"
+  JQ_SHA256: "5942c9b0934e510ee61eb3e30273f1b3fe2590df93933a93d7c58b81d19c8ff5"
+
 jobs:
   variables:
     uses: ./.github/workflows/variables.yaml
@@ -79,6 +85,9 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         name: Check out code
@@ -92,11 +101,29 @@ jobs:
           path: ${{ github.workspace }}
       - name: Set up Helm
         uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+        with:
+          version: ${{ env.HELM_VERSION }}
       - name: Verify the published Helm chart is available
         env:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
         run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}"
+      - name: Install kubectl
+        uses: azure/setup-kubectl@c0c8b32d33a5244f1e5947304550403b63930415 # v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7
         with:
@@ -109,24 +136,40 @@ jobs:
         uses: mikefarah/yq@c14f446382944492701b16c1ddb48bb9dbe683e3 # v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          {
+            echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
+            echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
+            echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+          } >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
@@ -134,20 +177,32 @@ jobs:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/defaults.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          mkdir -p "${LOG_DIR}"
+          ./tests/cases/defaults.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
+          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: containerd-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove credentials from the runner
+        if: always()
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"
 
   e2e-tests-nvidiadriver:
     needs: [variables, publish-helm-oci-chart]
@@ -156,6 +211,9 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    env:
+      KUBECONFIG: ${{ github.workspace }}/kubeconfig
+      LOG_DIR: ${{ github.workspace }}/logs
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         name: Check out code
@@ -169,11 +227,29 @@ jobs:
           path: ${{ github.workspace }}
       - name: Set up Helm
         uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+        with:
+          version: ${{ env.HELM_VERSION }}
       - name: Verify the published Helm chart is available
         env:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
         run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}"
+      - name: Install kubectl
+        uses: azure/setup-kubectl@c0c8b32d33a5244f1e5947304550403b63930415 # v4
+        with:
+          version: ${{ env.KUBECTL_VERSION }}
+      - name: Install jq and verify runner tooling
+        run: |
+          set -euo pipefail
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL -o "${RUNNER_TEMP}/bin/jq" \
+            "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64"
+          echo "${JQ_SHA256}  ${RUNNER_TEMP}/bin/jq" | sha256sum -c -
+          chmod +x "${RUNNER_TEMP}/bin/jq"
+          echo "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH"
+          helm version
+          kubectl version --client
+          "${RUNNER_TEMP}/bin/jq" --version
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7
         with:
@@ -186,24 +262,40 @@ jobs:
         uses: mikefarah/yq@c14f446382944492701b16c1ddb48bb9dbe683e3 # v4
         with:
           cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
-      - name: Set test environment
+      - name: Configure node SSH access
         env:
+          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
           PUBLIC_DNS_NAME: ${{ steps.get_public_dns_name.outputs.result }}
         run: |
-          echo "instance_hostname=ubuntu@${PUBLIC_DNS_NAME}" >> $GITHUB_ENV
-          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
-      - name: Write SSH key
-        env:
-          AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
+          set -euo pipefail
+          test -n "${PUBLIC_DNS_NAME}"
+          SSH_DIR="${RUNNER_TEMP}/holodeck-ssh"
+          mkdir -p "${SSH_DIR}"
+          chmod 700 "${SSH_DIR}"
+          install -m 600 /dev/null "${SSH_DIR}/id_rsa"
+          printf '%s\n' "${AWS_SSH_KEY}" > "${SSH_DIR}/id_rsa"
+          install -m 600 /dev/null "${SSH_DIR}/known_hosts"
+          {
+            echo "NODE_SSH_HOST=ubuntu@${PUBLIC_DNS_NAME}"
+            echo "NODE_SSH_KEY=${SSH_DIR}/id_rsa"
+            echo "NODE_SSH_KNOWN_HOSTS=${SSH_DIR}/known_hosts"
+          } >> "$GITHUB_ENV"
+      - name: Verify cluster access
         run: |
-          echo "${AWS_SSH_KEY}" > ${private_key} && chmod 400 ${private_key}
-      - name: Copy values override file to remote
+          set -euo pipefail
+          test -r "${KUBECONFIG}"
+          kubectl cluster-info
+          kubectl get nodes -o wide
+      - name: Select values override file
         if: ${{ inputs.use_values_override }}
         run: |
-          scp -i ${private_key} -o StrictHostKeyChecking=no \
-            ${{ github.workspace }}/values-overrides.yaml \
-            ${instance_hostname}:/tmp/values-overrides.yaml
-          echo "VALUES_FILE=/tmp/values-overrides.yaml" >> $GITHUB_ENV
+          set -euo pipefail
+          echo "VALUES_FILE=${GITHUB_WORKSPACE}/values-overrides.yaml" >> "$GITHUB_ENV"
+      - name: Load kernel modules on the node
+        run: |
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          ./tests/scripts/node-exec.sh load-modules
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
@@ -211,17 +303,29 @@ jobs:
           HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }}
           HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }}
           GPU_PRODUCT_NAME: "Tesla-T4"
-          SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
-          TEST_CASE: "./tests/cases/nvidia-driver.sh"
         run: |
-          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
-          ./tests/scripts/pull.sh /tmp/logs logs
-          exit $rc
+          set -euo pipefail
+          test -n "${NODE_SSH_HOST}"
+          mkdir -p "${LOG_DIR}"
+          ./tests/cases/nvidia-driver.sh
+      - name: Collect cluster diagnostics
+        if: always()
+        run: |
+          mkdir -p "${LOG_DIR}"
+          kubectl get nodes -o wide > "${LOG_DIR}/nodes.txt" 2>&1 || true
+          kubectl get pods -A -o wide > "${LOG_DIR}/pods.txt" 2>&1 || true
+          kubectl get events -A --sort-by=.lastTimestamp > "${LOG_DIR}/events.txt" 2>&1 || true
+          helm list -A > "${LOG_DIR}/helm-releases.txt" 2>&1 || true
       - name: Archive test logs
-        if: ${{ failure() }}
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: nvidiadriver-e2e-test-logs
           path: ./logs/
           retention-days: 15
+      - name: Remove credentials from the runner
+        if: always()
+        run: |
+          rm -rf "${RUNNER_TEMP}/holodeck-ssh"
+          rm -f "${KUBECONFIG}"
diff --git a/tests/README.md b/tests/README.md
index 9b14de246..3a4096ce5 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,5 +1,22 @@
 # GPU operator test utilities
 
+## Testing in CI
+CI no longer uses `local.sh` or `ci-run-e2e.sh`, and no longer syncs the project
+folder to the test instance. Those remain the developer path described below.
+
+Instead, the e2e workflow provisions a Holodeck environment with
+`kubernetes.remoteAccess` enabled, which gives the GitHub Actions runner a
+kubeconfig for the cluster. The case scripts (`cases/defaults.sh`,
+`cases/nvidia-driver.sh`) then run on the runner itself, and everything they do
+-- helm, kubectl, log collection -- goes over that kubeconfig.
+
+Only two operations still need a shell on the instance, and both go through
+`scripts/node-exec.sh`, which dispatches `scripts/node-operations.sh` over SSH:
+loading the `i2c_core` and `ipmi_msghandler` kernel modules, and killing the
+gpu-operator container for the operator restart test. With `NODE_SSH_HOST`
+unset, `node-exec.sh` runs the operation locally, so the developer path below is
+unaffected.
+
 ## Testing locally
 The `local.sh` script allows for triggering basic end-to-end testing of the GPU
 operator from a local machine.
diff --git a/tests/holodeck.yaml b/tests/holodeck.yaml
index 3050eaf68..880aa417a 100644
--- a/tests/holodeck.yaml
+++ b/tests/holodeck.yaml
@@ -24,3 +24,4 @@ spec:
     version: v1.35.4
     crictlVersion: v1.35.0
     calicoVersion: v3.31.5
+    remoteAccess: true
diff --git a/tests/scripts/.definitions.sh b/tests/scripts/.definitions.sh
index 737a019c6..3babf0ff2 100644
--- a/tests/scripts/.definitions.sh
+++ b/tests/scripts/.definitions.sh
@@ -14,6 +14,8 @@ TERRAFORM="terraform -chdir=${TERRAFORM_DIR}"
 
 # Set default values if not defined
 : ${HELM:="helm"}
+: "${KUBECTL_REQUEST_TIMEOUT:="15s"}"
+: "${KUBECTL_LOG_TIMEOUT:="120s"}"
 : ${LOG_DIR:="/tmp/logs"}
 : ${PROJECT:="$(basename "${PROJECT_DIR}")"}
 : ${TEST_NAMESPACE:="test-operator"}
diff --git a/tests/scripts/checks.sh b/tests/scripts/checks.sh
index 5056d48b2..9bfecd9c2 100755
--- a/tests/scripts/checks.sh
+++ b/tests/scripts/checks.sh
@@ -2,17 +2,17 @@
 
 check_pod_ready() {
 	local pod_label=$1
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
+		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Checking $pod_label pod readiness"
-		is_pod_ready=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
+		is_pod_ready=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
 
 		if [ "${is_pod_ready}" = "True" ]; then
 			# Check if the pod is not in terminating state
-			is_pod_terminating=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
+			is_pod_terminating=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -o jsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
 			if [ "${is_pod_terminating}" != "" ]; then
 				echo "pod $pod_label is in terminating state..."
 			else
@@ -21,48 +21,48 @@ check_pod_ready() {
 			fi
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
 
 check_pod_deleted() {
 	local pod_label=$1
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
+	local pod_list
 	while :; do
 		echo "Checking $pod_label pod"
-		kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE}
+		kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Checking if $pod_label pod has been deleted"
-		# note: $(kubectl get pods <options> -o jsonpath='.items' | jq length) does not work for older kubectl clients
-		num_pods=$(kubectl get pods -lapp=$pod_label -n ${TEST_NAMESPACE} -o json | jq '.items' | jq length)
-
-		if [ "${num_pods}" = 0 ]; then
-			echo "Pod $pod_label has been deleted"
-			break;
+		if pod_list=$(kubectl get pods -lapp="${pod_label}" -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+			if [ -z "${pod_list}" ]; then
+				echo "Pod $pod_label has been deleted"
+				break;
+			else
+				echo "Pod $pod_label has not been deleted"
+			fi
 		else
-			echo "Pod $pod_label has not been deleted"
+			api_unreachable "checking whether pod $pod_label has been deleted"
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
@@ -72,7 +72,7 @@ check_no_restarts() {
 	restartCount=$(kubectl get pod -lapp=$pod_label -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "$pod_label restarted multiple times: $restartCount"
-		kubectl logs -p -lapp=$pod_label --all-containers -n ${TEST_NAMESPACE}
+		kubectl logs -p -lapp="${pod_label}" --all-containers -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for pod $pod_label"
@@ -85,20 +85,15 @@ test_restart_operator() {
 	local ns=${1}
 	local runtime=${2}
 
-	if [[ x"${runtime}" == x"containerd" ]]; then
-		# The operator is the only container that has the string '"gpu-operator"'
-		# TODO: This requires permissions on containerd.sock
-		sudo crictl rm --force "$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')"
-	else
-		# The operator is the only container that has the string '"gpu-operator"'
-	    docker kill "$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')"
-	fi
+	local checks_script_dir
+	checks_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+	"${checks_script_dir}"/node-exec.sh restart-operator-container "${runtime}"
 
 	for i in $(seq 1 10); do
 		# Sleep a reasonable amount of time for k8s to update the container status to crashing
 		sleep 10
 
-		state=$(kubectl get pods -n "${ns}" -l "app.kubernetes.io/component=gpu-operator" \
+		state=$(kubectl get pods -n "${ns}" -l "app.kubernetes.io/component=gpu-operator" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
 			-o jsonpath='{.items[0].status.phase}')
 
 		echo "Checking state of the GPU Operator, it is: '$state'"
@@ -108,65 +103,78 @@ test_restart_operator() {
 	done
 
 	echo "Timeout reached, the GPU Operator is still not ready. See below for logs:"
-	kubectl logs -n gpu-operator "$(kubectl get pods -n "${ns}" -o json | jq -r '.items[0].metadata.name')"
+	kubectl logs -n gpu-operator --request-timeout="${KUBECTL_LOG_TIMEOUT}" "$(kubectl get pods -n "${ns}" -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" | jq -r '.items[0].metadata.name')"
 	exit 1
 }
 
+list_all_pods() {
+	kubectl get pods --all-namespaces -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name --no-headers --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
+}
+
+collect_pod_logs() {
+	local log_dir=$1
+	local namespaced_pods=$2
+	local namespace pod_name
+
+	while read -r namespace pod_name; do
+		[[ -n "${pod_name}" ]] || continue
+		echo "Generating logs for pod: ${pod_name} ns: ${namespace}"
+		echo "------------------------------------------------" >> "${log_dir}/${pod_name}.describe"
+		kubectl -n "${namespace}" describe pods "${pod_name}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" >> "${log_dir}/${pod_name}.describe" || true
+		kubectl -n "${namespace}" logs "${pod_name}" --all-containers=true --request-timeout="${KUBECTL_LOG_TIMEOUT}" > "${log_dir}/${pod_name}.logs" || true
+	done <<< "${namespaced_pods}"
+}
+
 check_gpu_pod_ready() {
 	local log_dir=$1
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
+	local next_collection=0
 
 	# Ensure the log directory exists
 	mkdir -p ${log_dir}
 
 	while :; do
-		pods="$(kubectl get --all-namespaces pods -o json | jq '.items[] | {name: .metadata.name, ns: .metadata.namespace}' | jq -s -c .)"
-		status=$(kubectl get pods gpu-operator-test -o json | jq -r .status.phase)
+		status=$(kubectl get pods gpu-operator-test --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -o jsonpath='{.status.phase}' || true)
 		if [ "${status}" = "Succeeded" ]; then
 			echo "GPU pod terminated successfully"
 			rc=0
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			break;
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
 			exit 1
 		fi
 
-		# Echo useful information on stdout
-		kubectl get pods --all-namespaces
-
-		for pod in $(echo "$pods" | jq -r .[].name); do
-			ns=$(echo "$pods" | jq -r ".[] | select(.name == \"$pod\") | .ns")
-			echo "Generating logs for pod: ${pod} ns: ${ns}"
-			echo "------------------------------------------------" >> "${log_dir}/${pod}.describe"
-			kubectl -n "${ns}" describe pods "${pod}" >> "${log_dir}/${pod}.describe"
-			kubectl -n "${ns}" logs "${pod}" --all-containers=true > "${log_dir}/${pod}.logs" || true
-		done
-
 		echo "Generating cluster logs"
 		echo "------------------------------------------------" >> "${log_dir}/cluster.logs"
-		kubectl get --all-namespaces pods >> "${log_dir}/cluster.logs"
+		kubectl get pods --all-namespaces --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" | tee -a "${log_dir}/cluster.logs" || true
+
+		if (( SECONDS >= next_collection )); then
+			collect_pod_logs "${log_dir}" "$(list_all_pods)"
+			next_collection=$((SECONDS + 30))
+		fi
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5;
 	done
 }
 
 # TODO: deduplicate the logic found in this file by moving the duplicate to a common method and parameterizing the labels to select on
 check_nvidia_driver_pods_ready() {
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
 	while :; do
 		echo "Checking nvidia driver pod"
-		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE}
+		kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Checking nvidia driver pod readiness"
-		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
+		is_pod_ready=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null || echo "terminated")
 
 		if [ "${is_pod_ready}" = "True" ]; then
 			# Check if the pod is not in terminating state
-			is_pod_terminating=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -ojsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
+			is_pod_terminating=$(kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" -ojsonpath='{.items[0].metadata.deletionGracePeriodSeconds}' 2>/dev/null || echo "terminated")
 			if [ "${is_pod_terminating}" != "" ]; then
 				echo "nvidia driver pod is in terminating state..."
 			else
@@ -175,16 +183,15 @@ check_nvidia_driver_pods_ready() {
 			fi
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			exit 1;
 		fi
 
 		# Echo useful information on stdout
-		kubectl get pods -n ${TEST_NAMESPACE}
+		kubectl get pods -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
@@ -193,34 +200,44 @@ check_no_driver_pod_restarts() {
 	restartCount=$(kubectl get pod -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o jsonpath='{.items[*].status.containerStatuses[0].restartCount}')
 	if [ $restartCount -gt 1 ]; then
 		echo "nvidia driver pod restarted multiple times: $restartCount"
-		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n ${TEST_NAMESPACE}
+		kubectl logs -p -l "app.kubernetes.io/component=nvidia-driver" --all-containers -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
 		exit 1
 	fi
 	echo "Repeated restarts not observed for the nvidia driver pod"
 	return 0
 }
 
+api_unreachable() {
+	echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') WARNING: cluster API unreachable while ${1}; treating as not ready and retrying"
+}
+
+kubectl_count() {
+	local output
+	output=$(kubectl "$@" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}") || return 1
+	echo "${output}" | grep -c . || true
+}
+
 print_driver_upgrade_debug() {
 	echo "current state of driver upgrade"
-	kubectl get node -l nvidia.com/gpu.present \
-		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
+	kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
+		-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 
 	echo ""
 	echo "driver pods"
-	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get pods -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "gpu operator operands"
-	kubectl get pods -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get pods -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "driver daemonsets"
-	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n ${TEST_NAMESPACE} -o wide || true
+	kubectl get daemonsets -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
 
 	echo ""
 	echo "NVIDIADriver status"
 	local nvidiadriver_status
-	if nvidiadriver_status=$(kubectl get nvidiadriver -o json 2>/dev/null); then
+	if nvidiadriver_status=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" 2>/dev/null); then
 		echo "${nvidiadriver_status}" | jq -r '
 			(["NAME", "DEFAULT", "STATE", "REASON", "MESSAGE"] | @tsv),
 			(
@@ -241,39 +258,65 @@ print_driver_upgrade_debug() {
 }
 
 wait_for_driver_upgrade_done() {
-	gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present --no-headers | wc -l)
-	local current_time=0
+	local deadline=$((SECONDS + 60 * 45))
+	local next_debug=0
+	local node_list=""
+	local node_count=""
+	local upgraded_count=0
+	local upgrade_state=""
+
+	gpu_node_count=""
+
 	echo "waiting for the gpu driver upgrade to complete"
 	while :; do
-		local upgraded_count=0
-		for node in $(kubectl get nodes -o NAME); do
-			upgrade_state=$(kubectl get $node -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}')
-			if [ "${upgrade_state}" = "upgrade-done" ]; then
-				upgraded_count=$((${upgraded_count} + 1))
+		upgraded_count=0
+
+		if [[ "${gpu_node_count:-0}" -le 0 ]]; then
+			if node_count=$(kubectl_count get node -l nvidia.com/gpu.present --no-headers); then
+				if (( node_count > 0 )); then
+					gpu_node_count="${node_count}"
+				fi
+			else
+				api_unreachable "counting the GPU nodes"
 			fi
-		done
-		if [[ $upgraded_count -eq $gpu_node_count ]]; then
+		fi
+
+		if node_list=$(kubectl get nodes -o NAME --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+			for node in ${node_list}; do
+				if upgrade_state=$(kubectl get "$node" -ojsonpath='{.metadata.labels.nvidia\.com/gpu-driver-upgrade-state}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+					if [ "${upgrade_state}" = "upgrade-done" ]; then
+						upgraded_count=$((upgraded_count + 1))
+					fi
+				else
+					api_unreachable "reading the upgrade state of ${node}"
+				fi
+			done
+		else
+			api_unreachable "listing the nodes"
+		fi
+
+		if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${upgraded_count}" -eq "${gpu_node_count}" ]]; then
 			echo "gpu driver upgrade completed successfully"
 			break;
 		else
-			echo "gpu driver still in progress. $upgraded_count/$gpu_node_count node(s) upgraded"
+			echo "gpu driver still in progress. $upgraded_count/${gpu_node_count:-unknown} node(s) upgraded"
 		fi
 
-		if [[ "${current_time}" -gt $((60 * 45)) ]]; then
+		if (( SECONDS > deadline )); then
 			echo "timeout reached"
 			print_driver_upgrade_debug
 			exit 1;
 		fi
 
-		if [[ $((current_time % 30)) -eq 0 ]]; then
+		if (( SECONDS >= next_debug )); then
 			print_driver_upgrade_debug
+			next_debug=$((SECONDS + 30))
 		else
-			kubectl get node -l nvidia.com/gpu.present \
-				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers
+			kubectl get node -l nvidia.com/gpu.present --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" \
+				-o custom-columns=NODE:.metadata.name,OWNER:.metadata.labels.nvidia\\.com/gpu-operator\\.driver\\.owner,UPGRADE_STATE:.metadata.labels.nvidia\\.com/gpu-driver-upgrade-state --no-headers || true
 		fi
 		
 		echo "Sleeping 5 seconds"
-		current_time=$((${current_time} + 5))
 		sleep 5
 	done
 }
diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
index 4f56d9f19..4f5e3fc2c 100755
--- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
+++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh
@@ -23,153 +23,176 @@ get_helm_release_name() {
 }
 
 wait_for_legacy_driver_daemonset_deleted() {
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for ClusterPolicy-owned driver DaemonSet to be deleted"
     while :; do
-        daemonset_count=$(kubectl get daemonset -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" --no-headers 2>/dev/null | wc -l)
-        if [[ "${daemonset_count}" -eq 0 ]]; then
-            break
+        if daemonsets=$(kubectl get daemonset -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            if [[ -z "${daemonsets}" ]]; then
+                break
+            fi
+        else
+            api_unreachable "checking for the ClusterPolicy-owned driver DaemonSet"
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver DaemonSet deletion"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_orphaned_legacy_driver_pod() {
     local pod_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for legacy driver pod/${pod_name} to become orphaned"
     while :; do
-        owner_count=$(kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o json | jq '.metadata.ownerReferences | length')
-        if [[ "${owner_count}" -eq 0 ]]; then
-            echo "legacy driver pod/${pod_name} is orphaned"
-            break
+        if pod_json=$(kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            owner_count=$(echo "${pod_json}" | jq '.metadata.ownerReferences | length')
+            if [[ "${owner_count}" -eq 0 ]]; then
+                echo "legacy driver pod/${pod_name} is orphaned"
+                break
+            fi
+        else
+            api_unreachable "checking whether legacy driver pod/${pod_name} is orphaned"
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for legacy driver pod to become orphaned"
-            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml
+            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_default_nvidiadriver() {
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for default NVIDIADriver to be rendered"
     while :; do
-        default_count=$(kubectl get nvidiadriver -o json 2>/dev/null | jq '[.items[] | select(.spec.default == true)] | length')
-        if [[ "${default_count}" -eq 1 ]]; then
-            break
+        if nvidiadrivers=$(kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            default_count=$(echo "${nvidiadrivers}" | jq '[.items[] | select(.spec.default == true)] | length')
+            if [[ "${default_count}" -eq 1 ]]; then
+                break
+            fi
+        else
+            api_unreachable "counting the default NVIDIADrivers"
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for default NVIDIADriver"
-            kubectl get nvidiadriver || true
+            kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_owner_labels() {
     local driver_name=$1
-    local elapsed_time=0
-    local gpu_node_count
+    local deadline=$((SECONDS + 300))
+    local gpu_node_count=""
+    local node_count=""
+    local owned_count
 
-    gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
-    echo "Waiting for ${gpu_node_count} GPU node(s) to be owned by NVIDIADriver/${driver_name}"
+    echo "Waiting for the GPU node(s) to be owned by NVIDIADriver/${driver_name}"
 
     while :; do
-        owned_count=$(kubectl get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers | wc -l)
-        if [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
-            break
+        if [[ "${gpu_node_count:-0}" -le 0 ]]; then
+            if node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+                if (( node_count > 0 )); then
+                    gpu_node_count="${node_count}"
+                fi
+            else
+                api_unreachable "counting the GPU nodes"
+            fi
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
+            if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
+                break
+            fi
+        else
+            api_unreachable "counting the nodes owned by NVIDIADriver/${driver_name}"
+        fi
+
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver owner labels"
-            kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+            kubectl get nodes -l nvidia.com/gpu.present=true -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" |
+                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_daemonset() {
     local driver_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for NVIDIADriver-owned driver DaemonSet"
     while :; do
-        daemonset_count=$(kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o json |
+        daemonset_count=$(kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "${TEST_NAMESPACE}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
             jq --arg driver_name "${driver_name}" '[.items[] | select(.spec.template.spec.nodeSelector["nvidia.com/gpu-operator.driver.owner"] == $driver_name)] | length')
         if [[ "${daemonset_count}" -gt 0 ]]; then
             break
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver-owned driver DaemonSet"
-            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml
+            kubectl get daemonset -n "${TEST_NAMESPACE}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 wait_for_legacy_driver_pod_deleted() {
     local pod_name=$1
-    local elapsed_time=0
+    local deadline=$((SECONDS + 300))
 
     echo "Waiting for orphaned legacy driver pod/${pod_name} to be deleted by the upgrade flow"
     while :; do
-        if ! kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" >/dev/null 2>&1; then
-            break
+        if legacy_pod=$(kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o name --ignore-not-found --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+            if [[ -z "${legacy_pod}" ]]; then
+                break
+            fi
+        else
+            api_unreachable "checking whether legacy driver pod/${pod_name} is deleted"
         fi
 
-        if [[ "${elapsed_time}" -gt 300 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for orphaned legacy driver pod deletion"
             print_driver_upgrade_debug
-            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml || true
+            kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         print_driver_upgrade_debug
         sleep 5
-        elapsed_time=$((${elapsed_time} + 5))
     done
 }
 
 legacy_driver_pod=$(kubectl get pod -l app=nvidia-driver-daemonset -n "${TEST_NAMESPACE}" -o jsonpath='{.items[0].metadata.name}')
 if [[ -z "${legacy_driver_pod}" ]]; then
     echo "legacy ClusterPolicy driver pod not found"
-    kubectl get pods -n "${TEST_NAMESPACE}" -o wide
+    kubectl get pods -n "${TEST_NAMESPACE}" -o wide --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
     exit 1
 fi
 
 operator_name=$(get_helm_release_name)
 if [[ -z "${operator_name}" ]]; then
     echo "GPU Operator Helm release not found in namespace ${TEST_NAMESPACE}"
-    ${HELM} list -n "${TEST_NAMESPACE}"
+    ${HELM} list -n "${TEST_NAMESPACE}" || true
     exit 1
 fi
 
diff --git a/tests/scripts/node-exec.sh b/tests/scripts/node-exec.sh
new file mode 100755
index 000000000..9025dc9d6
--- /dev/null
+++ b/tests/scripts/node-exec.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Copyright NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+readonly SCRIPT_DIR
+readonly NODE_OPERATIONS="${SCRIPT_DIR}/node-operations.sh"
+
+require_readable_file() {
+	local name="${1}"
+	local value="${2}"
+
+	if [[ -z "${value}" ]]; then
+		echo "Error: ${name} must be set when NODE_SSH_HOST is set" >&2
+		exit 1
+	fi
+	if [[ ! -r "${value}" ]]; then
+		echo "Error: ${name} '${value}' does not exist or is not readable" >&2
+		exit 1
+	fi
+}
+
+if [[ -z "${NODE_SSH_HOST:-}" ]]; then
+	echo "Running '$*' locally"
+	exec bash "${NODE_OPERATIONS}" "$@"
+fi
+
+require_readable_file "NODE_SSH_KEY" "${NODE_SSH_KEY:-}"
+require_readable_file "NODE_SSH_KNOWN_HOSTS" "${NODE_SSH_KNOWN_HOSTS:-}"
+
+REMOTE_ARGS=""
+if (( $# )); then
+	printf -v REMOTE_ARGS ' %q' "$@"
+fi
+
+echo "Running '$*' on ${NODE_SSH_HOST}"
+ssh -i "${NODE_SSH_KEY}" \
+	-o BatchMode=yes \
+	-o ConnectTimeout=30 \
+	-o StrictHostKeyChecking=accept-new \
+	-o UserKnownHostsFile="${NODE_SSH_KNOWN_HOSTS}" \
+	"${NODE_SSH_HOST}" \
+	"bash -s --${REMOTE_ARGS}" < "${NODE_OPERATIONS}"
diff --git a/tests/scripts/node-operations.sh b/tests/scripts/node-operations.sh
new file mode 100755
index 000000000..a37fa5ad4
--- /dev/null
+++ b/tests/scripts/node-operations.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+# Copyright NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+usage() {
+	cat <<'EOF'
+Usage: node-operations.sh <operation> [args...]
+
+Operations:
+  load-modules
+      Load the kernel modules required by the GPU Operator.
+
+  restart-operator-container <runtime>
+      Kill the running gpu-operator container so that kubernetes restarts it.
+      Supported runtimes: containerd, docker.
+EOF
+}
+
+load_modules() {
+	echo "Load kernel modules i2c_core and ipmi_msghandler"
+	sudo modprobe -a i2c_core ipmi_msghandler
+}
+
+restart_operator_container() {
+	local runtime="${1:-}"
+	local container_id=""
+
+	case "${runtime}" in
+	containerd)
+		# The operator is the only container that has the string '"gpu-operator"'
+		# TODO: This requires permissions on containerd.sock
+		container_id="$(sudo crictl ps --name gpu-operator | awk '{if(NR>1)print $1}')" || true
+		if [[ -z "${container_id}" ]]; then
+			echo "Error: no running gpu-operator container found via crictl" >&2
+			return 1
+		fi
+		sudo crictl rm --force "${container_id}"
+		;;
+	docker)
+		# The operator is the only container that has the string '"gpu-operator"'
+		container_id="$(docker ps --format '{{.ID}} {{.Command}}' | grep "gpu-operator" | cut -f 1 -d ' ')" || true
+		if [[ -z "${container_id}" ]]; then
+			echo "Error: no running gpu-operator container found via docker" >&2
+			return 1
+		fi
+		docker kill "${container_id}"
+		;;
+	*)
+		echo "Error: unknown runtime '${runtime}'. Supported runtimes: containerd, docker" >&2
+		return 1
+		;;
+	esac
+}
+
+main() {
+	if [[ $# -lt 1 ]]; then
+		usage >&2
+		exit 2
+	fi
+
+	local operation="${1}"
+	shift
+
+	case "${operation}" in
+		load-modules)
+			load_modules "$@"
+			;;
+		restart-operator-container)
+			restart_operator_container "$@"
+			;;
+		*)
+			echo "Error: unknown operation '${operation}'" >&2
+			usage >&2
+			exit 2
+			;;
+	esac
+}
+
+main "$@"
diff --git a/tests/scripts/update-clusterpolicy.sh b/tests/scripts/update-clusterpolicy.sh
index 7a53901c7..232b33dd3 100755
--- a/tests/scripts/update-clusterpolicy.sh
+++ b/tests/scripts/update-clusterpolicy.sh
@@ -150,7 +150,7 @@ test_gpu_sharing() {
     kubectl wait --for=condition=available --timeout=300s deployment/nvidia-plugin-test -n $TEST_NAMESPACE
     if [ $? -ne 0 ]; then
         echo "cannot run parallel pods with GPU sharing enabled"
-        kubectl get pods -l app=nvidia-plugin-test -n $TEST_NAMESPACE
+        kubectl get pods -l app=nvidia-plugin-test -n "${TEST_NAMESPACE}" --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
         exit 1
     fi
 
@@ -243,14 +243,22 @@ test_custom_labels_override() {
   for operand in $operands
   do
     echo "checking $operand labels"
-    for pod in $(kubectl get pods -n "$TEST_NAMESPACE" -l app="$operand" --output=jsonpath={.items..metadata.name})
+    if ! operand_pods=$(kubectl get pods -n "$TEST_NAMESPACE" -l app="$operand" --output=jsonpath={.items..metadata.name} --request-timeout="${KUBECTL_REQUEST_TIMEOUT}"); then
+      echo "cannot list $operand pods to verify the overridden labels"
+      exit 1
+    fi
+    if [ -z "$operand_pods" ]; then
+      echo "no $operand pods found when verifying the overridden labels"
+      exit 1
+    fi
+    for pod in $operand_pods
     do
-      cp_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath={.metadata.labels.cloudprovider})
+      cp_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath='{.metadata.labels.cloudprovider}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}")
       if [ "$cp_label_value" != "aws" ]; then
           echo "Custom Label cloudprovider is incorrect when clusterpolicy labels are overridden - $pod"
           exit 1
       fi
-      platform_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath={.metadata.labels.platform})
+      platform_label_value=$(kubectl get pod -n "$TEST_NAMESPACE" "$pod" --output jsonpath='{.metadata.labels.platform}' --request-timeout="${KUBECTL_REQUEST_TIMEOUT}")
       if [ "$platform_label_value" != "kubernetes" ]; then
           echo "Custom Label platform is incorrect when clusterpolicy labels are overridden - $pod"
           exit 1
diff --git a/tests/scripts/update-nvidiadriver.sh b/tests/scripts/update-nvidiadriver.sh
index d10467365..a0f812ee7 100755
--- a/tests/scripts/update-nvidiadriver.sh
+++ b/tests/scripts/update-nvidiadriver.sh
@@ -16,12 +16,12 @@ DEFAULT_NVIDIA_DRIVER_NAME="${DEFAULT_NVIDIA_DRIVER_NAME:-e2e-default-driver}"
 DUPLICATE_DEFAULT_NVIDIA_DRIVER_NAME="${DUPLICATE_DEFAULT_NVIDIA_DRIVER_NAME:-e2e-duplicate-default-driver}"
 
 get_default_nvidiadriver_name() {
-    kubectl get nvidiadriver -o json |
+    kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
         jq -r '.items[] | select(.spec.default == true) | .metadata.name' | head -n 1
 }
 
 get_default_nvidiadriver_count() {
-    kubectl get nvidiadriver -o json |
+    kubectl get nvidiadriver -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
         jq '[.items[] | select(.spec.default == true)] | length'
 }
 
@@ -56,7 +56,7 @@ set_default_driver() {
 
 wait_for_default_nvidiadriver() {
     local expected_name=$1
-    local current_time=0
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${expected_name} to be the only default"
     while :; do
@@ -67,14 +67,13 @@ wait_for_default_nvidiadriver() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${expected_name} to be the only default"
-            kubectl get nvidiadriver
+            kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
@@ -83,7 +82,7 @@ test_arbitrary_name_default_nvidiadriver() {
     current_default=$(get_default_nvidiadriver_name)
     if [[ -z "${current_default}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver
+        kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
         exit 1
     fi
 
@@ -110,7 +109,7 @@ create_nvidiadriver() {
     default_name=$(get_default_nvidiadriver_name)
     if [[ -z "${default_name}" ]]; then
         echo "default NVIDIADriver not found"
-        kubectl get nvidiadriver
+        kubectl get nvidiadriver --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
         exit 1
     fi
 
@@ -120,41 +119,54 @@ create_nvidiadriver() {
 
 wait_for_nvidiadriver_owner() {
     local driver_name=$1
-    local current_time=0
-    local gpu_node_count
+    local deadline=$((SECONDS + $((60 * 15))))
+    local gpu_node_count=""
+    local node_count=""
+    local owned_count
 
-    gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
-    echo "Waiting for ${gpu_node_count} GPU node(s) to be owned by NVIDIADriver/${driver_name}"
+    echo "Waiting for the GPU node(s) to be owned by NVIDIADriver/${driver_name}"
 
     while :; do
-        owned_count=$(kubectl get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers | wc -l)
-        if [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
-            echo "All GPU nodes are owned by NVIDIADriver/${driver_name}"
-            break
+        if [[ "${gpu_node_count:-0}" -le 0 ]]; then
+            if node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+                if (( node_count > 0 )); then
+                    gpu_node_count="${node_count}"
+                fi
+            else
+                api_unreachable "counting the GPU nodes"
+            fi
+        fi
+
+        if owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
+            if [[ "${gpu_node_count:-0}" -gt 0 ]] && [[ "${owned_count}" -eq "${gpu_node_count}" ]]; then
+                echo "All GPU nodes are owned by NVIDIADriver/${driver_name}"
+                break
+            fi
+        else
+            api_unreachable "counting the nodes owned by NVIDIADriver/${driver_name}"
         fi
 
-        if [[ "${current_time}" -gt $((60 * 15)) ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} ownership"
-            kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+            kubectl get nodes -l nvidia.com/gpu.present=true -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" |
+                jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
             exit 1
         fi
 
-        echo "NVIDIADriver/${driver_name} owns ${owned_count}/${gpu_node_count} GPU node(s)"
+        echo "NVIDIADriver/${driver_name} owns ${owned_count:-unknown}/${gpu_node_count:-unknown} GPU node(s)"
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
 get_nvidiadriver_daemonsets() {
     local driver_name=$1
-    kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o json |
+    kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" |
         jq --arg driver_name "${driver_name}" '.items | map(select(.spec.template.spec.nodeSelector["nvidia.com/gpu-operator.driver.owner"] == $driver_name))'
 }
 
 wait_for_nvidiadriver_daemonsets() {
     local driver_name=$1
-    local current_time=0
+    local deadline=$((SECONDS + $((60 * 15))))
 
     echo "Waiting for daemonsets owned by NVIDIADriver/${driver_name}"
     while :; do
@@ -164,14 +176,13 @@ wait_for_nvidiadriver_daemonsets() {
             break
         fi
 
-        if [[ "${current_time}" -gt $((60 * 15)) ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for daemonsets owned by NVIDIADriver/${driver_name}"
-            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml
+            kubectl get daemonset -l "app.kubernetes.io/component=nvidia-driver" -n "$TEST_NAMESPACE" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
@@ -184,20 +195,19 @@ test_driver_image_updates() {
     fi
 
     # Verify update is applied to Driver Daemonset
-    local current_time=0
+    local deadline=$((SECONDS + 120))
     while :; do
         if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e --arg version "${TARGET_DRIVER_VERSION}" 'length > 0 and all(.[]; .spec.template.spec.containers[0].image | contains($version))' >/dev/null; then
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "Image update failed for driver daemonset to version $TARGET_DRIVER_VERSION"
-            get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
+            get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
     echo "driver daemonset image updated successfully to version $TARGET_DRIVER_VERSION"
 
@@ -223,20 +233,19 @@ test_custom_labels_override() {
 
   # Wait for the operator to update the pod template with new labels
   echo "Waiting for DaemonSet pod template to be updated with new labels..."
-  local current_time=0
+  local deadline=$((SECONDS + 120))
   while :; do
     if get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" | jq -e 'length > 0 and all(.[]; .spec.template.metadata.labels.cloudprovider == "aws" and .spec.template.metadata.labels.platform == "kubernetes")' >/dev/null; then
       break
     fi
 
-    if [[ "${current_time}" -gt 120 ]]; then
+    if (( SECONDS > deadline )); then
       echo "timeout reached waiting for DaemonSet pod template labels"
-      get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}"
+      get_nvidiadriver_daemonsets "${NVIDIA_DRIVER_NAME}" || true
       exit 1
     fi
 
     sleep 5
-    current_time=$((${current_time} + 5))
   done
 
   # Delete driver pod to force recreation with updated labels. Existing pods are not automatically restarted due to the DaemonSet's 'OnDelete` updateStrategy.
@@ -249,11 +258,21 @@ test_custom_labels_override() {
   check_nvidia_driver_pods_ready
 
   echo "checking nvidia-driver-daemonset labels"
-  labeled_pod_count=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver,cloudprovider=aws,platform=kubernetes" --no-headers | wc -l)
-  gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
+  if ! labeled_pod_count=$(kubectl_count get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver,cloudprovider=aws,platform=kubernetes" --no-headers); then
+    echo "cannot count the labelled NVIDIADriver/${NVIDIA_DRIVER_NAME} pods"
+    exit 1
+  fi
+  if ! gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+    echo "cannot count the GPU nodes"
+    exit 1
+  fi
+  if [[ "${gpu_node_count}" -eq 0 ]]; then
+    echo "no GPU nodes found while verifying NVIDIADriver/${NVIDIA_DRIVER_NAME} labels"
+    exit 1
+  fi
   if [[ "${labeled_pod_count}" -ne "${gpu_node_count}" ]]; then
     echo "Custom labels are missing from one or more NVIDIADriver/${NVIDIA_DRIVER_NAME} pods"
-    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels
+    kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/component=nvidia-driver" --show-labels --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" || true
     exit 1
   fi
 }
@@ -263,12 +282,22 @@ assert_nvidiadriver_owner_count() {
     local gpu_node_count
     local owned_count
 
-    gpu_node_count=$(kubectl get node -l nvidia.com/gpu.present=true --no-headers | wc -l)
-    owned_count=$(kubectl get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers | wc -l)
+    if ! gpu_node_count=$(kubectl_count get node -l nvidia.com/gpu.present=true --no-headers); then
+        echo "cannot count the GPU nodes"
+        exit 1
+    fi
+    if ! owned_count=$(kubectl_count get nodes -l "nvidia.com/gpu.present=true,nvidia.com/gpu-operator.driver.owner=${driver_name}" --no-headers); then
+        echo "cannot count the nodes owned by NVIDIADriver/${driver_name}"
+        exit 1
+    fi
+    if [[ "${gpu_node_count}" -eq 0 ]]; then
+        echo "no GPU nodes found while checking NVIDIADriver/${driver_name} ownership"
+        exit 1
+    fi
     if [[ "${owned_count}" -ne "${gpu_node_count}" ]]; then
         echo "Expected ${gpu_node_count} GPU node(s) to remain owned by NVIDIADriver/${driver_name}, found ${owned_count}"
-        kubectl get nodes -l nvidia.com/gpu.present=true -o json |
-            jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv'
+        kubectl get nodes -l nvidia.com/gpu.present=true -o json --request-timeout="${KUBECTL_LOG_TIMEOUT}" |
+            jq -r '.items[] | [.metadata.name, (.metadata.labels["nvidia.com/gpu-operator.driver.owner"] // "-")] | @tsv' || true
         exit 1
     fi
 }
@@ -276,35 +305,34 @@ assert_nvidiadriver_owner_count() {
 wait_for_nvidiadriver_condition_message() {
     local driver_name=$1
     local message=$2
-    local current_time=0
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${driver_name} status message to contain: ${message}"
     while :; do
-        if kubectl get nvidiadriver/"${driver_name}" -o json | jq -e --arg message "${message}" '
+        if kubectl get nvidiadriver/"${driver_name}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" | jq -e --arg message "${message}" '
             (.status.state // "") == "notReady" and
             ([.status.conditions[]?.message // ""] | any(contains($message)))
         ' >/dev/null; then
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} status message"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml
+            kubectl get nvidiadriver/"${driver_name}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }
 
 wait_for_nvidiadriver_ready() {
     local driver_name=$1
-    local current_time=0
+    local deadline=$((SECONDS + 120))
 
     echo "Waiting for NVIDIADriver/${driver_name} to report Ready"
     while :; do
-        if kubectl get nvidiadriver/"${driver_name}" -o json | jq -e '
+        if kubectl get nvidiadriver/"${driver_name}" -o json --request-timeout="${KUBECTL_REQUEST_TIMEOUT}" | jq -e '
             (.status.state // "") == "ready" and
             ([.status.conditions[]? | select(.type == "Ready" and .status == "True")] | length > 0) and
             ([.status.conditions[]? | select(.type == "Error" and .status == "True")] | length == 0)
@@ -312,14 +340,13 @@ wait_for_nvidiadriver_ready() {
             break
         fi
 
-        if [[ "${current_time}" -gt 120 ]]; then
+        if (( SECONDS > deadline )); then
             echo "timeout reached waiting for NVIDIADriver/${driver_name} to report Ready"
-            kubectl get nvidiadriver/"${driver_name}" -o yaml
+            kubectl get nvidiadriver/"${driver_name}" -o yaml --request-timeout="${KUBECTL_LOG_TIMEOUT}" || true
             exit 1
         fi
 
         sleep 5
-        current_time=$((${current_time} + 5))
     done
 }