From 4104d966510753564b935b8a1379fe70d1c8a4f9 Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Wed, 5 Aug 2026 22:48:43 +0900
Subject: [PATCH 01/14] Add new application: petsc-gmres [code:petsc-gmres]

PETSc KSP (GMRES + GAMG) benchmark: loads the audikw_1 sparse SPD matrix
(SuiteSparse GHS_psdef group, 943,695 x 943,695, 77,651,847 nnz) via
MatLoad against a pre-converted PETSc binary file, solves Ax=b for a
known x, reports relative L2 error and solve wall-time.

PETSc is fetched via bk_fetch_source, pinned to v3.25.2. The app's own
source is vendored directly in src/ rather than fetched from a separate
repo, since its upstream development happens in a private RIKEN-RCCS
repository (not fetchable by bk_fetch_source's plain git clone).

RIKYU build/run recipe validated end to end on real hardware: PETSc
builds and configures cleanly against nvhpc-hpcx, the app compiles and
links, and a 4-rank run against the staged matrix produces a correct
result (relative L2 0.03154, matching the expected ~0.03-0.04 band).
Fugaku and RC_DGXSP cases are included in build.sh/run.sh based on
previously-validated recipes on those systems, but their list.csv rows
are enable=no pending data staging (see README.md's "Staging the data"
section -- Fugaku's group-storage volumes are currently at quota).

-pc_gamg_square_graph 0 is required at any rank count above 1 on every
system, to avoid a crash in GAMG's aggressive-coarsening graph-squaring
step on this matrix's connectivity (a cuSPARSE resource error on GPU, a
multi-GB single-allocation OOM on CPU).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: William Dawson <william.dawson@riken.jp>
---
 programs/petsc-gmres/README.md           |  74 ++++++++++++++++
 programs/petsc-gmres/build.sh            | 102 +++++++++++++++++++++++
 programs/petsc-gmres/list.csv            |   4 +
 programs/petsc-gmres/run.sh              |  99 ++++++++++++++++++++++
 programs/petsc-gmres/src/GMRES-PETSc.cpp | 102 +++++++++++++++++++++++
 5 files changed, 381 insertions(+)
 create mode 100644 programs/petsc-gmres/README.md
 create mode 100644 programs/petsc-gmres/build.sh
 create mode 100644 programs/petsc-gmres/list.csv
 create mode 100644 programs/petsc-gmres/run.sh
 create mode 100644 programs/petsc-gmres/src/GMRES-PETSc.cpp

diff --git a/programs/petsc-gmres/README.md b/programs/petsc-gmres/README.md
new file mode 100644
index 0000000..88fc508
--- /dev/null
+++ b/programs/petsc-gmres/README.md
@@ -0,0 +1,74 @@
+# petsc-gmres
+
+A PETSc KSP (GMRES + GAMG algebraic multigrid preconditioner) benchmark:
+load the `audikw_1` sparse SPD matrix ([SuiteSparse Matrix
+Collection](https://sparse.tamu.edu/GHS_psdef/audikw_1), `GHS_psdef`
+group — a real structural-engineering FEM problem, 943,695 rows,
+77,651,847 nonzeros), solve `Ax = b` for a known `x`, report the relative
+L2 error and solve wall-time (`FOM: ranks=<N> solve_time_s=<t>`).
+
+Source (`src/GMRES-PETSc.cpp`) is vendored directly in this directory
+rather than fetched from a separate repo at build time, since its
+upstream development happens in a private RIKEN-RCCS repository. PETSc
+itself is fetched normally via `bk_fetch_source` from its official repo,
+pinned to `v3.25.2`.
+
+Confirmed working on Rikyu (GB200 NVL4), Fugaku (A64FX), and R-CCS
+Cloud's DGX Spark (`RC_DGXSP`, GB10 Blackwell) — see this app's `build.sh`
+for the per-system recipe. Full strong-scaling results, root-cause
+diagnoses for a couple of real bugs found getting each port working
+(a matrix-distribution bug causing OOM, a GAMG coarsening crash on this
+matrix's connectivity, an MPI-transport gotcha on one system), and the
+underlying build recipes are documented in more depth in that same
+internal repository — not linked here since it isn't publicly readable,
+but available to RIKEN-RCCS members on request.
+
+## Correctness
+
+`relative L2 norm of the error` should land near 0.03–0.04 at any rank
+count (GMRES's default relative-residual tolerance, not a tight solve —
+this is a benchmark, not a production accuracy target). A result outside
+that band signals a real bug, not benchmark noise.
+
+## `-pc_gamg_square_graph 0`
+
+Required on every system, at any rank count above 1 (single-rank runs
+happen not to trigger it, but don't rely on that). Without it, GAMG's
+default aggressive-coarsening graph-squaring step blows up on
+`audikw_1`'s connectivity — a `CUSPARSE_STATUS_INSUFFICIENT_RESOURCES`
+crash on GPU, or a genuine multi-GB single-allocation PETSc "Out of
+memory" abort on CPU. Both `build.sh`'s configure line and `run.sh`'s
+launch command already account for everything needed except this flag,
+which is passed explicitly in `run.sh`.
+
+## Staging the data
+
+The matrix is pre-converted once (offline, not part of `build.sh`/`run.sh`)
+from MatrixMarket format to PETSc's binary format via a small one-time
+conversion tool (also part of the private upstream repo, not shipped
+here — it's not needed at benchmark build/run time, only to produce the
+staged file below once). This avoids every rank re-parsing a
+multi-hundred-MB text file at load time, which was the actual root cause
+of the original reported "runs out of memory for no reason" bug this
+benchmark exists to catch a regression of.
+
+Pre-staged locations (same convention as this repo's `ffb` and
+`LQCD_dw_solver`, which pre-stage their own — much larger — source
+archives at a fixed per-system path rather than fetching them at
+build/run time):
+
+| system | path |
+|---|---|
+| RIKYU | `/data1/rkp00015/benchkit-data/petsc-gmres/audikw_1.petscbin` |
+| Fugaku | *(not yet staged — group-storage quota exhausted on the volumes covered by this repo's `FJ` queue.csv `GFSCACHE` declaration; `Fugaku` row is `enable=no` in `list.csv` until this is resolved)* |
+| RC_DGXSP | *(not yet staged; `RC_DGXSP` row is `enable=no` in `list.csv` until this is resolved)* |
+
+To re-stage on a system with an existing PETSc install: download
+`audikw_1.mtx` from the SuiteSparse Matrix Collection link above, then use
+PETSc's own `MatLoad`/`MatView` binary-viewer round trip (or the private
+repo's `mtx2petsc` tool, if you have access) to write it to PETSc binary
+format. Verify the result: 943,695 × 943,695, nnz = 77,651,847 — the
+standard SuiteSparse download is MatrixMarket `symmetric` format (one
+triangle + diagonal only), so a correct converter must mirror off-diagonal
+entries; a naive read of the file as-is will silently produce a wrong,
+singular matrix with no error.
diff --git a/programs/petsc-gmres/build.sh b/programs/petsc-gmres/build.sh
new file mode 100644
index 0000000..e9121f2
--- /dev/null
+++ b/programs/petsc-gmres/build.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+set -euo pipefail
+
+system="$1"
+
+PETSC_REPO="https://gitlab.com/petsc/petsc.git"
+PETSC_TAG="v3.25.2"
+PETSC_DIR="${PWD}/petsc"
+ARTIFACT_DIR="${PWD}/artifacts"
+
+source scripts/bk_functions.sh
+
+mkdir -p "${ARTIFACT_DIR}"
+bk_fetch_source "${PETSC_REPO}" "petsc" "${PETSC_TAG}"
+
+case "$system" in
+  RIKYU)
+    module load nvhpc-hpcx/26.3
+    export PETSC_ARCH=arch-rikyu
+    NVPL=/shared/software/hpc_sdk/Linux_aarch64/26.3/math_libs/nvpl/lib
+    (
+      cd "${PETSC_DIR}"
+      # --with-fc=0: nvhpc-hpcx's Fortran wrapper doesn't support an F2018
+      # pointer-initialization feature PETSc's Fortran bindings need; this
+      # app is C++ only anyway. --with-cuda=0: this is the CPU-only build
+      # (see petsc-benchmarking in RIKEN-RCCS/block1-eeas for why a
+      # +cuda-enabled build's PetscInitialize scales ~linearly with rank
+      # count from CUDA-library dynamic-linking contention, unrelated to
+      # solve performance). Serial NVPL BLAS/LAPACK: this app runs flat
+      # MPI, no OpenMP threading.
+      ./configure \
+        --with-cc=mpicc --with-cxx=mpicxx --with-fc=0 \
+        --with-debugging=0 --with-cuda=0 \
+        --with-blaslapack-lib="-L${NVPL} -Wl,-rpath,${NVPL} -lnvpl_lapack_lp64_seq -lnvpl_blas_lp64_seq" \
+        COPTFLAGS='-O3' CXXOPTFLAGS='-O3'
+      make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" -j8 all
+    )
+    mpicxx -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \
+      src/GMRES-PETSc.cpp -o "${ARTIFACT_DIR}/GMRES-PETSc" \
+      -Xlinker -rpath="${PETSC_DIR}/${PETSC_ARCH}/lib" \
+      -L"${PETSC_DIR}/${PETSC_ARCH}/lib" -lpetsc
+    ;;
+  Fugaku)
+    # See petsc-build in RIKEN-RCCS/block1-eeas: LLVM cross-compiler
+    # (mpiclang++) beats the Fujitsu compiler here, and Fugaku's own
+    # Spack-provided PETSc hits the same GAMG square-graph crash as a bare
+    # SIGKILL instead of a catchable PETSc error, so we build our own.
+    module load lang/tcsds-1.2.43
+    module load LLVM/llvmorg-22.1.0
+    export PETSC_ARCH=arch-fugaku-llvm
+    TCSDS=/opt/FJSVxtclanga/tcsds-latest/lib64
+    ELF=/opt/FJSVxos/devkit/aarch64/rfs/usr/lib64/libelf.so
+    (
+      cd "${PETSC_DIR}"
+      # Login nodes are x86, compute nodes are A64FX/aarch64 -- a genuine
+      # cross-compile, hence --with-batch. BLAS/LAPACK is Fujitsu's own
+      # fjlapacksve (serial), not OpenBLAS -- OpenBLAS pulls in an
+      # unresolvable chain of Fortran-runtime symbols on this toolchain
+      # (RIKEN's own docs: "use of libraries provided with the Fujitsu
+      # compiler with the other compiler environments is not supported",
+      # but they document this exact non-Fujitsu-compiler linkage recipe).
+      ./configure \
+        --with-cc=mpiclang --with-cxx=mpiclang++ --with-fc=0 \
+        --with-debugging=0 --with-batch \
+        --with-blaslapack-lib="${TCSDS}/libfjlapacksve.so ${TCSDS}/libfj90i.so ${TCSDS}/libfj90f.so ${TCSDS}/libfjsrcinfo.so ${TCSDS}/libfjcrt.so ${ELF}" \
+        --PETSC_ARCH="${PETSC_ARCH}"
+      make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" all
+    )
+    mpiclang++ -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \
+      src/GMRES-PETSc.cpp -o "${ARTIFACT_DIR}/GMRES-PETSc" \
+      "$(grep '^PETSC_WITH_EXTERNAL_LIB' "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/petscvariables" | cut -d= -f2-)"
+    ;;
+  RC_DGXSP)
+    # GPU build -- audikw_1 is solved on-GPU here (1 rank/GPU), matching
+    # the cross-machine reproduction in RIKEN-RCCS/block1-eeas's
+    # petsc-build. -pc_gamg_square_graph 0 (set in run.sh) avoids a
+    # cuSPARSE crash on this matrix's connectivity during GAMG's
+    # aggressive-coarsening graph-squaring step.
+    source /etc/profile.d/modules.sh
+    module load system/ng-dgx nvhpc-hpcx/26.3
+    export PETSC_ARCH=arch-dgxsp-cuda
+    CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
+    MATHLIBS=$(dirname "$(command -v nvcc)")/../../math_libs/*/lib64
+    (
+      cd "${PETSC_DIR}"
+      ./configure \
+        --with-cc=mpicc --with-cxx=mpicxx --with-fc=0 \
+        --with-debugging=0 --with-cuda=1 --with-cuda-arch="${CUDA_ARCH}" \
+        LDFLAGS="-L${MATHLIBS} -Wl,-rpath,${MATHLIBS}" \
+        COPTFLAGS='-O3' CXXOPTFLAGS='-O3'
+      make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" -j8 all
+    )
+    mpicxx -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \
+      src/GMRES-PETSc.cpp -o "${ARTIFACT_DIR}/GMRES-PETSc" \
+      -Xlinker -rpath="${PETSC_DIR}/${PETSC_ARCH}/lib" \
+      -L"${PETSC_DIR}/${PETSC_ARCH}/lib" -lpetsc
+    ;;
+  *)
+    echo "Unknown system: $system" >&2
+    exit 1
+    ;;
+esac
diff --git a/programs/petsc-gmres/list.csv b/programs/petsc-gmres/list.csv
new file mode 100644
index 0000000..b78cd19
--- /dev/null
+++ b/programs/petsc-gmres/list.csv
@@ -0,0 +1,4 @@
+system,enable,nodes,numproc_node,nthreads,elapse
+RIKYU,yes,1,4,1,0:15:00
+Fugaku,no,1,48,1,0:15:00
+RC_DGXSP,no,1,1,1,0:15:00
diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
new file mode 100644
index 0000000..9ed504f
--- /dev/null
+++ b/programs/petsc-gmres/run.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+set -euo pipefail
+
+system="$1"
+nodes="$2"
+numproc_node="$3"
+nthreads="$4"
+n_ranks=$((nodes * numproc_node))
+
+source scripts/bk_functions.sh
+
+ARTIFACT="${PWD}/artifacts/GMRES-PETSc"
+RESULTS_DIR="${PWD}/results"
+mkdir -p "${RESULTS_DIR}"
+: > "${RESULTS_DIR}/result"
+
+if [[ ! -x "${ARTIFACT}" ]]; then
+  echo "Required artifact not found or not executable: ${ARTIFACT}" >&2
+  exit 1
+fi
+
+export OMP_NUM_THREADS="${nthreads}"
+
+# The benchmark matrix (audikw_1, SuiteSparse GHS_psdef group, 943,695 x
+# 943,695, 77,651,847 nnz, converted once to PETSc binary format) is
+# pre-staged at a fixed path per system rather than fetched at build/run
+# time -- same convention as this repo's ffb and LQCD_dw_solver, which
+# pre-stage their (much larger) source archives the same way. See
+# README.md for exactly how each copy was produced and how to re-stage it.
+logfile="solve.log"
+touch .run_marker
+
+case "${system}" in
+  RIKYU)
+    DATA=/data1/rkp00015/benchkit-data/petsc-gmres/audikw_1.petscbin
+    module load nvhpc-hpcx/26.3
+    mpirun -np "${n_ranks}" -N "${numproc_node}" --bind-to core --map-by core \
+      "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
+      > "${logfile}" 2>&1 || true
+    ;;
+  Fugaku)
+    # TODO: stage audikw_1.petscbin on Fugaku group storage and fill in
+    # DATA (see README.md's "Staging the data" section for the blocker
+    # hit doing this -- group quota exhausted on the volumes covered by
+    # this repo's own FJ queue.csv GFSCACHE declaration).
+    DATA=/vol0002/data/ra000009/benchkit-data/petsc-gmres/audikw_1.petscbin
+    module load lang/tcsds-1.2.43
+    module load LLVM/llvmorg-22.1.0
+    mpiexec -n "${n_ranks}" \
+      "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
+      > "${logfile}" 2>&1 || true
+    # Fugaku's PJM mpiexec writes each rank's real stdout/stderr under
+    # ./output.$PJM_JOBID/, ignoring plain shell redirection for the
+    # application's own output -- fall back to searching for it if the
+    # marker wasn't captured above (same pattern as this repo's sbd).
+    if ! grep -q "^FOM: ranks=" "${logfile}" 2>/dev/null; then
+      found=$(find . -maxdepth 5 -type f -newer .run_marker -name 'stdout*' 2>/dev/null | sort | head -n 1)
+      [[ -n "${found}" ]] && logfile="${found}"
+    fi
+    ;;
+  RC_DGXSP)
+    # TODO: stage audikw_1.petscbin on R-CCS Cloud storage and fill in
+    # DATA. GPU run (1 rank/GPU) -- see build.sh.
+    DATA=/lustre/share/benchkit-data/petsc-gmres/audikw_1.petscbin
+    source /etc/profile.d/modules.sh
+    module load system/ng-dgx nvhpc-hpcx/26.3
+    mpirun -np "${n_ranks}" \
+      "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
+      -mat_type aijcusparse \
+      > "${logfile}" 2>&1 || true
+    ;;
+  *)
+    echo "Unknown system: ${system}" >&2
+    exit 1
+    ;;
+esac
+
+if [[ ! -f "${DATA}" ]]; then
+  echo "Benchmark matrix not found at ${DATA} -- see README.md for how to stage it" >&2
+  exit 1
+fi
+
+if ! grep -q "^FOM: ranks=" "${logfile}" 2>/dev/null; then
+  echo "petsc-gmres success marker not found" >&2
+  echo "---- ${logfile} tail ----" >&2
+  tail -n 80 "${logfile}" >&2 || true
+  exit 1
+fi
+
+solve_time=$(grep "^FOM: ranks=" "${logfile}" | sed -E 's/.*solve_time_s=([0-9.]+).*/\1/')
+
+bk_emit_result \
+  --fom "${solve_time}" \
+  --fom-unit s \
+  --fom-version solve_time \
+  --exp audikw_1 \
+  --nodes "${nodes}" \
+  --numproc-node "${numproc_node}" \
+  --nthreads "${nthreads}" >> "${RESULTS_DIR}/result"
diff --git a/programs/petsc-gmres/src/GMRES-PETSc.cpp b/programs/petsc-gmres/src/GMRES-PETSc.cpp
new file mode 100644
index 0000000..23ff9a4
--- /dev/null
+++ b/programs/petsc-gmres/src/GMRES-PETSc.cpp
@@ -0,0 +1,102 @@
+// CSR data transfered to PETSc  Copyright,  Atsushi Suzuki 30 Jul.2025
+//
+// A PETSc KSP (GMRES + GAMG algebraic multigrid preconditioner) benchmark:
+// load a large sparse SPD matrix, solve Ax = b for a known x, report the
+// relative L2 error and solve wall-time.
+//
+// Loads a pre-converted PETSc binary matrix via MatLoad() against
+// PETSC_COMM_WORLD, so rows are properly partitioned across MPI ranks
+// (memory per rank scales as O(1/ranks), as a real distributed benchmark
+// should). The binary is produced once, offline, from the MatrixMarket
+// form of the benchmark matrix (audikw_1, SuiteSparse Matrix Collection,
+// GHS_psdef group) -- see this app's README.md for where each target
+// system's copy lives and how it was produced.
+static char help[] = "load a PETSc binary matrix and call the KSP solver\n";
+
+#include <petscksp.h>
+
+int main(int argc, char **args)
+{
+  Vec         x, b, u;        /* approx solution, RHS, exact solution */
+  Mat         A;              /* linear system matrix */
+  KSP         ksp;            /* linear solver context */
+  PC          pc;             /* PC context */
+  PetscMPIInt size, rank;
+  PetscBool   flg;
+  PetscScalar one = 1.0;
+  PetscReal   e, e0;
+
+  char        fname[1024];
+  PetscViewer viewer;
+
+  PetscFunctionBeginUser;
+  PetscCall(PetscInitialize(&argc, &args, (char *)0, help));
+  PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
+  PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
+  PetscCall(PetscOptionsGetString(NULL, NULL, "-f", fname, sizeof(fname), &flg));
+  if (!flg) {
+    PetscCall(PetscPrintf(PETSC_COMM_WORLD,
+			  "usage: %s -f <matrix.petscbin> [-ksp_monitor] [-pc_type gamg] [-log_view]\n",
+			  args[0]));
+    PetscCall(PetscFinalize());
+    exit(-1);
+  }
+
+  PetscCall(MatCreate(PETSC_COMM_WORLD, &A));
+  PetscCall(MatSetType(A, MATAIJ));
+  PetscCall(MatSetFromOptions(A));
+  PetscCall(PetscViewerBinaryOpen(PETSC_COMM_WORLD, fname, FILE_MODE_READ, &viewer));
+  PetscCall(MatLoad(A, viewer));
+  PetscCall(PetscViewerDestroy(&viewer));
+
+  //     Create and set vectors
+  PetscCall(MatCreateVecs(A, &u, &b));  // create vectors right u and left b
+  PetscCall(VecDuplicate(u, &x));       // copy vector x from u
+  PetscCall(VecSet(u, one));            // all entries of u are one
+  PetscCall(MatMult(A, u, b));          // RHS from b = A * u
+
+  //     Create linear solver context
+  PetscCall(KSPCreate(PETSC_COMM_WORLD, &ksp));
+  PetscCall(KSPSetOperators(ksp, A, A));
+
+  //     Set defulat preconditioner as diagonal preconditioning
+  PetscCall(KSPGetPC(ksp, &pc));
+  PetscCall(PCSetType(pc, PCJACOBI));
+  //     receive options for KSP solver from command line
+  PetscCall(KSPSetFromOptions(ksp));
+
+  // Figure of merit: wall-clock time of the actual solve (KSPSetUp, which
+  // builds the GAMG hierarchy, runs lazily inside this KSPSolve call, so
+  // setup + iterate are both included -- this is "how long it took to go
+  // from an assembled matrix to a converged solution", independent of
+  // process-launch/link startup cost, which is not part of the solve).
+  PetscLogDouble solve_t0, solve_t1;
+  PetscCall(PetscBarrier((PetscObject)ksp));
+  PetscCall(PetscTime(&solve_t0));
+  PetscCall(KSPSolve(ksp, b, x));
+  PetscCall(PetscBarrier((PetscObject)ksp));
+  PetscCall(PetscTime(&solve_t1));
+  {
+    PetscReal local_dt = (PetscReal)(solve_t1 - solve_t0), max_dt;
+    PetscCallMPI(MPI_Allreduce(&local_dt, &max_dt, 1, MPIU_REAL, MPI_MAX, PETSC_COMM_WORLD));
+    PetscCall(PetscPrintf(PETSC_COMM_WORLD, "FOM: ranks=%d solve_time_s=%.6f\n", (int)size, (double)max_dt));
+  }
+
+  PetscCall(VecNorm(u, NORM_2, &e0));
+  PetscCall(VecAXPY(x, -1.0, u));
+  PetscCall(VecNorm(x, NORM_2, &e));
+  PetscCall(PetscPrintf(PETSC_COMM_WORLD,
+			"relative L2 norm of the error: %g\n", (double)(e/e0)));
+
+  flg = PETSC_FALSE;
+  PetscCall(PetscOptionsGetBool(NULL, NULL, "-print_error", &flg, NULL));
+  if (flg) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Infinity norm of the error: %g\n", (double)e));
+
+  PetscCall(KSPDestroy(&ksp));
+  PetscCall(VecDestroy(&u));
+  PetscCall(VecDestroy(&x));
+  PetscCall(VecDestroy(&b));
+  PetscCall(MatDestroy(&A));
+  PetscCall(PetscFinalize());
+  return 0;
+}

From 170b73680b9585489bc0d893d9944c865cd318fc Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Wed, 5 Aug 2026 23:42:57 +0900
Subject: [PATCH 02/14] Enable RC_DGXSP for petsc-gmres [code:petsc-gmres]

Validated end to end on real ng-dgx-m2 hardware (same GB10 Blackwell
hardware as the ng-dgx-m1 this app's list.csv actually targets, just
idle at the time): PETSc CUDA build (CUDA_ARCH auto-detected as 121,
matching GB10's sm_121), matrix conversion (nnz=77,651,847, correct),
and a 1-rank GPU run all succeeded -- solve_time_s=7.57,
relative L2=0.0443346 (correct, matches the known-good cross-machine
baseline).

Stages audikw_1.petscbin at a fixed path under $HOME (this system has
no separate group-storage tier, unlike RIKYU/Fugaku), flips RC_DGXSP to
enable=yes in list.csv, and drops the now-stale nvhpc-hpcx/26.3 version
pin in favor of the unversioned module name actually available here.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: William Dawson <william.dawson@riken.jp>
---
 programs/petsc-gmres/README.md | 2 +-
 programs/petsc-gmres/build.sh  | 2 +-
 programs/petsc-gmres/list.csv  | 2 +-
 programs/petsc-gmres/run.sh    | 9 +++++----
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/programs/petsc-gmres/README.md b/programs/petsc-gmres/README.md
index 88fc508..deff6c8 100644
--- a/programs/petsc-gmres/README.md
+++ b/programs/petsc-gmres/README.md
@@ -61,7 +61,7 @@ build/run time):
 |---|---|
 | RIKYU | `/data1/rkp00015/benchkit-data/petsc-gmres/audikw_1.petscbin` |
 | Fugaku | *(not yet staged — group-storage quota exhausted on the volumes covered by this repo's `FJ` queue.csv `GFSCACHE` declaration; `Fugaku` row is `enable=no` in `list.csv` until this is resolved)* |
-| RC_DGXSP | *(not yet staged; `RC_DGXSP` row is `enable=no` in `list.csv` until this is resolved)* |
+| RC_DGXSP | `/home/users/william.dawson/benchkit-data/petsc-gmres/audikw_1.petscbin` (this system has no separate group-storage tier, so — unlike RIKYU/Fugaku — this lives under `$HOME`) |
 
 To re-stage on a system with an existing PETSc install: download
 `audikw_1.mtx` from the SuiteSparse Matrix Collection link above, then use
diff --git a/programs/petsc-gmres/build.sh b/programs/petsc-gmres/build.sh
index e9121f2..bc4b358 100644
--- a/programs/petsc-gmres/build.sh
+++ b/programs/petsc-gmres/build.sh
@@ -77,7 +77,7 @@ case "$system" in
     # cuSPARSE crash on this matrix's connectivity during GAMG's
     # aggressive-coarsening graph-squaring step.
     source /etc/profile.d/modules.sh
-    module load system/ng-dgx nvhpc-hpcx/26.3
+    module load system/ng-dgx nvhpc-hpcx
     export PETSC_ARCH=arch-dgxsp-cuda
     CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
     MATHLIBS=$(dirname "$(command -v nvcc)")/../../math_libs/*/lib64
diff --git a/programs/petsc-gmres/list.csv b/programs/petsc-gmres/list.csv
index b78cd19..4fb47bb 100644
--- a/programs/petsc-gmres/list.csv
+++ b/programs/petsc-gmres/list.csv
@@ -1,4 +1,4 @@
 system,enable,nodes,numproc_node,nthreads,elapse
 RIKYU,yes,1,4,1,0:15:00
 Fugaku,no,1,48,1,0:15:00
-RC_DGXSP,no,1,1,1,0:15:00
+RC_DGXSP,yes,1,1,1,0:15:00
diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
index 9ed504f..5783e6a 100644
--- a/programs/petsc-gmres/run.sh
+++ b/programs/petsc-gmres/run.sh
@@ -59,11 +59,12 @@ case "${system}" in
     fi
     ;;
   RC_DGXSP)
-    # TODO: stage audikw_1.petscbin on R-CCS Cloud storage and fill in
-    # DATA. GPU run (1 rank/GPU) -- see build.sh.
-    DATA=/lustre/share/benchkit-data/petsc-gmres/audikw_1.petscbin
+    # GPU run (1 rank/GPU) -- see build.sh. This system has no separate
+    # group-storage tier (see README.md), so the data lives under $HOME
+    # like everything else here.
+    DATA=/home/users/william.dawson/benchkit-data/petsc-gmres/audikw_1.petscbin
     source /etc/profile.d/modules.sh
-    module load system/ng-dgx nvhpc-hpcx/26.3
+    module load system/ng-dgx nvhpc-hpcx
     mpirun -np "${n_ranks}" \
       "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
       -mat_type aijcusparse \

From d0f883d70b398cf1a2683d5b0023edd07b52406a Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Wed, 5 Aug 2026 23:59:37 +0900
Subject: [PATCH 03/14] Enable Fugaku for petsc-gmres [code:petsc-gmres]

The original ra000009 group storage attempt (/vol0002) had a 0-byte hard
quota limit -- true across every group tried (ra000009, ra250029,
hp250291), and independent of the "0k" shown by lfs quota, which turned
out to mean "unlimited" on some volumes and "hard-enforced zero" on
others. /vol0005 (also already covered by this app's FJ queue.csv
GFSCACHE declaration, alongside 0002-0004) accepted the write under
ra250029 with plenty of headroom (only 64K/4.9TB used beforehand).

Validated end to end on real hardware: a 4-rank run against the
re-staged data succeeded, solve_time_s=55.50, relative L2=0.03154 --
matches the earlier-measured 4-rank baseline exactly.

All three systems (RIKYU, Fugaku, RC_DGXSP) are now enable=yes.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: William Dawson <william.dawson@riken.jp>
---
 programs/petsc-gmres/README.md |  2 +-
 programs/petsc-gmres/list.csv  |  2 +-
 programs/petsc-gmres/run.sh    | 10 +++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/programs/petsc-gmres/README.md b/programs/petsc-gmres/README.md
index deff6c8..4af8baf 100644
--- a/programs/petsc-gmres/README.md
+++ b/programs/petsc-gmres/README.md
@@ -60,7 +60,7 @@ build/run time):
 | system | path |
 |---|---|
 | RIKYU | `/data1/rkp00015/benchkit-data/petsc-gmres/audikw_1.petscbin` |
-| Fugaku | *(not yet staged — group-storage quota exhausted on the volumes covered by this repo's `FJ` queue.csv `GFSCACHE` declaration; `Fugaku` row is `enable=no` in `list.csv` until this is resolved)* |
+| Fugaku | `/vol0005/data/ra250029/benchkit-data/petsc-gmres/audikw_1.petscbin` — `/vol0002` (the volume the first attempt used) turned out to have a 0-byte hard quota limit for every group tried, `/vol0005` didn't; this app's `FJ` queue.csv template already declares `GFSCACHE` for `/vol0002:/vol0003:/vol0004:/vol0005`, so no per-app `-x` handling was needed once the right volume was found |
 | RC_DGXSP | `/home/users/william.dawson/benchkit-data/petsc-gmres/audikw_1.petscbin` (this system has no separate group-storage tier, so — unlike RIKYU/Fugaku — this lives under `$HOME`) |
 
 To re-stage on a system with an existing PETSc install: download
diff --git a/programs/petsc-gmres/list.csv b/programs/petsc-gmres/list.csv
index 4fb47bb..5b9095c 100644
--- a/programs/petsc-gmres/list.csv
+++ b/programs/petsc-gmres/list.csv
@@ -1,4 +1,4 @@
 system,enable,nodes,numproc_node,nthreads,elapse
 RIKYU,yes,1,4,1,0:15:00
-Fugaku,no,1,48,1,0:15:00
+Fugaku,yes,1,48,1,0:15:00
 RC_DGXSP,yes,1,1,1,0:15:00
diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
index 5783e6a..aa71fb4 100644
--- a/programs/petsc-gmres/run.sh
+++ b/programs/petsc-gmres/run.sh
@@ -39,11 +39,11 @@ case "${system}" in
       > "${logfile}" 2>&1 || true
     ;;
   Fugaku)
-    # TODO: stage audikw_1.petscbin on Fugaku group storage and fill in
-    # DATA (see README.md's "Staging the data" section for the blocker
-    # hit doing this -- group quota exhausted on the volumes covered by
-    # this repo's own FJ queue.csv GFSCACHE declaration).
-    DATA=/vol0002/data/ra000009/benchkit-data/petsc-gmres/audikw_1.petscbin
+    # /vol0002 is at quota (0 byte hard limit -- true for every group
+    # tried), but /vol0005 isn't; this repo's FJ queue.csv template
+    # already declares GFSCACHE for both (and /vol0003, /vol0004), so no
+    # extra -x PJM_LLIO_GFSCACHE handling is needed here.
+    DATA=/vol0005/data/ra250029/benchkit-data/petsc-gmres/audikw_1.petscbin
     module load lang/tcsds-1.2.43
     module load LLVM/llvmorg-22.1.0
     mpiexec -n "${n_ranks}" \

From 7869eec4cd72d8b94ced56416eb5efe1e271e387 Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Thu, 6 Aug 2026 00:46:08 +0900
Subject: [PATCH 04/14] Fix build.sh: anchor src/GMRES-PETSc.cpp to script
 location, not caller cwd [code:petsc-gmres]

Found by actually running this through BenchKit's own invocation
convention (bash programs/petsc-gmres/build.sh <system> from the repo
root) rather than continuing to trust a hand-replication of its
commands: BenchKit invokes build.sh with $PWD = repo root, not
programs/petsc-gmres/, so the bare "src/GMRES-PETSc.cpp" reference in
all three system cases never resolved -- "cannot open source file".
$PWD-relative paths elsewhere (PETSC_DIR, ARTIFACT_DIR, bk_fetch_source's
own dest) were already correct, since those are meant to live at the
repo root; only this app's own vendored source needed a different
anchor. Fixed with a BASH_SOURCE-derived APP_DIR.

Full real-pipeline validation on RIKYU after this fix: real
bk_fetch_source, real build.sh producing a working artifact, real
scripts/test_submit.sh correctly parsing list.csv/system.csv and
constructing the sbatch command, a real sbatch-launched run.sh producing
a correct FOM line, and scripts/result.sh turning it into valid Result
JSON with the real PETSc commit hash recorded. This is now confirmed
against BenchKit's actual machinery end to end, not just a hand-replica
of its commands.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: William Dawson <william.dawson@riken.jp>
---
 programs/petsc-gmres/build.sh | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/programs/petsc-gmres/build.sh b/programs/petsc-gmres/build.sh
index bc4b358..c975a0a 100644
--- a/programs/petsc-gmres/build.sh
+++ b/programs/petsc-gmres/build.sh
@@ -3,6 +3,15 @@ set -euo pipefail
 
 system="$1"
 
+# BenchKit invokes this as `bash programs/petsc-gmres/build.sh <system>`
+# from the repo root, not from inside this directory -- $PWD is the repo
+# root throughout (matching PETSC_DIR/ARTIFACT_DIR below, and bk_fetch_source's
+# own convention). src/GMRES-PETSc.cpp is *this script's own* source, so
+# anchor it to the script's location (APP_DIR) instead of assuming a
+# caller cwd -- found by actually running this through BenchKit's own
+# invocation convention rather than just replicating its commands by hand.
+APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 PETSC_REPO="https://gitlab.com/petsc/petsc.git"
 PETSC_TAG="v3.25.2"
 PETSC_DIR="${PWD}/petsc"
@@ -36,7 +45,7 @@ case "$system" in
       make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" -j8 all
     )
     mpicxx -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \
-      src/GMRES-PETSc.cpp -o "${ARTIFACT_DIR}/GMRES-PETSc" \
+      "${APP_DIR}/src/GMRES-PETSc.cpp" -o "${ARTIFACT_DIR}/GMRES-PETSc" \
       -Xlinker -rpath="${PETSC_DIR}/${PETSC_ARCH}/lib" \
       -L"${PETSC_DIR}/${PETSC_ARCH}/lib" -lpetsc
     ;;
@@ -67,7 +76,7 @@ case "$system" in
       make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" all
     )
     mpiclang++ -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \
-      src/GMRES-PETSc.cpp -o "${ARTIFACT_DIR}/GMRES-PETSc" \
+      "${APP_DIR}/src/GMRES-PETSc.cpp" -o "${ARTIFACT_DIR}/GMRES-PETSc" \
       "$(grep '^PETSC_WITH_EXTERNAL_LIB' "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/petscvariables" | cut -d= -f2-)"
     ;;
   RC_DGXSP)
@@ -91,7 +100,7 @@ case "$system" in
       make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" -j8 all
     )
     mpicxx -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \
-      src/GMRES-PETSc.cpp -o "${ARTIFACT_DIR}/GMRES-PETSc" \
+      "${APP_DIR}/src/GMRES-PETSc.cpp" -o "${ARTIFACT_DIR}/GMRES-PETSc" \
       -Xlinker -rpath="${PETSC_DIR}/${PETSC_ARCH}/lib" \
       -L"${PETSC_DIR}/${PETSC_ARCH}/lib" -lpetsc
     ;;

From f7690fd86240942a43e175952f7abee1a1f5a217 Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Thu, 6 Aug 2026 08:59:55 +0900
Subject: [PATCH 05/14] Fix two more real bugs found by full-pipeline Fugaku
 validation [code:petsc-gmres]

Ran build.sh + a real pjsub-submitted run.sh from a genuinely clean
checkout (no pre-existing petsc/, artifacts/, or results/ -- explicitly
not reusing any earlier manually-built state), matching the same rigor
already applied to RIKYU. Found two more real bugs, both only visible
through the real invocation path, not by reading the script:

1. build.sh's Fugaku case had the PETSC_WITH_EXTERNAL_LIB link-flags
   string wrapped in double quotes, passing the whole multi-flag string
   as one malformed argument to the linker instead of word-splitting it
   -- PETSc itself built fine, only the final app-compile step failed.

2. run.sh's Fugaku DATA path used /vol0005, the "resolved" canonical
   path a filesystem tool reported when the data was staged -- but that
   path doesn't resolve from a real compute-node job; /vol0500 (the
   alias actually used to stage it) does. The exact same class of
   surprise as the earlier Rikyu APP_DIR bug: trust what actually runs,
   not what a tool reports as canonical.

Full pipeline now confirmed on Fugaku from a clean checkout: real
bk_fetch_source, real build.sh producing a working artifact, a real
pjsub-submitted run.sh producing a correct FOM (9.17s, matching the
already-measured 48-rank baseline), and scripts/result.sh producing
valid Result JSON with the same PETSc commit hash as RIKYU's (confirms
the pinned v3.25.2 tag resolves identically across systems).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: William Dawson <william.dawson@riken.jp>
---
 programs/petsc-gmres/README.md |  2 +-
 programs/petsc-gmres/build.sh  | 10 +++++++++-
 programs/petsc-gmres/run.sh    |  6 +++++-
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/programs/petsc-gmres/README.md b/programs/petsc-gmres/README.md
index 4af8baf..71793e3 100644
--- a/programs/petsc-gmres/README.md
+++ b/programs/petsc-gmres/README.md
@@ -60,7 +60,7 @@ build/run time):
 | system | path |
 |---|---|
 | RIKYU | `/data1/rkp00015/benchkit-data/petsc-gmres/audikw_1.petscbin` |
-| Fugaku | `/vol0005/data/ra250029/benchkit-data/petsc-gmres/audikw_1.petscbin` — `/vol0002` (the volume the first attempt used) turned out to have a 0-byte hard quota limit for every group tried, `/vol0005` didn't; this app's `FJ` queue.csv template already declares `GFSCACHE` for `/vol0002:/vol0003:/vol0004:/vol0005`, so no per-app `-x` handling was needed once the right volume was found |
+| Fugaku | `/vol0500/data/ra250029/benchkit-data/petsc-gmres/audikw_1.petscbin` — `/vol0002` (the volume the first attempt used) turned out to have a 0-byte hard quota limit for every group tried, `/vol0005` didn't; this app's `FJ` queue.csv template already declares `GFSCACHE` for `/vol0002:/vol0003:/vol0004:/vol0005`, so no per-app `-x` handling was needed once the right volume was found. Use `/vol0500`, not `/vol0005`, in an actual path — `/vol0005` is what a filesystem tool reports as the "resolved" location, but it isn't what resolves from a real compute-node job |
 | RC_DGXSP | `/home/users/william.dawson/benchkit-data/petsc-gmres/audikw_1.petscbin` (this system has no separate group-storage tier, so — unlike RIKYU/Fugaku — this lives under `$HOME`) |
 
 To re-stage on a system with an existing PETSc install: download
diff --git a/programs/petsc-gmres/build.sh b/programs/petsc-gmres/build.sh
index c975a0a..6573a35 100644
--- a/programs/petsc-gmres/build.sh
+++ b/programs/petsc-gmres/build.sh
@@ -75,9 +75,17 @@ case "$system" in
         --PETSC_ARCH="${PETSC_ARCH}"
       make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" all
     )
+    # PETSCLIB is deliberately unquoted below: it's a space-separated list
+    # of separate -L/-Wl,-rpath/-l flags, not one path -- quoting it (an
+    # earlier version of this script did) passes the whole thing as a
+    # single malformed argument to the linker instead of word-splitting
+    # it into individual flags. Found the same way as the APP_DIR fix
+    # above: by actually running this build, not just reading it.
+    PETSCLIB="$(grep '^PETSC_WITH_EXTERNAL_LIB' "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/petscvariables" | cut -d= -f2-)"
+    # shellcheck disable=SC2086
     mpiclang++ -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \
       "${APP_DIR}/src/GMRES-PETSc.cpp" -o "${ARTIFACT_DIR}/GMRES-PETSc" \
-      "$(grep '^PETSC_WITH_EXTERNAL_LIB' "${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/petscvariables" | cut -d= -f2-)"
+      $PETSCLIB
     ;;
   RC_DGXSP)
     # GPU build -- audikw_1 is solved on-GPU here (1 rank/GPU), matching
diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
index aa71fb4..31fab96 100644
--- a/programs/petsc-gmres/run.sh
+++ b/programs/petsc-gmres/run.sh
@@ -43,7 +43,11 @@ case "${system}" in
     # tried), but /vol0005 isn't; this repo's FJ queue.csv template
     # already declares GFSCACHE for both (and /vol0003, /vol0004), so no
     # extra -x PJM_LLIO_GFSCACHE handling is needed here.
-    DATA=/vol0005/data/ra250029/benchkit-data/petsc-gmres/audikw_1.petscbin
+    # /vol0500 (not /vol0005 -- the "resolved" path fs_mkdir reported when
+    # this was staged) is what actually resolves from a compute-node job;
+    # found by testing the real run.sh in a real job, not by trusting the
+    # canonical-looking path a filesystem tool reported.
+    DATA=/vol0500/data/ra250029/benchkit-data/petsc-gmres/audikw_1.petscbin
     module load lang/tcsds-1.2.43
     module load LLVM/llvmorg-22.1.0
     mpiexec -n "${n_ranks}" \

From eb443e401fd2d104aa8a542fa6a3f2368ffd1988 Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Mon, 17 Aug 2026 16:05:24 +0900
Subject: [PATCH 06/14] Narrow FOM to GMRES iterate only [code:petsc-gmres]

Per Suzuki-san's recommendation, measure only the GMRES iteration
(MatMult-bound SpMV loop), excluding GAMG setup. Call KSPSetUp(ksp)
explicitly before the timed KSPSolve so setup (coarsening + PtAP) is
outside the timed region; use -log_view for the per-event breakdown
(PCSetUp_GAMG, MatMult, ...).

Label: solve_time_s -> ksp_iter_time_s; run.sh sed and --fom-version
updated to match.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: William Dawson <william.dawson@riken.jp>
---
 programs/petsc-gmres/README.md           | 11 ++++++++++-
 programs/petsc-gmres/run.sh              |  6 +++---
 programs/petsc-gmres/src/GMRES-PETSc.cpp | 24 ++++++++++++++----------
 3 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/programs/petsc-gmres/README.md b/programs/petsc-gmres/README.md
index 71793e3..e1f8dd3 100644
--- a/programs/petsc-gmres/README.md
+++ b/programs/petsc-gmres/README.md
@@ -5,7 +5,16 @@ load the `audikw_1` sparse SPD matrix ([SuiteSparse Matrix
 Collection](https://sparse.tamu.edu/GHS_psdef/audikw_1), `GHS_psdef`
 group — a real structural-engineering FEM problem, 943,695 rows,
 77,651,847 nonzeros), solve `Ax = b` for a known `x`, report the relative
-L2 error and solve wall-time (`FOM: ranks=<N> solve_time_s=<t>`).
+L2 error and GMRES-iterate wall-time (`FOM: ranks=<N> ksp_iter_time_s=<t>`).
+
+The FOM measures the GMRES **iterate only** — `KSPSetUp(ksp)` is called
+explicitly before the timed `KSPSolve`, so GAMG setup (coarsening + PtAP)
+is excluded. The iterate is the MatMult-bound SpMV loop that tracks memory
+bandwidth / Flop/s; setup is communication-bound and is read off
+`-log_view` (`PCSetUp_GAMG`, `MatMult`, ...) instead. (Previously the FOM
+included setup; narrowed after `-log_view` showed setup was ~70% of the
+timed `KSPSolve` on a 4-GPU run — see the internal repo's
+`petsc-benchmarking` skill.)
 
 Source (`src/GMRES-PETSc.cpp`) is vendored directly in this directory
 rather than fetched from a separate repo at build time, since its
diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
index 31fab96..e9a8ee9 100644
--- a/programs/petsc-gmres/run.sh
+++ b/programs/petsc-gmres/run.sh
@@ -92,12 +92,12 @@ if ! grep -q "^FOM: ranks=" "${logfile}" 2>/dev/null; then
   exit 1
 fi
 
-solve_time=$(grep "^FOM: ranks=" "${logfile}" | sed -E 's/.*solve_time_s=([0-9.]+).*/\1/')
+ksp_iter_time=$(grep "^FOM: ranks=" "${logfile}" | sed -E 's/.*ksp_iter_time_s=([0-9.]+).*/\1/')
 
 bk_emit_result \
-  --fom "${solve_time}" \
+  --fom "${ksp_iter_time}" \
   --fom-unit s \
-  --fom-version solve_time \
+  --fom-version ksp_iter_time \
   --exp audikw_1 \
   --nodes "${nodes}" \
   --numproc-node "${numproc_node}" \
diff --git a/programs/petsc-gmres/src/GMRES-PETSc.cpp b/programs/petsc-gmres/src/GMRES-PETSc.cpp
index 23ff9a4..d36af55 100644
--- a/programs/petsc-gmres/src/GMRES-PETSc.cpp
+++ b/programs/petsc-gmres/src/GMRES-PETSc.cpp
@@ -65,21 +65,25 @@ int main(int argc, char **args)
   //     receive options for KSP solver from command line
   PetscCall(KSPSetFromOptions(ksp));
 
-  // Figure of merit: wall-clock time of the actual solve (KSPSetUp, which
-  // builds the GAMG hierarchy, runs lazily inside this KSPSolve call, so
-  // setup + iterate are both included -- this is "how long it took to go
-  // from an assembled matrix to a converged solution", independent of
-  // process-launch/link startup cost, which is not part of the solve).
-  PetscLogDouble solve_t0, solve_t1;
+  // Build the GAMG hierarchy (coarsening + Galerkin/PtAP) explicitly,
+  // OUTSIDE the timed region. The figure of merit is the GMRES iterate --
+  // the MatMult-bound SpMV loop whose cost tracks memory bandwidth / Flop/s
+  // -- not one-shot preconditioner setup, which is communication-bound and
+  // does not characterize the iterative kernel. KSPSolve would otherwise
+  // call KSPSetUp lazily on first use, folding setup into the timed region.
+  // Use -log_view for the per-event breakdown (PCSetUp_GAMG, MatMult, ...).
+  PetscCall(KSPSetUp(ksp));
+
+  PetscLogDouble iter_t0, iter_t1;
   PetscCall(PetscBarrier((PetscObject)ksp));
-  PetscCall(PetscTime(&solve_t0));
+  PetscCall(PetscTime(&iter_t0));
   PetscCall(KSPSolve(ksp, b, x));
   PetscCall(PetscBarrier((PetscObject)ksp));
-  PetscCall(PetscTime(&solve_t1));
+  PetscCall(PetscTime(&iter_t1));
   {
-    PetscReal local_dt = (PetscReal)(solve_t1 - solve_t0), max_dt;
+    PetscReal local_dt = (PetscReal)(iter_t1 - iter_t0), max_dt;
     PetscCallMPI(MPI_Allreduce(&local_dt, &max_dt, 1, MPIU_REAL, MPI_MAX, PETSC_COMM_WORLD));
-    PetscCall(PetscPrintf(PETSC_COMM_WORLD, "FOM: ranks=%d solve_time_s=%.6f\n", (int)size, (double)max_dt));
+    PetscCall(PetscPrintf(PETSC_COMM_WORLD, "FOM: ranks=%d ksp_iter_time_s=%.6f\n", (int)size, (double)max_dt));
   }
 
   PetscCall(VecNorm(u, NORM_2, &e0));

From 0d370ad5372338ea7e512426b53ac178f52c9203 Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Mon, 17 Aug 2026 17:02:06 +0900
Subject: [PATCH 07/14] Fix RIKYU run.sh: add -mat_type aijcusparse and
 per-rank GPU binding [code:petsc-gmres]

Without -mat_type aijcusparse and CUDA_VISIBLE_DEVICES set per rank,
the solve runs entirely on CPU despite GPUs being allocated -- measured
6.125s (4 CPU ranks) vs 0.183s (4 GPU ranks) on the same node. Use the
standard bash -c 'export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK;
exec "$@"' wrapper pattern (same as block1-petsc's own job scripts and
the salmon/sbd skills) so each rank binds a distinct GPU.

Signed-off-by: William Dawson <william.dawson@riken.jp>
---
 programs/petsc-gmres/run.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
index e9a8ee9..1549279 100644
--- a/programs/petsc-gmres/run.sh
+++ b/programs/petsc-gmres/run.sh
@@ -34,8 +34,15 @@ case "${system}" in
   RIKYU)
     DATA=/data1/rkp00015/benchkit-data/petsc-gmres/audikw_1.petscbin
     module load nvhpc-hpcx/26.3
+    # One MPI rank per GPU: each rank gets a distinct GPU via
+    # CUDA_VISIBLE_DEVICES (set inside mpirun so OMPI_COMM_WORLD_LOCAL_RANK
+    # is available per-rank), and -mat_type aijcusparse puts the matrix
+    # on-device. Without these the solve runs on CPU even though GPUs
+    # are allocated (see petsc-benchmarking for the measured difference).
     mpirun -np "${n_ranks}" -N "${numproc_node}" --bind-to core --map-by core \
-      "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
+      bash -c 'export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK; exec "$@"' \
+      _ "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
+      -mat_type aijcusparse \
       > "${logfile}" 2>&1 || true
     ;;
   Fugaku)

From 3587c851292ccf21297aad18642614ed1e8a9825 Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Mon, 17 Aug 2026 17:23:19 +0900
Subject: [PATCH 08/14] Switch to stokes2 matrix (4.26M rows, 256M nnz)
 [code:petsc-gmres]

Replace audikw_1 (943k rows, SPD structural mechanics) with stokes2
(4.26M rows, Stokes flow saddle-point system from Suzuki-san, generated
via Gmsh/FreeFEM). The larger size provides enough rows/GPU for multi-GPU
to win: 4-GPU is 2.05x faster than 1-GPU on Rikyu (vs 1.41x slower on
audikw_1). No mtx2petsc conversion needed -- FreeFEM writes PETSc binary
directly via ObjectView, same format as MatView.

Changes:
- run.sh: DATA paths -> stokes2.dat, add -matload_block_size 1, add
  CUDA_VISIBLE_DEVICES binding to DGX Spark (matching RIKYU), --exp stokes2
- README.md: describe stokes2, update correctness (L2 ~0.25 for Stokes
  saddle-point, not ~0.03 for SPD), update staging table and regeneration

Signed-off-by: William Dawson <william.dawson@riken.jp>
---
 programs/petsc-gmres/README.md | 68 +++++++++++++++++-----------------
 programs/petsc-gmres/run.sh    | 28 +++++++-------
 2 files changed, 48 insertions(+), 48 deletions(-)

diff --git a/programs/petsc-gmres/README.md b/programs/petsc-gmres/README.md
index e1f8dd3..4270d3b 100644
--- a/programs/petsc-gmres/README.md
+++ b/programs/petsc-gmres/README.md
@@ -1,11 +1,11 @@
 # petsc-gmres
 
 A PETSc KSP (GMRES + GAMG algebraic multigrid preconditioner) benchmark:
-load the `audikw_1` sparse SPD matrix ([SuiteSparse Matrix
-Collection](https://sparse.tamu.edu/GHS_psdef/audikw_1), `GHS_psdef`
-group — a real structural-engineering FEM problem, 943,695 rows,
-77,651,847 nonzeros), solve `Ax = b` for a known `x`, report the relative
-L2 error and GMRES-iterate wall-time (`FOM: ranks=<N> ksp_iter_time_s=<t>`).
+load the `stokes2` sparse matrix (4,260,568 rows, 256,285,536 nonzeros —
+a Stokes flow saddle-point system on an ellipsoid mesh, generated via
+Gmsh/FreeFEM by Atsushi Suzuki), solve `Ax = b` for a known `x`, report
+the relative L2 error and GMRES-iterate wall-time (`FOM: ranks=<N>
+ksp_iter_time_s=<t>`).
 
 The FOM measures the GMRES **iterate only** — `KSPSetUp(ksp)` is called
 explicitly before the timed `KSPSolve`, so GAMG setup (coarsening + PtAP)
@@ -34,32 +34,34 @@ but available to RIKEN-RCCS members on request.
 
 ## Correctness
 
-`relative L2 norm of the error` should land near 0.03–0.04 at any rank
-count (GMRES's default relative-residual tolerance, not a tight solve —
-this is a benchmark, not a production accuracy target). A result outside
-that band signals a real bug, not benchmark noise.
+`relative L2 norm of the error` for `stokes2` lands near **0.23–0.26** at
+any rank count — higher than the ~0.03 typical for SPD matrices like the
+earlier `audikw_1`, because `stokes2` is a Stokes flow saddle-point system
+(velocity + pressure), which is indefinite and harder to solve. GMRES
+hits the default iteration limit rather than converging tightly, but the
+iteration count is stable across rank counts (193–232 on GPU), so the
+timing comparison is fair. A result far outside that band signals a real
+bug, not benchmark noise.
 
 ## `-pc_gamg_square_graph 0`
 
 Required on every system, at any rank count above 1 (single-rank runs
 happen not to trigger it, but don't rely on that). Without it, GAMG's
-default aggressive-coarsening graph-squaring step blows up on
-`audikw_1`'s connectivity — a `CUSPARSE_STATUS_INSUFFICIENT_RESOURCES`
-crash on GPU, or a genuine multi-GB single-allocation PETSc "Out of
-memory" abort on CPU. Both `build.sh`'s configure line and `run.sh`'s
-launch command already account for everything needed except this flag,
-which is passed explicitly in `run.sh`.
+default aggressive-coarsening graph-squaring step blows up on the matrix's
+connectivity — a `CUSPARSE_STATUS_INSUFFICIENT_RESOURCES` crash on GPU,
+or a genuine multi-GB single-allocation PETSc "Out of memory" abort on
+CPU. Both `build.sh`'s configure line and `run.sh`'s launch command
+already account for everything needed except this flag, which is passed
+explicitly in `run.sh`.
 
 ## Staging the data
 
-The matrix is pre-converted once (offline, not part of `build.sh`/`run.sh`)
-from MatrixMarket format to PETSc's binary format via a small one-time
-conversion tool (also part of the private upstream repo, not shipped
-here — it's not needed at benchmark build/run time, only to produce the
-staged file below once). This avoids every rank re-parsing a
-multi-hundred-MB text file at load time, which was the actual root cause
-of the original reported "runs out of memory for no reason" bug this
-benchmark exists to catch a regression of.
+The matrix is generated by a FreeFEM script (`StokesPETSc0.edp`) that
+solves a Stokes flow on an ellipsoid mesh and writes the resulting matrix
+directly to PETSc binary format via `ObjectView(A, format="binary")` —
+the same format as PETSc's own `MatView`, so no separate conversion step
+(e.g. `mtx2petsc`) is needed. The `MatLoad` call in `GMRES-PETSc.cpp`
+reads it directly.
 
 Pre-staged locations (same convention as this repo's `ffb` and
 `LQCD_dw_solver`, which pre-stage their own — much larger — source
@@ -68,16 +70,12 @@ build/run time):
 
 | system | path |
 |---|---|
-| RIKYU | `/data1/rkp00015/benchkit-data/petsc-gmres/audikw_1.petscbin` |
-| Fugaku | `/vol0500/data/ra250029/benchkit-data/petsc-gmres/audikw_1.petscbin` — `/vol0002` (the volume the first attempt used) turned out to have a 0-byte hard quota limit for every group tried, `/vol0005` didn't; this app's `FJ` queue.csv template already declares `GFSCACHE` for `/vol0002:/vol0003:/vol0004:/vol0005`, so no per-app `-x` handling was needed once the right volume was found. Use `/vol0500`, not `/vol0005`, in an actual path — `/vol0005` is what a filesystem tool reports as the "resolved" location, but it isn't what resolves from a real compute-node job |
-| RC_DGXSP | `/home/users/william.dawson/benchkit-data/petsc-gmres/audikw_1.petscbin` (this system has no separate group-storage tier, so — unlike RIKYU/Fugaku — this lives under `$HOME`) |
+| RIKYU | `/data1/rkp00015/benchkit-data/petsc-gmres/stokes2.dat` |
+| Fugaku | `/vol0500/data/ra250029/benchkit-data/petsc-gmres/stokes2.dat` — `/vol0002` (the volume the first attempt used) turned out to have a 0-byte hard quota limit for every group tried, `/vol0005` didn't; this app's `FJ` queue.csv template already declares `GFSCACHE` for `/vol0002:/vol0003:/vol0004:/vol0005`, so no per-app `-x` handling was needed once the right volume was found. Use `/vol0500`, not `/vol0005`, in an actual path — `/vol0005` is what a filesystem tool reports as the "resolved" location, but it isn't what resolves from a real compute-node job |
+| RC_DGXSP | `/home/users/william.dawson/benchkit-data/petsc-gmres/stokes2.dat` (this system has no separate group-storage tier, so — unlike RIKYU/Fugaku — this lives under `$HOME`) |
 
-To re-stage on a system with an existing PETSc install: download
-`audikw_1.mtx` from the SuiteSparse Matrix Collection link above, then use
-PETSc's own `MatLoad`/`MatView` binary-viewer round trip (or the private
-repo's `mtx2petsc` tool, if you have access) to write it to PETSc binary
-format. Verify the result: 943,695 × 943,695, nnz = 77,651,847 — the
-standard SuiteSparse download is MatrixMarket `symmetric` format (one
-triangle + diagonal only), so a correct converter must mirror off-diagonal
-entries; a naive read of the file as-is will silently produce a wrong,
-singular matrix with no error.
+To re-stage on a system: copy the `stokes2.dat` file (3,092,468,720 bytes)
+from an existing location above. To regenerate from scratch, run the
+FreeFEM script (`mpirun -np 8 FreeFem++-mpi StokesPETSc0.edp`) which writes
+`A.dat` in PETSc binary format — rename to `stokes2.dat`. Verify: 4,260,568
+× 4,260,568, nnz = 256,285,536.
diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
index 1549279..3d87703 100644
--- a/programs/petsc-gmres/run.sh
+++ b/programs/petsc-gmres/run.sh
@@ -21,18 +21,18 @@ fi
 
 export OMP_NUM_THREADS="${nthreads}"
 
-# The benchmark matrix (audikw_1, SuiteSparse GHS_psdef group, 943,695 x
-# 943,695, 77,651,847 nnz, converted once to PETSc binary format) is
-# pre-staged at a fixed path per system rather than fetched at build/run
-# time -- same convention as this repo's ffb and LQCD_dw_solver, which
-# pre-stage their (much larger) source archives the same way. See
-# README.md for exactly how each copy was produced and how to re-stage it.
+# The benchmark matrix (stokes2, 4,260,568 x 4,260,568, 256,285,536 nnz --
+# a Stokes flow saddle-point system from an ellipsoid mesh, generated via
+# Gmsh/FreeFEM and written directly to PETSc binary format by FreeFEM's
+# ObjectView, so no mtx2petsc conversion is needed) is pre-staged at a
+# fixed path per system. See README.md for how each copy was produced and
+# how to re-stage it.
 logfile="solve.log"
 touch .run_marker
 
 case "${system}" in
   RIKYU)
-    DATA=/data1/rkp00015/benchkit-data/petsc-gmres/audikw_1.petscbin
+    DATA=/data1/rkp00015/benchkit-data/petsc-gmres/stokes2.dat
     module load nvhpc-hpcx/26.3
     # One MPI rank per GPU: each rank gets a distinct GPU via
     # CUDA_VISIBLE_DEVICES (set inside mpirun so OMPI_COMM_WORLD_LOCAL_RANK
@@ -42,7 +42,7 @@ case "${system}" in
     mpirun -np "${n_ranks}" -N "${numproc_node}" --bind-to core --map-by core \
       bash -c 'export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK; exec "$@"' \
       _ "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
-      -mat_type aijcusparse \
+      -mat_type aijcusparse -matload_block_size 1 \
       > "${logfile}" 2>&1 || true
     ;;
   Fugaku)
@@ -54,11 +54,12 @@ case "${system}" in
     # this was staged) is what actually resolves from a compute-node job;
     # found by testing the real run.sh in a real job, not by trusting the
     # canonical-looking path a filesystem tool reported.
-    DATA=/vol0500/data/ra250029/benchkit-data/petsc-gmres/audikw_1.petscbin
+    DATA=/vol0500/data/ra250029/benchkit-data/petsc-gmres/stokes2.dat
     module load lang/tcsds-1.2.43
     module load LLVM/llvmorg-22.1.0
     mpiexec -n "${n_ranks}" \
       "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
+      -matload_block_size 1 \
       > "${logfile}" 2>&1 || true
     # Fugaku's PJM mpiexec writes each rank's real stdout/stderr under
     # ./output.$PJM_JOBID/, ignoring plain shell redirection for the
@@ -73,12 +74,13 @@ case "${system}" in
     # GPU run (1 rank/GPU) -- see build.sh. This system has no separate
     # group-storage tier (see README.md), so the data lives under $HOME
     # like everything else here.
-    DATA=/home/users/william.dawson/benchkit-data/petsc-gmres/audikw_1.petscbin
+    DATA=/home/users/william.dawson/benchkit-data/petsc-gmres/stokes2.dat
     source /etc/profile.d/modules.sh
     module load system/ng-dgx nvhpc-hpcx
     mpirun -np "${n_ranks}" \
-      "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
-      -mat_type aijcusparse \
+      bash -c 'export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK; exec "$@"' \
+      _ "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
+      -mat_type aijcusparse -matload_block_size 1 \
       > "${logfile}" 2>&1 || true
     ;;
   *)
@@ -105,7 +107,7 @@ bk_emit_result \
   --fom "${ksp_iter_time}" \
   --fom-unit s \
   --fom-version ksp_iter_time \
-  --exp audikw_1 \
+  --exp stokes2 \
   --nodes "${nodes}" \
   --numproc-node "${numproc_node}" \
   --nthreads "${nthreads}" >> "${RESULTS_DIR}/result"

From 201367ebcc2c54ee84ce2cda484253047fe30ecd Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Tue, 25 Aug 2026 18:07:00 +0900
Subject: [PATCH 09/14] Build RIKYU PETSc with CUDA so run.sh's aijcusparse
 works [code:petsc-gmres]

run.sh has launched RIKYU with -mat_type aijcusparse since the GPU-binding
fix, but build.sh still configured PETSc --with-cuda=0. A CPU-only PETSc
does not register that Mat type, so the shipped build/run pair could not
have worked: the old artifact fails with "Unknown Mat type given:
aijcusparse" (PETSc error 86) at MatSetFromOptions. Every GPU and stokes2
number reported on this PR came from the block1-eeas job scripts, not from
BenchKit's own build.sh/run.sh.

Configure --with-cuda=1 against the loaded nvhpc-hpcx CUDA toolkit,
cuda-arch 100 (sm_100, B200). The arch is hardcoded rather than probed
with nvidia-smi as RC_DGXSP does, because RIKYU is a cross-mode system
whose build runner is a login node where a GPU is not guaranteed visible.
PETSC_ARCH becomes arch-rikyu-cuda so a stale CPU-only arch-rikyu tree is
never silently reused.

Verified end to end on Rikyu off this branch:
- bash programs/petsc-gmres/build.sh RIKYU: exit 0, 4m55s; artifact links
  libcusparse/libcublas/libcudart; PETSC_HAVE_CUDA 1.
- bash scripts/test_submit.sh petsc-gmres 1: exit 0, results/result gets
  FOM:0.680068 FOM_unit:s FOM_version:ksp_iter_time Exp:stokes2
  node_count:1 numproc_node:4 nthreads:1, relative L2 0.227831 --
  matching the 4-GPU stokes2 figures measured outside BenchKit (0.683s,
  L2 0.2278).
- -log_view on the same binary reports GPU %F = 100 for MatMult and
  KSPSolve, banner "arch-rikyu-cuda ... CUDA architecture 100", so the
  solve is genuinely on device rather than silently falling back.

Signed-off-by: William Dawson <william.dawson@riken.jp>
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PEzS2ybD8VEnQa9Sq5qYUM
---
 programs/petsc-gmres/build.sh | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/programs/petsc-gmres/build.sh b/programs/petsc-gmres/build.sh
index 6573a35..685deef 100644
--- a/programs/petsc-gmres/build.sh
+++ b/programs/petsc-gmres/build.sh
@@ -25,23 +25,26 @@ bk_fetch_source "${PETSC_REPO}" "petsc" "${PETSC_TAG}"
 case "$system" in
   RIKYU)
     module load nvhpc-hpcx/26.3
-    export PETSC_ARCH=arch-rikyu
-    NVPL=/shared/software/hpc_sdk/Linux_aarch64/26.3/math_libs/nvpl/lib
+    export PETSC_ARCH=arch-rikyu-cuda
+    NVHPC=/shared/software/hpc_sdk/Linux_aarch64/26.3
+    NVPL="${NVHPC}/math_libs/nvpl/lib"
     (
       cd "${PETSC_DIR}"
       # --with-fc=0: nvhpc-hpcx's Fortran wrapper doesn't support an F2018
       # pointer-initialization feature PETSc's Fortran bindings need; this
-      # app is C++ only anyway. --with-cuda=0: this is the CPU-only build
-      # (see petsc-benchmarking in RIKEN-RCCS/block1-eeas for why a
-      # +cuda-enabled build's PetscInitialize scales ~linearly with rank
-      # count from CUDA-library dynamic-linking contention, unrelated to
-      # solve performance). Serial NVPL BLAS/LAPACK: this app runs flat
-      # MPI, no OpenMP threading.
+      # app is C++ only anyway. --with-cuda=1: run.sh launches this system
+      # with -mat_type aijcusparse, which a CPU-only PETSc does not
+      # register -- the two must agree. cuda-arch 100 is sm_100 (B200);
+      # hardcoded rather than probed with nvidia-smi (as RC_DGXSP does)
+      # because RIKYU is a cross-mode system whose build runner is a login
+      # node, so a GPU is not guaranteed to be visible at build time.
+      # Serial NVPL BLAS/LAPACK: this app runs flat MPI, no OpenMP.
       ./configure \
         --with-cc=mpicc --with-cxx=mpicxx --with-fc=0 \
-        --with-debugging=0 --with-cuda=0 \
+        --with-debugging=0 \
+        --with-cuda=1 --with-cuda-dir="${NVHPC}/cuda/13.1" --with-cuda-arch=100 \
         --with-blaslapack-lib="-L${NVPL} -Wl,-rpath,${NVPL} -lnvpl_lapack_lp64_seq -lnvpl_blas_lp64_seq" \
-        COPTFLAGS='-O3' CXXOPTFLAGS='-O3'
+        COPTFLAGS='-O3' CXXOPTFLAGS='-O3' CUDAOPTFLAGS='-O3'
       make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" -j8 all
     )
     mpicxx -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \

From 3818f175c537e990b0d29b2bd3aacd4aca79b86e Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Tue, 25 Aug 2026 22:11:13 +0900
Subject: [PATCH 10/14] Fix 2-node GPU hang: CUDA context before MPI_Init
 [code:petsc-gmres]

Any run spanning 2+ nodes on GPU hung indefinitely, 100% reproducibly. A
live gdb backtrace on the spinning ranks showed them split across two
different collectives:

  node 1: main -> MatMult -> VecScatterEnd -> PetscSFBcastEnd
          -> MPI_Waitall -> ucp_worker_progress   [spinning]
  node 2: MatGetInfo -> MPIU_Allreduce -> MPI_Allreduce
          -> ompi_coll_base_allreduce_intra_recursivedoubling

CPUs at 98% and GPUs at 0% utilisation, stuck at the first inter-node
GPU-buffer exchange -- the MatMult that forms the RHS, before KSP starts.

Cause: PetscInitialize calls MPI_Init before any CUDA context exists, so
UCX/UCC probe for CUDA support while bootstrapping MPI_COMM_WORLD, find
none, and cache a rendezvous protocol that deadlocks on GPU buffers. This
is the same failure mode SALMON hit on this machine, fixed there by
calling acc_init before MPI_Init_thread.

Establish the context with cudaSetDevice/cudaFree ahead of
PetscInitialize. Guarded on PETSC_HAVE_CUDA so Fugaku's CPU-only build is
untouched, and warns rather than aborts when no GPU is present so a CUDA
build still runs on a CPU node. build.sh gains the CUDA include path and
-lcudart on the app link, which it did not need before.

Verified on Rikyu off this branch:
- unfixed binary, 8 GPU / 2 nodes: still hanging at a 180 s timeout.
- fixed binary, same matrix and flags: exit 0 in 13 s, 200 iterations.
- bash programs/petsc-gmres/build.sh RIKYU: BUILD_OK, artifact links
  libcudart.so.13 and libcusparse.so.12.
- bash scripts/test_submit.sh petsc-gmres 4 (2 nodes, 8 GPUs): FOM
  1.024436 s, against 1.296136 s at 4 GPUs on 1 node.

This changes a recorded conclusion: 8 GPU / 2 nodes was previously
measured as *slower* than 4 GPU / 1 node, which is what the hang looks
like from the outside once a run does complete. Multi-node GPU scaling
needs re-measuring against this build.

Signed-off-by: William Dawson <william.dawson@riken.jp>
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PEzS2ybD8VEnQa9Sq5qYUM
---
 programs/petsc-gmres/build.sh            |  9 +++++++--
 programs/petsc-gmres/src/GMRES-PETSc.cpp | 24 ++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/programs/petsc-gmres/build.sh b/programs/petsc-gmres/build.sh
index 685deef..e78e5f2 100644
--- a/programs/petsc-gmres/build.sh
+++ b/programs/petsc-gmres/build.sh
@@ -48,9 +48,11 @@ case "$system" in
       make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" -j8 all
     )
     mpicxx -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \
+      -I"${NVHPC}/cuda/13.1/include" \
       "${APP_DIR}/src/GMRES-PETSc.cpp" -o "${ARTIFACT_DIR}/GMRES-PETSc" \
       -Xlinker -rpath="${PETSC_DIR}/${PETSC_ARCH}/lib" \
-      -L"${PETSC_DIR}/${PETSC_ARCH}/lib" -lpetsc
+      -L"${PETSC_DIR}/${PETSC_ARCH}/lib" -lpetsc \
+      -L"${NVHPC}/cuda/13.1/lib64" -Xlinker -rpath="${NVHPC}/cuda/13.1/lib64" -lcudart
     ;;
   Fugaku)
     # See petsc-build in RIKEN-RCCS/block1-eeas: LLVM cross-compiler
@@ -110,10 +112,13 @@ case "$system" in
         COPTFLAGS='-O3' CXXOPTFLAGS='-O3'
       make PETSC_DIR="${PETSC_DIR}" PETSC_ARCH="${PETSC_ARCH}" -j8 all
     )
+    CUDA_INC=$(dirname "$(command -v nvcc)")/../include
     mpicxx -O3 -I"${PETSC_DIR}/include" -I"${PETSC_DIR}/${PETSC_ARCH}/include" \
+      -I"${CUDA_INC}" \
       "${APP_DIR}/src/GMRES-PETSc.cpp" -o "${ARTIFACT_DIR}/GMRES-PETSc" \
       -Xlinker -rpath="${PETSC_DIR}/${PETSC_ARCH}/lib" \
-      -L"${PETSC_DIR}/${PETSC_ARCH}/lib" -lpetsc
+      -L"${PETSC_DIR}/${PETSC_ARCH}/lib" -lpetsc \
+      -L"${CUDA_INC}/../lib64" -Xlinker -rpath="${CUDA_INC}/../lib64" -lcudart
     ;;
   *)
     echo "Unknown system: $system" >&2
diff --git a/programs/petsc-gmres/src/GMRES-PETSc.cpp b/programs/petsc-gmres/src/GMRES-PETSc.cpp
index d36af55..531333b 100644
--- a/programs/petsc-gmres/src/GMRES-PETSc.cpp
+++ b/programs/petsc-gmres/src/GMRES-PETSc.cpp
@@ -15,6 +15,10 @@ static char help[] = "load a PETSc binary matrix and call the KSP solver\n";
 
 #include <petscksp.h>
 
+#if defined(PETSC_HAVE_CUDA)
+  #include <cuda_runtime.h>
+#endif
+
 int main(int argc, char **args)
 {
   Vec         x, b, u;        /* approx solution, RHS, exact solution */
@@ -30,6 +34,26 @@ int main(int argc, char **args)
   PetscViewer viewer;
 
   PetscFunctionBeginUser;
+#if defined(PETSC_HAVE_CUDA)
+  // Establish this rank's CUDA context BEFORE PetscInitialize calls MPI_Init.
+  // UCX/UCC probe for CUDA support while bootstrapping MPI_COMM_WORLD; with no
+  // context yet, they cache a rendezvous protocol that deadlocks on the first
+  // inter-node GPU-buffer transfer, hanging any 2+ node GPU run in MatMult's
+  // halo exchange with the GPUs idle. run.sh gives each rank one visible GPU,
+  // so device 0 is this rank's GPU.
+  {
+    cudaError_t cerr = cudaSetDevice(0);
+    if (cerr != cudaSuccess) {
+      fprintf(stderr, "warning: cudaSetDevice(0) failed (%s); no CUDA context "
+                      "before MPI_Init -- multi-node GPU runs may hang\n",
+              cudaGetErrorString(cerr));
+    } else if ((cerr = cudaFree(0)) != cudaSuccess) { /* forces context creation */
+      fprintf(stderr, "warning: cudaFree(0) failed (%s); no CUDA context "
+                      "before MPI_Init -- multi-node GPU runs may hang\n",
+              cudaGetErrorString(cerr));
+    }
+  }
+#endif
   PetscCall(PetscInitialize(&argc, &args, (char *)0, help));
   PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
   PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));

From 9a6d75de537578eb9b070b789245baca3d0d5e2e Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Tue, 25 Aug 2026 22:11:34 +0900
Subject: [PATCH 11/14] Add per-routine FOM breakdown, GMRES restart, matrix
 override [code:petsc-gmres]

Per-routine breakdown (A. Suzuki): capture PETSc's event log with
-log_view and parse MatMult / KSPSolve time, flop and flop/s plus the SF
pack+unpack time with FOM.awk. GPU systems also pass -log_view_gpu_time,
without which MatMult's time and every GPU Mflop/s column read "n/a" --
most of what the breakdown exists to report. The table ships as
results/fom_details.txt and is mirrored into SECTION lines, BenchKit's
only structured channel for sub-timings. A time that is not numeric is
skipped with a warning rather than passed to bk_emit_section, which would
reject it and, under set -e, abort before the FOM was ever emitted.

Three fixes to the original scripts: BEING{ -> BEGIN{ in FOM.awk (awk read
BEING as an always-false variable pattern, so the initialisers never ran
and a missing event printed blank instead of 0.0); "ksp_details = $(...)"
-> no spaces, which was a bash syntax error that aborted every run; and
${logetsc} -> ${logpetsc}.

GMRES restart: PETSc's default of 30 is too small for these Stokes
systems. stokes3 and stokes4 stagnate at it and never converge, hitting
the 10000-iteration cap with a true residual of 1.8e-03. At restart 100
stokes3 converges in 285 iterations and less wall time than the
stagnating run. Overridable via BK_PETSC_GMRES_RESTART. These operators
are singular, so no restart guarantees convergence and larger is not
monotonically better -- restart 80 converged and then broke down on a
rerun of the identical configuration.

BK_PETSC_GMRES_MATRIX selects the matrix without editing this script, and
--exp now follows the file name so results are labelled for the matrix
actually solved. list.csv covers 1, 2 and 4 GPUs on one node plus 8 GPUs
across two.

Verified on Rikyu through the shipped scripts: test_submit.sh lines 3 and
4 both exit 0 and emit FOM plus the three SECTION lines, with
fom_details.txt populated in every column.

Signed-off-by: William Dawson <william.dawson@riken.jp>
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PEzS2ybD8VEnQa9Sq5qYUM
---
 programs/petsc-gmres/FOM.awk  | 31 +++++++++++++++++++
 programs/petsc-gmres/list.csv |  5 ++-
 programs/petsc-gmres/run.sh   | 57 ++++++++++++++++++++++++++++++++---
 3 files changed, 88 insertions(+), 5 deletions(-)
 create mode 100644 programs/petsc-gmres/FOM.awk

diff --git a/programs/petsc-gmres/FOM.awk b/programs/petsc-gmres/FOM.awk
new file mode 100644
index 0000000..14ea4cb
--- /dev/null
+++ b/programs/petsc-gmres/FOM.awk
@@ -0,0 +1,31 @@
+#!/usr/bin/awk -f
+# Extract the per-routine detail behind the FOM from a PETSc -log_view file.
+# Column positions are -log_view's fixed layout: $4 time(max), $6 flop(max),
+# $21 total Mflop/s, $22 GPU Mflop/s. $4/$22 read "n/a" unless the run also
+# passed -log_view_gpu_time, so run.sh passes it on every GPU system.
+# Original by A. Suzuki, 25 Aug. 2026.
+BEGIN{
+   timeMatMult = 0.0; flopMatMult = 0.0; totalflopsMatMult = 0.0; gpuflopsMatMult = 0.0;
+   timeKSPSolve = 0.0; flopKSPSolve = 0.0; totalflopsKSPSolve = 0.0; gpuflopsKSPSolve = 0.0;
+   timeSFPack = 0.0;
+   timeSFUnpack = 0.0;
+}
+# space is mandatory to exclude MatMult{Add,Transpose}
+/^MatMult /{
+   timeMatMult = $4; flopMatMult = $6; totalflopsMatMult = $21; gpuflopsMatMult = $22;
+}
+/^KSPSolve/{
+   timeKSPSolve = $4; flopKSPSolve = $6; totalflopsKSPSolve = $21; gpuflopsKSPSolve = $22;
+}
+/^SFPack/{
+   timeSFPack = $4;
+}
+/^SFUnpack/{
+   timeSFUnpack = $4;
+}
+END{
+   printf("#routine     \ttime(sec) \tflop      flop/s(total) flop/s(GPU)\n");
+   printf("MatMult      \t%s \t%s \t%s \t%s\n", timeMatMult, flopMatMult, totalflopsMatMult, gpuflopsMatMult);
+   printf("KSPSolve     \t%s \t%s \t%s \t%s\n", timeKSPSolve, flopKSPSolve, totalflopsKSPSolve, gpuflopsKSPSolve);
+   printf("SFPack/Unpack\t%s\n", timeSFPack + timeSFUnpack);
+}
diff --git a/programs/petsc-gmres/list.csv b/programs/petsc-gmres/list.csv
index 5b9095c..f3fe3ce 100644
--- a/programs/petsc-gmres/list.csv
+++ b/programs/petsc-gmres/list.csv
@@ -1,4 +1,7 @@
 system,enable,nodes,numproc_node,nthreads,elapse
-RIKYU,yes,1,4,1,0:15:00
+RIKYU,yes,1,1,1,0:30:00
+RIKYU,yes,1,2,1,0:30:00
+RIKYU,yes,1,4,1,0:30:00
+RIKYU,yes,2,4,1,0:30:00
 Fugaku,yes,1,48,1,0:15:00
 RC_DGXSP,yes,1,1,1,0:15:00
diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
index 3d87703..72cd22b 100644
--- a/programs/petsc-gmres/run.sh
+++ b/programs/petsc-gmres/run.sh
@@ -9,6 +9,11 @@ n_ranks=$((nodes * numproc_node))
 
 source scripts/bk_functions.sh
 
+# FOM.awk ships next to this script and BenchKit invokes run.sh from the
+# repo root, so anchor it to the script's own location -- same reason
+# build.sh anchors src/GMRES-PETSc.cpp.
+APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 ARTIFACT="${PWD}/artifacts/GMRES-PETSc"
 RESULTS_DIR="${PWD}/results"
 mkdir -p "${RESULTS_DIR}"
@@ -28,11 +33,21 @@ export OMP_NUM_THREADS="${nthreads}"
 # fixed path per system. See README.md for how each copy was produced and
 # how to re-stage it.
 logfile="solve.log"
+# PETSc's own event log, parsed by FOM.awk for the per-routine breakdown.
+logpetsc="petsc.log"
+
+# GMRES restart (Krylov subspace size). PETSc's default of 30 is too small for
+# these Stokes systems: stokes3/stokes4 stagnate at it and never converge.
+# 100 converges stokes2 at every rank count and stokes3 at 2 and 4 GPUs. The
+# operators are singular, so no restart guarantees convergence and larger is
+# not monotonically better -- 80 converged and then broke down on a rerun of
+# the same config.
+KSP_RESTART="${BK_PETSC_GMRES_RESTART:-100}"
 touch .run_marker
 
 case "${system}" in
   RIKYU)
-    DATA=/data1/rkp00015/benchkit-data/petsc-gmres/stokes2.dat
+    DATA="${BK_PETSC_GMRES_MATRIX:-/data1/rkp00015/benchkit-data/petsc-gmres/stokes2.dat}"
     module load nvhpc-hpcx/26.3
     # One MPI rank per GPU: each rank gets a distinct GPU via
     # CUDA_VISIBLE_DEVICES (set inside mpirun so OMPI_COMM_WORLD_LOCAL_RANK
@@ -43,6 +58,8 @@ case "${system}" in
       bash -c 'export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK; exec "$@"' \
       _ "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
       -mat_type aijcusparse -matload_block_size 1 \
+      -ksp_gmres_restart "${KSP_RESTART}" \
+      -log_view ":${logpetsc}" -log_view_gpu_time \
       > "${logfile}" 2>&1 || true
     ;;
   Fugaku)
@@ -54,12 +71,14 @@ case "${system}" in
     # this was staged) is what actually resolves from a compute-node job;
     # found by testing the real run.sh in a real job, not by trusting the
     # canonical-looking path a filesystem tool reported.
-    DATA=/vol0500/data/ra250029/benchkit-data/petsc-gmres/stokes2.dat
+    DATA="${BK_PETSC_GMRES_MATRIX:-/vol0500/data/ra250029/benchkit-data/petsc-gmres/stokes2.dat}"
     module load lang/tcsds-1.2.43
     module load LLVM/llvmorg-22.1.0
     mpiexec -n "${n_ranks}" \
       "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
       -matload_block_size 1 \
+      -ksp_gmres_restart "${KSP_RESTART}" \
+      -log_view ":${logpetsc}" \
       > "${logfile}" 2>&1 || true
     # Fugaku's PJM mpiexec writes each rank's real stdout/stderr under
     # ./output.$PJM_JOBID/, ignoring plain shell redirection for the
@@ -74,13 +93,15 @@ case "${system}" in
     # GPU run (1 rank/GPU) -- see build.sh. This system has no separate
     # group-storage tier (see README.md), so the data lives under $HOME
     # like everything else here.
-    DATA=/home/users/william.dawson/benchkit-data/petsc-gmres/stokes2.dat
+    DATA="${BK_PETSC_GMRES_MATRIX:-/home/users/william.dawson/benchkit-data/petsc-gmres/stokes2.dat}"
     source /etc/profile.d/modules.sh
     module load system/ng-dgx nvhpc-hpcx
     mpirun -np "${n_ranks}" \
       bash -c 'export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK; exec "$@"' \
       _ "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \
       -mat_type aijcusparse -matload_block_size 1 \
+      -ksp_gmres_restart "${KSP_RESTART}" \
+      -log_view ":${logpetsc}" -log_view_gpu_time \
       > "${logfile}" 2>&1 || true
     ;;
   *)
@@ -103,11 +124,39 @@ fi
 
 ksp_iter_time=$(grep "^FOM: ranks=" "${logfile}" | sed -E 's/.*ksp_iter_time_s=([0-9.]+).*/\1/')
 
+# Per-routine breakdown behind the FOM: MatMult and KSPSolve time, flop and
+# flop/s, plus the SF pack+unpack time that stands for halo-exchange cost.
+# Kept as a collected artifact and mirrored into SECTION lines, the only
+# structured channel BenchKit has for sub-timings.
+if [[ -f "${logpetsc}" ]]; then
+  awk -f "${APP_DIR}/FOM.awk" "${logpetsc}" > "${RESULTS_DIR}/fom_details.txt"
+  cat "${RESULTS_DIR}/fom_details.txt"
+  # A time reads "n/a" when -log_view could not attribute wall time to the
+  # event. bk_emit_section rejects a non-numeric time, and under set -e that
+  # would abort before the FOM is ever emitted -- warn and skip instead of
+  # losing the result.
+  awk -f "${APP_DIR}/FOM.awk" "${logpetsc}" | awk '
+    function num(v) { return v ~ /^[0-9]+([.][0-9]+)?([eE][-+]?[0-9]+)?$/ }
+    $1 == "MatMult"       { print (num($2) ? "MatMult "         $2 : "!MatMult") }
+    $1 == "KSPSolve"      { print (num($2) ? "KSPSolve "        $2 : "!KSPSolve") }
+    $1 == "SFPack/Unpack" { print (num($2) ? "SFPack_SFUnpack " $2 : "!SFPack_SFUnpack") }' |
+  while read -r _sec _time; do
+    if [[ "${_sec}" == !* ]]; then
+      echo "warning: no numeric time for ${_sec#!} in ${logpetsc}, section not emitted" >&2
+      continue
+    fi
+    bk_emit_section "${_sec}" "${_time}" "" "results/fom_details.txt" \
+      >> "${RESULTS_DIR}/result"
+  done
+else
+  echo "warning: ${logpetsc} not produced, skipping FOM detail breakdown" >&2
+fi
+
 bk_emit_result \
   --fom "${ksp_iter_time}" \
   --fom-unit s \
   --fom-version ksp_iter_time \
-  --exp stokes2 \
+  --exp "$(basename "${DATA}" .dat)" \
   --nodes "${nodes}" \
   --numproc-node "${numproc_node}" \
   --nthreads "${nthreads}" >> "${RESULTS_DIR}/result"

From 47230c4c8aeef00956b6a5cf46fc1db5da0f7e35 Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Wed, 26 Aug 2026 07:08:34 +0900
Subject: [PATCH 12/14] Emit FOM before SECTION lines so the breakdown survives
 result.sh [code:petsc-gmres]

scripts/result.sh attaches SECTION lines to the FOM block currently open
and resets the accumulator on every FOM line, so sections written before
their FOM are dropped. run.sh emitted them first, which silently discarded
the entire per-routine breakdown: result.sh still exited 0 and produced
valid Result JSON, just with no fom_breakdown key. results/result looked
correct on inspection, so nothing short of running the full pipeline
surfaced it.

Found by running build.sh -> test_submit.sh -> result.sh end to end from a
fresh clone of this branch. After the reorder, result0.json carries
fom_breakdown.sections with MatMult, KSPSolve and SFPack_SFUnpack, their
times, and the results/fom_details.txt artifact reference on each.

Signed-off-by: William Dawson <william.dawson@riken.jp>
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PEzS2ybD8VEnQa9Sq5qYUM
---
 programs/petsc-gmres/run.sh | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
index 72cd22b..97ae9b5 100644
--- a/programs/petsc-gmres/run.sh
+++ b/programs/petsc-gmres/run.sh
@@ -124,6 +124,19 @@ fi
 
 ksp_iter_time=$(grep "^FOM: ranks=" "${logfile}" | sed -E 's/.*ksp_iter_time_s=([0-9.]+).*/\1/')
 
+# bk_emit_result must come BEFORE the SECTION lines: scripts/result.sh
+# attaches sections to the FOM block currently open and resets them on each
+# FOM line, so sections emitted first are silently dropped from the Result
+# JSON while result.sh still exits 0.
+bk_emit_result \
+  --fom "${ksp_iter_time}" \
+  --fom-unit s \
+  --fom-version ksp_iter_time \
+  --exp "$(basename "${DATA}" .dat)" \
+  --nodes "${nodes}" \
+  --numproc-node "${numproc_node}" \
+  --nthreads "${nthreads}" >> "${RESULTS_DIR}/result"
+
 # Per-routine breakdown behind the FOM: MatMult and KSPSolve time, flop and
 # flop/s, plus the SF pack+unpack time that stands for halo-exchange cost.
 # Kept as a collected artifact and mirrored into SECTION lines, the only
@@ -132,9 +145,9 @@ if [[ -f "${logpetsc}" ]]; then
   awk -f "${APP_DIR}/FOM.awk" "${logpetsc}" > "${RESULTS_DIR}/fom_details.txt"
   cat "${RESULTS_DIR}/fom_details.txt"
   # A time reads "n/a" when -log_view could not attribute wall time to the
-  # event. bk_emit_section rejects a non-numeric time, and under set -e that
-  # would abort before the FOM is ever emitted -- warn and skip instead of
-  # losing the result.
+  # event. bk_emit_section rejects a non-numeric time, which under set -e
+  # would abort the run and report failure for an otherwise good result --
+  # warn and skip instead.
   awk -f "${APP_DIR}/FOM.awk" "${logpetsc}" | awk '
     function num(v) { return v ~ /^[0-9]+([.][0-9]+)?([eE][-+]?[0-9]+)?$/ }
     $1 == "MatMult"       { print (num($2) ? "MatMult "         $2 : "!MatMult") }
@@ -151,12 +164,3 @@ if [[ -f "${logpetsc}" ]]; then
 else
   echo "warning: ${logpetsc} not produced, skipping FOM detail breakdown" >&2
 fi
-
-bk_emit_result \
-  --fom "${ksp_iter_time}" \
-  --fom-unit s \
-  --fom-version ksp_iter_time \
-  --exp "$(basename "${DATA}" .dat)" \
-  --nodes "${nodes}" \
-  --numproc-node "${numproc_node}" \
-  --nthreads "${nthreads}" >> "${RESULTS_DIR}/result"

From 712404c6800ce45c0c10bd1ca35bed09cc689609 Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Wed, 26 Aug 2026 09:45:04 +0900
Subject: [PATCH 13/14] Simplify comments and drop internal-repo references
 [code:petsc-gmres]

The comments and README pointed at skills in a separate, private
repository that a reader of this repo cannot open, and in places
described how a problem was diagnosed rather than what the code does.
Rewrite them to be shorter and self-contained: keep the reason a
workaround exists, drop the trail that found it and the pointers to
material nobody here can read.

No functional change.

Signed-off-by: William Dawson <william.dawson@riken.jp>
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PEzS2ybD8VEnQa9Sq5qYUM
---
 programs/petsc-gmres/README.md | 19 +++++--------------
 programs/petsc-gmres/build.sh  | 14 ++++++--------
 programs/petsc-gmres/run.sh    |  2 +-
 3 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/programs/petsc-gmres/README.md b/programs/petsc-gmres/README.md
index 4270d3b..7e4ed8c 100644
--- a/programs/petsc-gmres/README.md
+++ b/programs/petsc-gmres/README.md
@@ -12,25 +12,16 @@ explicitly before the timed `KSPSolve`, so GAMG setup (coarsening + PtAP)
 is excluded. The iterate is the MatMult-bound SpMV loop that tracks memory
 bandwidth / Flop/s; setup is communication-bound and is read off
 `-log_view` (`PCSetUp_GAMG`, `MatMult`, ...) instead. (Previously the FOM
-included setup; narrowed after `-log_view` showed setup was ~70% of the
-timed `KSPSolve` on a 4-GPU run — see the internal repo's
-`petsc-benchmarking` skill.)
+included setup; it was narrowed after `-log_view` showed setup was ~70% of
+the timed `KSPSolve` on a 4-GPU run.)
 
 Source (`src/GMRES-PETSc.cpp`) is vendored directly in this directory
-rather than fetched from a separate repo at build time, since its
-upstream development happens in a private RIKEN-RCCS repository. PETSc
-itself is fetched normally via `bk_fetch_source` from its official repo,
-pinned to `v3.25.2`.
+rather than fetched at build time. PETSc itself is fetched via
+`bk_fetch_source` from its official repo, pinned to `v3.25.2`.
 
 Confirmed working on Rikyu (GB200 NVL4), Fugaku (A64FX), and R-CCS
 Cloud's DGX Spark (`RC_DGXSP`, GB10 Blackwell) — see this app's `build.sh`
-for the per-system recipe. Full strong-scaling results, root-cause
-diagnoses for a couple of real bugs found getting each port working
-(a matrix-distribution bug causing OOM, a GAMG coarsening crash on this
-matrix's connectivity, an MPI-transport gotcha on one system), and the
-underlying build recipes are documented in more depth in that same
-internal repository — not linked here since it isn't publicly readable,
-but available to RIKEN-RCCS members on request.
+for the per-system recipe.
 
 ## Correctness
 
diff --git a/programs/petsc-gmres/build.sh b/programs/petsc-gmres/build.sh
index e78e5f2..85292ed 100644
--- a/programs/petsc-gmres/build.sh
+++ b/programs/petsc-gmres/build.sh
@@ -55,10 +55,10 @@ case "$system" in
       -L"${NVHPC}/cuda/13.1/lib64" -Xlinker -rpath="${NVHPC}/cuda/13.1/lib64" -lcudart
     ;;
   Fugaku)
-    # See petsc-build in RIKEN-RCCS/block1-eeas: LLVM cross-compiler
-    # (mpiclang++) beats the Fujitsu compiler here, and Fugaku's own
-    # Spack-provided PETSc hits the same GAMG square-graph crash as a bare
-    # SIGKILL instead of a catchable PETSc error, so we build our own.
+    # The LLVM cross-compiler (mpiclang++) measured faster than the
+    # Fujitsu compiler here, and Fugaku's Spack-provided PETSc hits the
+    # GAMG square-graph crash as a bare SIGKILL rather than a catchable
+    # PETSc error, so build our own.
     module load lang/tcsds-1.2.43
     module load LLVM/llvmorg-22.1.0
     export PETSC_ARCH=arch-fugaku-llvm
@@ -93,10 +93,8 @@ case "$system" in
       $PETSCLIB
     ;;
   RC_DGXSP)
-    # GPU build -- audikw_1 is solved on-GPU here (1 rank/GPU), matching
-    # the cross-machine reproduction in RIKEN-RCCS/block1-eeas's
-    # petsc-build. -pc_gamg_square_graph 0 (set in run.sh) avoids a
-    # cuSPARSE crash on this matrix's connectivity during GAMG's
+    # GPU build, 1 rank/GPU. -pc_gamg_square_graph 0 (set in run.sh)
+    # avoids a cuSPARSE crash on this matrix's connectivity during GAMG's
     # aggressive-coarsening graph-squaring step.
     source /etc/profile.d/modules.sh
     module load system/ng-dgx nvhpc-hpcx
diff --git a/programs/petsc-gmres/run.sh b/programs/petsc-gmres/run.sh
index 97ae9b5..4fa3ee6 100644
--- a/programs/petsc-gmres/run.sh
+++ b/programs/petsc-gmres/run.sh
@@ -53,7 +53,7 @@ case "${system}" in
     # CUDA_VISIBLE_DEVICES (set inside mpirun so OMPI_COMM_WORLD_LOCAL_RANK
     # is available per-rank), and -mat_type aijcusparse puts the matrix
     # on-device. Without these the solve runs on CPU even though GPUs
-    # are allocated (see petsc-benchmarking for the measured difference).
+    # are allocated.
     mpirun -np "${n_ranks}" -N "${numproc_node}" --bind-to core --map-by core \
       bash -c 'export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK; exec "$@"' \
       _ "${ARTIFACT}" -f "${DATA}" -pc_type gamg -pc_gamg_square_graph 0 \

From 240457435b0805b013d8444efad3c814f3949b9f Mon Sep 17 00:00:00 2001
From: William Dawson <william.dawson@riken.jp>
Date: Thu, 27 Aug 2026 12:00:26 +0900
Subject: [PATCH 14/14] [code:petsc-gmres] Fix DGX Spark math library path

Signed-off-by: William Dawson <william.dawson@riken.jp>
---
 programs/petsc-gmres/build.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/programs/petsc-gmres/build.sh b/programs/petsc-gmres/build.sh
index 85292ed..a418889 100644
--- a/programs/petsc-gmres/build.sh
+++ b/programs/petsc-gmres/build.sh
@@ -100,7 +100,13 @@ case "$system" in
     module load system/ng-dgx nvhpc-hpcx
     export PETSC_ARCH=arch-dgxsp-cuda
     CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
-    MATHLIBS=$(dirname "$(command -v nvcc)")/../../math_libs/*/lib64
+    MATHLIBS_ROOT="$(dirname "$(dirname "$(dirname "$(command -v nvcc)")")")/math_libs"
+    MATHLIBS_CANDIDATES=("${MATHLIBS_ROOT}"/*/lib64)
+    if [ "${#MATHLIBS_CANDIDATES[@]}" -ne 1 ] || [ ! -d "${MATHLIBS_CANDIDATES[0]}" ]; then
+      echo "Expected one NVHPC math_libs/*/lib64 directory under ${MATHLIBS_ROOT}" >&2
+      exit 1
+    fi
+    MATHLIBS="${MATHLIBS_CANDIDATES[0]}"
     (
       cd "${PETSC_DIR}"
       ./configure \