diff --git a/programs/sbd/README.md b/programs/sbd/README.md new file mode 100644 index 0000000..60c44c3 --- /dev/null +++ b/programs/sbd/README.md @@ -0,0 +1,85 @@ +# SBD benchmark + +This BenchKit application runs the NVIDIA Thrust TPB selected-basis +diagonalization benchmark from `github.com/r-ccs-cms/sbd`. + +The RIKYU recipe uses the H2O cc-pVDZ FCIDUMP with the `1em7` selected alpha +determinant file (about 628 million product determinants), one MPI rank per +B200 GPU, 32 OpenMP threads per rank, and the rank-distributed/index-reordered/NCCL +build configuration validated in the SubWG2 benchmark study. 32 threads is +Rikyu's site-enforced CPU cap per requested GPU on shared nodes; it also +covers the pre-Davidson setup pass (`RemakeHelpers`/`TaskCostSize`), which +runs on the host rather than the GPU and is slow to the point of resembling +a hang on large inputs if starved of threads. The inputs are staged from the +SBD source clone into `artifacts/` by `build.sh` (see +`data/h2o/README.md` for provenance) and can be overridden with +`BK_SBD_INPUT_DIR` when a project-storage copy is preferred. + +The DGX Spark route uses the H2O `1em5` input (about 30.4 million product +determinants), the Thrust backend with `cc120`, NVHPC/HPC-X CUDA 13 `26.3`, +and one MPI rank on the single GB10 GPU. The larger `1em7` case reaches about +98 GB before its first Davidson iteration and is killed by the 128 GB node's +memory limit. `1em5` retains a substantial GPU workload while leaving enough +memory headroom and runtime margin for continuous benchmarking. NCCL is disabled +for the single-GPU case, and all three explicit communicator sizes are one. The +Rikyu rows retain the `1em7` input and validated `1 x 2 x 2` explicit layout; +their remaining rank factor becomes SBD's implicit Hamiltonian communicator. + +The R-CCS Cloud GH200 route uses the H2O `1em6` input (about 191 million +product determinants), the Thrust backend with `cc90`, NVHPC/HPC-X CUDA 13 +`26.3`, and one MPI rank on its unified Grace Hopper superchip. No Slurm GPU +request is made because `qc-gh200` exposes its single GPU as part of the node. +NCCL is disabled for the single-rank case. + +The R-CCS Cloud FX700 route uses the H2O `1em4` input (about 2.38 million +product determinants) with four MPI ranks and 12 OpenMP threads per rank, +bound one rank per A64FX NUMA/CMG domain. SBD requires C++17 features missing +from Fujitsu compiler 4.11.1's bundled libc++, so this route uses the system +GCC 8.5/MPICH stack with 512-bit SVE enabled. It links Fujitsu's optimized +LAPACK and its required runtime libraries by absolute path. The absolute paths +are intentional: adding the Fujitsu library directory with `-L` causes +MPICH's trailing `-lmpi` to resolve to Fujitsu MPI instead, mixing two MPI +implementations in one executable. + +The input files are not committed to this repository; `build.sh` stages them +from its `bk_fetch_source` clone of SBD's `data/h2o` directory into `artifacts/`. +`data/h2o/README.md` records the upstream revision, checksums, and Apache-2.0 +provenance. + +The emitted FOM is SBD's internal Davidson time in seconds. Shell wall time is +not used because MPI startup and scheduler overhead are substantial on RIKYU. +The run is accepted only when its energy matches the established reference for +its selected input within a combined `1e-12 + 1e-11 |E|` tolerance. + +The DGX Spark route was validated on `ng-dgx-m2` with NVHPC 26.3. The exact +energy was `-76.24373504205295 Ha`; the internal Davidson FOM was +`228.928853 s`, and the complete Slurm job took 4 minutes 9 seconds. + +The R-CCS Cloud GH200 route was directly validated on `qc-gh200-01` with +NVHPC 26.3. The exact energy was `-76.24377593489788 Ha`; the internal +Davidson FOM was `520.734720 s`, the multiply section was `22.730310 s`, and +the complete Slurm job took 9 minutes 34 seconds. + +The R-CCS Cloud FX700 route was directly validated on `fx29` with GCC 8.5, +MPICH 4.0, 512-bit SVE, and Fujitsu LAPACK. The exact energy was +`-76.2429584823075 Ha`; the internal Davidson FOM was `747.636165 s`, the +multiply section was `33.897903 s`, and the complete Slurm job took 13 minutes +3 seconds. + +The Rikyu route was validated with NVHPC 26.3 on project `rkp00012`. All +three rows converged to `-76.243776776861 Ha`: + +| B200 GPUs | Nodes | Internal Davidson FOM (s) | 4-GPU-relative speedup | +|---:|---:|---:|---:| +| 4 | 1 | 565.510 | 1.00x | +| 8 | 2 | 290.222 | 1.95x | +| 16 | 4 | 152.144 | 3.72x | + +A fresh BenchKit validation rebuilt upstream SBD commit +`9481f290c2f49d4f8e5df9b0c9c87ea0f7937c2c` through `build.sh`, confirmed the +effective `cc100` rank-distributed/index-reordered/NCCL configuration, and +submitted the eight-GPU row through `scripts/test_submit.sh`. The job converged +to `-76.2437767768609 Ha` with a `290.281226 s` internal Davidson FOM, and +`scripts/result.sh` produced valid result JSON with source provenance and the +`mult` timing section. Only this eight-GPU row has been independently re-run +through `test_submit.sh`; the four- and sixteen-GPU rows above have not. diff --git a/programs/sbd/build.sh b/programs/sbd/build.sh new file mode 100755 index 0000000..0b8da9f --- /dev/null +++ b/programs/sbd/build.sh @@ -0,0 +1,102 @@ +#!/bin/bash +set -euo pipefail + +system="$1" +source scripts/bk_functions.sh + +REPO_URL="https://github.com/r-ccs-cms/sbd.git" +REPO_DIR="sbd" +BUILD_DIR="" +ARTIFACT_DIR="${PWD}/artifacts" + +mkdir -p "${ARTIFACT_DIR}" +bk_fetch_source "${REPO_URL}" "${REPO_DIR}" "main" +cd "${REPO_DIR}" + +case "${system}" in + RIKYU) + module purge + module load nvhpc/26.3 + BUILD_DIR="build-rikyu-nvhpc-thrust-rankdist-nccl" + nccl_root="/shared/software/hpc_sdk/Linux_aarch64/26.3/comm_libs/13.1/nccl" + cmake -S . -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DSBD_GPU_BACKEND=thrust \ + -DSBD_GPU_ARCH=cc100 \ + -DSBD_THRUST_SAFE_MPI_ALLREDUCE=ON \ + -DSBD_USE_RANK_DISTRIBUTION=ON \ + -DSBD_USE_BLOCK_RANK_DISTRIBUTION=ON \ + -DSBD_REORDER_INDEX_ARRAY=ON \ + -DSBD_USE_NCCL=ON \ + -DCMAKE_CXX_FLAGS="-I${nccl_root}/include" \ + -DCMAKE_EXE_LINKER_FLAGS="-L${nccl_root}/lib -lnccl" + cmake --build "${BUILD_DIR}" --parallel + ;; + RC_DGXSP) + source /etc/profile.d/modules.sh + module purge + module load system/ng-dgx nvhpc-hpcx-cuda13/26.3 + BUILD_DIR="build-dgxsp-nvhpc-thrust-rankdist" + cmake -S . -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DSBD_GPU_BACKEND=thrust \ + -DSBD_GPU_ARCH=cc120 \ + -DSBD_THRUST_SAFE_MPI_ALLREDUCE=ON \ + -DSBD_USE_RANK_DISTRIBUTION=ON \ + -DSBD_USE_BLOCK_RANK_DISTRIBUTION=ON \ + -DSBD_REORDER_INDEX_ARRAY=ON \ + -DSBD_USE_NCCL=OFF + cmake --build "${BUILD_DIR}" --parallel + ;; + RC_GH200) + module purge + module load system/qc-gh200 nvhpc-hpcx-cuda13/26.3 + BUILD_DIR="build-gh200-nvhpc-thrust-rankdist" + cmake -S . -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DSBD_GPU_BACKEND=thrust \ + -DSBD_GPU_ARCH=cc90 \ + -DSBD_THRUST_SAFE_MPI_ALLREDUCE=ON \ + -DSBD_USE_RANK_DISTRIBUTION=ON \ + -DSBD_USE_BLOCK_RANK_DISTRIBUTION=ON \ + -DSBD_REORDER_INDEX_ARRAY=ON \ + -DSBD_USE_NCCL=OFF + cmake --build "${BUILD_DIR}" --parallel + ;; + RC_FX700) + module purge + module load system/fx700 mpi/mpich-aarch64 + BUILD_DIR="build-fx700-gcc-mpich-rankdist" + fjlib="/opt/FJSVstclanga/cp-1.0.30.01/lib64" + cmake -S . -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=mpicxx \ + -DCMAKE_CXX_FLAGS="-march=armv8.2-a+sve -msve-vector-bits=512" \ + -DBLAS_LIBRARIES="${fjlib}/libfjlapack.so" \ + -DLAPACK_LIBRARIES="${fjlib}/libfjlapack.so" \ + -DCMAKE_CXX_STANDARD_LIBRARIES="${fjlib}/libfj90i.so ${fjlib}/libfj90f.so ${fjlib}/libfjsrcinfo.so -lelf" \ + -DSBD_USE_RANK_DISTRIBUTION=ON \ + -DSBD_USE_BLOCK_RANK_DISTRIBUTION=ON \ + -DSBD_REORDER_INDEX_ARRAY=ON + cmake --build "${BUILD_DIR}" --parallel + ;; + *) + echo "Unknown system: ${system}" >&2 + exit 1 + ;; +esac + +binary="${BUILD_DIR}/apps/chemistry_tpb_selected_basis_diagonalization/diag" +if [[ ! -x "${binary}" ]]; then + echo "SBD executable not found: ${binary}" >&2 + exit 1 +fi +cp "${binary}" "${ARTIFACT_DIR}/diag" + +# Stage benchmark inputs from the source clone into artifacts so run.sh can +# retrieve them on the compute node even when the source tree is absent +# (cross-build mode). See programs/sbd/data/h2o/README.md for provenance. +for input_file in fcidump.txt h2o-1em4-alpha.txt h2o-1em5-alpha.txt \ + h2o-1em6-alpha.txt h2o-1em7-alpha.txt; do + cp "data/h2o/${input_file}" "${ARTIFACT_DIR}/${input_file}" +done diff --git a/programs/sbd/data/h2o/README.md b/programs/sbd/data/h2o/README.md new file mode 100644 index 0000000..aba17bd --- /dev/null +++ b/programs/sbd/data/h2o/README.md @@ -0,0 +1,27 @@ +# SBD H₂O benchmark inputs + +The SBD H₂O inputs are not committed to the BenchKit repository. `build.sh` +stages them from the SBD repository's tracked `data/h2o/` directory into +`artifacts/` during the build, and `run.sh` reads them from there. This file +records their upstream provenance and checksums. + +- Repository: https://github.com/r-ccs-cms/sbd +- Upstream revision: `02324eee32a49f3203522d230bcbc34ef032a6a6` +- Files: `fcidump.txt`, `h2o-1em4-alpha.txt`, `h2o-1em5-alpha.txt`, + `h2o-1em6-alpha.txt`, `h2o-1em7-alpha.txt` + +SHA-256 checksums: + +```text +a3c2302834a33dce7260e8050a3f5180e05dbba1bb748f3e2f6410a7eacbd94d fcidump.txt +858c1ef9d430aafe0c325281c7257c2e6cb0e77310a707c1109c55747db02139 h2o-1em4-alpha.txt +40b29271726daa2be5a5b181535647164e915646710c2641e46dd1ed078a2ee6 h2o-1em5-alpha.txt +c36920e57507d27dd49d2b729662cbb5a92afce26d61fb0e044e2691599eb804 h2o-1em6-alpha.txt +1f7af972b56143a9ae6862c02288668656a5bd3fb561337c430e0e478f7cb716 h2o-1em7-alpha.txt +``` + +The upstream SBD repository distributes these files under its Apache-2.0 +license (`LICENSE.txt`). `build.sh` copies them from its `bk_fetch_source` +clone of the repository, so the BenchKit application runs without committing +the inputs or relying on private project storage. Set `BK_SBD_INPUT_DIR` to +use an equivalent project-storage copy instead. diff --git a/programs/sbd/list.csv b/programs/sbd/list.csv new file mode 100644 index 0000000..781a511 --- /dev/null +++ b/programs/sbd/list.csv @@ -0,0 +1,7 @@ +system,enable,nodes,numproc_node,nthreads,elapse +RIKYU,yes,1,4,32,0:30:00 +RIKYU,yes,2,4,32,0:30:00 +RIKYU,yes,4,4,32,0:30:00 +RC_DGXSP,yes,1,1,1,1:00:00 +RC_GH200,yes,1,1,1,1:00:00 +RC_FX700,yes,1,4,12,1:00:00 diff --git a/programs/sbd/run.sh b/programs/sbd/run.sh new file mode 100755 index 0000000..f4b4893 --- /dev/null +++ b/programs/sbd/run.sh @@ -0,0 +1,144 @@ +#!/bin/bash +set -euo pipefail + +system="$1" +nodes="$2" +numproc_node="$3" +nthreads="$4" +n_ranks=$((nodes * numproc_node)) + +source scripts/bk_functions.sh + +RESULTS_DIR="${PWD}/results" +RUN_DIR="${PWD}/sbd_run" +INPUT_DIR="${BK_SBD_INPUT_DIR:-${PWD}/artifacts}" + +mkdir -p "${RESULTS_DIR}" +: > "${RESULTS_DIR}/result" + +case "${system}" in + RIKYU) + module purge + module load nvhpc/26.3 + determinant_file=h2o-1em7-alpha.txt + experiment=H2O-1em7 + reference_energy=-76.243776776861 + energy_abs_tolerance=1.0e-12 + task_comm_size=1 + adet_comm_size=2 + bdet_comm_size=2 + ;; + RC_DGXSP) + source /etc/profile.d/modules.sh + module purge + module load system/ng-dgx nvhpc-hpcx-cuda13/26.3 + determinant_file=h2o-1em5-alpha.txt + experiment=H2O-1em5 + reference_energy=-76.24373504205295 + energy_abs_tolerance=1.0e-12 + task_comm_size=1 + adet_comm_size=1 + bdet_comm_size=1 + ;; + RC_GH200) + module purge + module load system/qc-gh200 nvhpc-hpcx-cuda13/26.3 + determinant_file=h2o-1em6-alpha.txt + experiment=H2O-1em6 + reference_energy=-76.2437759348979 + energy_abs_tolerance=1.0e-12 + task_comm_size=1 + adet_comm_size=1 + bdet_comm_size=1 + ;; + RC_FX700) + module purge + module load system/fx700 mpi/mpich-aarch64 + determinant_file=h2o-1em4-alpha.txt + experiment=H2O-1em4 + reference_energy=-76.2429584823075 + energy_abs_tolerance=1.0e-12 + task_comm_size=1 + adet_comm_size=2 + bdet_comm_size=2 + ;; + *) + echo "Unknown system: ${system}" >&2 + exit 1 + ;; +esac + +for input_file in fcidump.txt "${determinant_file}"; do + if [[ ! -f "${INPUT_DIR}/${input_file}" ]]; then + echo "SBD input not found: ${INPUT_DIR}/${input_file}" >&2 + exit 1 + fi +done +if [[ ! -x artifacts/diag ]]; then + echo "Required artifact not found or not executable: artifacts/diag" >&2 + exit 1 +fi + +rm -rf "${RUN_DIR}" +mkdir -p "${RUN_DIR}" +cp artifacts/diag "${RUN_DIR}/diag" +cp "${INPUT_DIR}/fcidump.txt" "${INPUT_DIR}/${determinant_file}" "${RUN_DIR}/" +cd "${RUN_DIR}" +export OMP_NUM_THREADS="${nthreads}" + +diag_args=( + --fcidump fcidump.txt + --adetfile "${determinant_file}" + --method 0 + --block 10 + --iteration 4 + --tolerance 1.0e-8 + --init 0 + --shuffle 0 + --carryover_type 0 + --rdm 0 + --task_comm_size "${task_comm_size}" + --adet_comm_size "${adet_comm_size}" + --bdet_comm_size "${bdet_comm_size}" +) + +if [[ "${system}" == "RC_FX700" ]]; then + mpirun -np "${n_ranks}" -bind-to numa ./diag "${diag_args[@]}" \ + > diag.log 2>&1 +else + mpirun -np "${n_ranks}" bash -lc \ + 'export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK; exec "$@"' \ + bash ./diag "${diag_args[@]}" > diag.log 2>&1 +fi + +davidson_time=$(grep -E 'Elapsed time for davidson ' diag.log | tail -n 1 | awk '{print $(NF-1)}') +energy=$(grep -E '^ Energy = ' diag.log | tail -n 1 | awk '{print $3}') +mult_time=$(grep -E 'Elapsed time for mult ' diag.log | tail -n 1 | awk '{print $(NF-1)}') + +if [[ -z "${davidson_time}" || -z "${energy}" ]]; then + echo "SBD completion markers not found" >&2 + tail -n 80 diag.log >&2 + exit 1 +fi +if ! awk -v actual="${energy}" -v reference="${reference_energy}" \ + -v abs_tolerance="${energy_abs_tolerance}" \ + 'BEGIN { + diff = actual - reference + if (diff < 0) diff = -diff + scale = reference + if (scale < 0) scale = -scale + exit !(diff <= abs_tolerance + 1.0e-11 * scale) + }'; then + echo "SBD energy mismatch: got ${energy}, expected ${reference_energy}" >&2 + exit 1 +fi + +cp diag.log "${RESULTS_DIR}/" +bk_emit_result --fom "${davidson_time}" --fom-unit s \ + --fom-version "davidson_internal_s" --exp "${experiment}" \ + --nodes "${nodes}" --numproc-node "${numproc_node}" \ + --nthreads "${nthreads}" >> "${RESULTS_DIR}/result" +if [[ -n "${mult_time}" ]]; then + bk_emit_section mult "${mult_time}" >> "${RESULTS_DIR}/result" +fi +printf 'SBD energy: %s\n' "${energy}" >&2