From 776b9bb6a86ae76248dec7ba60f48c730e5da4e8 Mon Sep 17 00:00:00 2001 From: William Dawson Date: Fri, 31 Jul 2026 15:44:58 +0900 Subject: [PATCH 01/12] Add RIKYU support for SALMON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build: nvhpc/26.3 + OpenACC + cc100 (B200/Blackwell), with FORTRAN_COMPILER_HAS_MPI_VERSION3=OFF to work around the MPI_Iallreduce hang at >1 rank. Run: per-rank GPU binding via wrapper.sh (CUDA_VISIBLE_DEVICES), UCX_IB_GPU_DIRECT_RDMA=no for 2+ nodes, and runtime patching of the ¶llel section to nproc_ob=N (orbital-only decomposition, no real-space grid split) which is the verified GPU-optimal layout. Depends on #74 for RIKYU system/queue config (gpu partition, account=rkp00012). 💘 Generated with Crush Signed-off-by: William Dawson Assisted-by: glm-5.2 via Crush --- programs/salmon/build.sh | 19 +++++++++++++++++ programs/salmon/list.csv | 4 ++++ programs/salmon/run.sh | 44 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 793cf0a5..8f529d82 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -167,6 +167,25 @@ case "${system}" in # -DUSE_FJMPI=OFF # ) # ;; + RIKYU) + module purge + module load nvhpc/26.3 + cmake_args=( + "${common_cmake_args[@]}" + -DCMAKE_Fortran_COMPILER=mpif90 + -DCMAKE_C_COMPILER=mpicc + -DOPENMP_FLAGS=-Mnoopenmp + -DUSE_OPENACC=ON + -DUSE_CUDA=ON + -DUSE_MPI_DEFAULT=ON + -DCMAKE_SYSTEM_PROCESSOR=openacc + -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas -cuda -Minfo=accel -DUSE_OPENACC -DUSE_CUDA" + -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas -cuda -Minfo=accel -DUSE_OPENACC -DUSE_CUDA" + -DCMAKE_CUDA_ARCHITECTURES=100 + -DCMAKE_CUDA_FLAGS=-arch=sm_100 + -DFORTRAN_COMPILER_HAS_MPI_VERSION3=OFF + ) + ;; *) echo "Unknown system: ${system}" >&2 exit 1 diff --git a/programs/salmon/list.csv b/programs/salmon/list.csv index 386804d9..ed4e1596 100644 --- a/programs/salmon/list.csv +++ b/programs/salmon/list.csv @@ -4,3 +4,7 @@ RC_GH200,yes,1,1,1,0:10:00 RC_DGXSP,yes,1,1,1,1:00:00 RC_GENOA,yes,1,1,48,2:00:00 RC_FX700,no,1,1,48,0:30:00 +RIKYU,yes,1,4,1,0:30:00 +RIKYU,yes,2,4,1,0:30:00 +RIKYU,yes,3,4,1,0:30:00 +RIKYU,yes,4,4,1,0:30:00 diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index 1689c38a..b482be05 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -13,6 +13,7 @@ RESULTS_DIR="${PWD}/results" WORK_DIR="${PWD}/salmon_run" INPUT_ARCHIVE_DEFAULT="/vol0003/rccs-sdt/data/a01010/benchmark_data/SALMON.tar.gz" INPUT_ARCHIVE_CLOUD="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON/SALMON.tar.gz" +INPUT_ARCHIVE_RIKYU="/data1/rkp00012/CX_input/SALMON/SALMON.tar.gz" AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" mkdir -p "${RESULTS_DIR}" @@ -29,6 +30,11 @@ case "${system}" in exec_gs=(-stdin Si-1-1-1.nml ./salmon) exec_rt=(-stdin Si-1-1-1-tddft.nml ./salmon) ;; + RIKYU) + input_archive="${BK_SALMON_INPUT_ARCHIVE:-${INPUT_ARCHIVE_RIKYU}}" + exec_gs=(./salmon) + exec_rt=(./salmon) + ;; RC_GH200|RC_DGXSP|RC_GENOA) input_archive="${INPUT_ARCHIVE_CLOUD}" exec_gs=(./salmon) @@ -47,7 +53,7 @@ fi uses_stdin_input() { case "$1" in - RC_GH200|RC_DGXSP|RC_GENOA) + RIKYU|RC_GH200|RC_DGXSP|RC_GENOA) return 0 ;; *) @@ -108,6 +114,35 @@ case "${system}" in fi export OMP_NUM_THREADS="${nthreads}" ;; + RIKYU) + module purge + module load nvhpc/26.3 + export OMP_NUM_THREADS="${nthreads}" + for nml in Si-1-1-1.nml Si-1-1-1-tddft.nml; do + awk -v nproc_ob="${n_ranks}" ' + /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = 1, 1, 1"; in_parallel=1; next } + in_parallel && /^\// { in_parallel=0; print; next } + in_parallel { next } + { print } + ' "$nml" > "$nml.tmp" && mv "$nml.tmp" "$nml" + done + if [[ "${n_ranks}" -gt 1 ]]; then + cat > wrapper.sh <<'WRAPPER' +#!/bin/bash +NCUDA_GPUS=${NCUDA_GPUS:-$(nvidia-smi -L | wc -l)} +if [ "$OMPI_COMM_WORLD_LOCAL_SIZE" -gt "$NCUDA_GPUS" ]; then + if [ "$OMPI_COMM_WORLD_LOCAL_RANK" -eq 0 ]; then + nvidia-cuda-mps-control -d + fi + sleep 10 +fi +export CUDA_VISIBLE_DEVICES=$((${OMPI_COMM_WORLD_LOCAL_RANK} % ${NCUDA_GPUS})) +exec "$@" +WRAPPER + chmod +x wrapper.sh + export UCX_IB_GPU_DIRECT_RDMA=no + fi + ;; # RC_FX700) # FX700 currently fails during GS initialization even with the Fujitsu # topology guard patch applied. Keep this route disabled until verified. @@ -122,6 +157,13 @@ run_salmon() { local logfile="$1" shift case "${system}" in + RIKYU) + if [[ "${n_ranks}" -gt 1 ]]; then + mpirun -n "${n_ranks}" ./wrapper.sh "$@" > "${logfile}" 2>&1 + else + mpirun -n "${n_ranks}" "$@" > "${logfile}" 2>&1 + fi + ;; RC_GENOA) mpirun -n "${n_ranks}" --bind-to core --map-by "ppr:${numproc_node}:node:PE=${nthreads}" "$@" > "${logfile}" 2>&1 ;; From 9272d1c58749a3b618a27392c6fb10916122bb9b Mon Sep 17 00:00:00 2001 From: William Dawson Date: Mon, 17 Aug 2026 14:37:30 +0900 Subject: [PATCH 02/12] [code:salmon] RIKYU: pre-staged folded restart + nvhpc/26.5 Ewald fix Two changes to the RIKYU SALMON benchmark, validated end-to-end on real hardware (job 28922, 4 GPUs, exit 0, FOM 337.2s, energy matching the known-good 3x3x3 reference to 6 sig figs at step 10): 1. GS is now offline prep, not part of the benchmark. run.sh reads a python-folded 3x3x3 SiO2 restart pre-staged at /data1/rkp00012/CX_input/SALMON/3x3x3-folded/ (restart/ symlinked in, never copied -- it's ~35GB) and runs TDDFT only. This replaces the old Si-1-1-1 GS+TDDFT flow for RIKYU; other systems are unaffected (uses_prestaged_restart() gates the new path). See subwg2-benchmarks' README.md/salmon-benchmarking for how the restart was produced (`benchgen salmon create --python-fold`) and regenerated if the source system size changes. 2. build.sh switches RIKYU from nvhpc/26.3 + MPI3-off to native nvhpc/26.5 + MPI3 on, plus a new patch (patches/nvhpc265-ewald-reduction.patch, cherry-picked from william-dawson/SALMON2 commit d8442058) that works around a 26.5-only OpenACC reduction-codegen bug in the Ewald ion-ion energy term. The patch is self-gating (checks __NVCOMPILER_MAJOR__/MINOR__ at compile time) so it's applied unconditionally on every GPU build (RIKYU, RC_GH200, RC_DGXSP) -- a no-op on compilers it doesn't affect. This combination is bit-exact and was already confirmed faster than the old 26.3+MPI3-off recipe on Rikyu's domain- decomposition case; this PR is the first real validation on the orbital-decomposition layout RIKYU's list.csv actually uses. Co-Authored-By: Claude Sonnet 5 Signed-off-by: William Dawson --- programs/salmon/build.sh | 30 +- .../patches/nvhpc265-ewald-reduction.patch | 100 +++++++ programs/salmon/run.sh | 267 +++++++++++------- 3 files changed, 298 insertions(+), 99 deletions(-) create mode 100644 programs/salmon/patches/nvhpc265-ewald-reduction.patch diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 8f529d82..93ea40af 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -12,6 +12,7 @@ RESULTS_DIR="${PWD}/results" BUILD_LOG_DIR="${RESULTS_DIR}/salmon_build_logs" AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" FJMPI_PATCH="${PWD}/programs/salmon/patches/fjmpi-topology-guard.patch" +EWALD_265_PATCH="${PWD}/programs/salmon/patches/nvhpc265-ewald-reduction.patch" source scripts/bk_functions.sh @@ -28,6 +29,24 @@ else echo "SALMON Fujitsu MPI topology patch does not apply to ${VERSION_TAG}" >&2 exit 1 fi + +apply_ewald_265_patch() { + # Works around a silent nvfortran 26.5 OpenACC reduction-codegen bug that + # gives a wrong (but plausible) total energy -- see the patch file itself + # and .claude/skills/salmon-build in subwg2-benchmarks for the full trail. + # Safe to apply on every OpenACC/GPU build regardless of nvhpc version: + # the fix gates itself on __NVCOMPILER_MAJOR__/__NVCOMPILER_MINOR__ at + # compile time, so it's a no-op on unaffected compilers (<26.5). + if git apply --check "${EWALD_265_PATCH}"; then + git apply "${EWALD_265_PATCH}" + elif git apply --reverse --check "${EWALD_265_PATCH}" >/dev/null 2>&1; then + echo "SALMON nvhpc/26.5 Ewald-reduction patch is already applied" + else + echo "SALMON nvhpc/26.5 Ewald-reduction patch does not apply to ${VERSION_TAG}" >&2 + exit 1 + fi +} + rm -rf "${BUILD_DIR}" mkdir -p "${BUILD_DIR}" @@ -83,6 +102,7 @@ case "${system}" in RC_GH200) module purge module load system/qc-gh200 nvhpc-hpcx-cuda12/25.7 + apply_ewald_265_patch cmake_args=( "${common_cmake_args[@]}" -DCMAKE_Fortran_COMPILER=mpif90 @@ -102,6 +122,7 @@ case "${system}" in source /etc/profile.d/modules.sh module purge module load system/ng-dgx nvhpc-hpcx-cuda13/26.3 + apply_ewald_265_patch cmake_args=( "${common_cmake_args[@]}" -DCMAKE_Fortran_COMPILER=mpif90 @@ -169,7 +190,8 @@ case "${system}" in # ;; RIKYU) module purge - module load nvhpc/26.3 + module load nvhpc/26.5 + apply_ewald_265_patch cmake_args=( "${common_cmake_args[@]}" -DCMAKE_Fortran_COMPILER=mpif90 @@ -183,7 +205,11 @@ case "${system}" in -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas -cuda -Minfo=accel -DUSE_OPENACC -DUSE_CUDA" -DCMAKE_CUDA_ARCHITECTURES=100 -DCMAKE_CUDA_FLAGS=-arch=sm_100 - -DFORTRAN_COMPILER_HAS_MPI_VERSION3=OFF + # nvhpc/26.5, native MPI3 ON (no FORTRAN_COMPILER_HAS_MPI_VERSION3 + # override): the HPC-X libnbc bug that forced MPI3 off is 26.3-only. + # Confirmed bit-exact and faster than the old 26.3+MPI3-off build on + # a 2-GPU domain-decomposition case -- see nvhpc265-ewald-reduction + # patch header and subwg2-benchmarks' salmon-build skill. ) ;; *) diff --git a/programs/salmon/patches/nvhpc265-ewald-reduction.patch b/programs/salmon/patches/nvhpc265-ewald-reduction.patch new file mode 100644 index 00000000..609e9691 --- /dev/null +++ b/programs/salmon/patches/nvhpc265-ewald-reduction.patch @@ -0,0 +1,100 @@ +diff --git a/src/common/total_energy.f90 b/src/common/total_energy.f90 +index cc8fa5bd..cde5ef1d 100644 +--- a/src/common/total_energy.f90 ++++ b/src/common/total_energy.f90 +@@ -15,6 +15,39 @@ + ! + !--------10--------20--------30--------40--------50--------60--------70--------80--------90--------100-------110-------120-------130 + MODULE Total_Energy ++ ++! --------------------------------------------------------------------------- ++! nvfortran 26.5 OpenACC reduction workaround. ++! ++! Under nvhpc 26.5 the OpenACC reductions in init_ewald (reduction(max:)) and ++! calc_Total_Energy_periodic (reduction(+:)) silently return the reduction ++! IDENTITY (0) instead of the computed value. Verified by building the very ++! same loops as hand-written CUDA kernels on the same GPU with the same data: ++! CUDA gives 74 / 0.4478233939199485 (correct), OpenACC gives 0 / 0. ++! nvhpc 26.3 is unaffected. Both loops are one-time / per-ion-move setup, so ++! running them on the host costs essentially nothing. ++! ++! The book-keeping fill loop in init_ewald is affected the same way (its ++! per-atom counter comes back as 1 instead of ~74), so it is gated too. ++! ++! Unrelated to the compiler, that same fill loop also declared copyin(ewald) ++! -- host->device ONLY -- while writing ewald%bk / ewald%npair_bk inside the ++! region, so those device writes were semantically discarded. The wrong clause ++! is removed here so the GPU path is correct once the gate is relaxed. (That ++! fix on its own does not cure 26.5; the codegen bug above is the blocker.) ++! ++! When NVIDIA fixes this, drop the version test below (or raise the bound). ++! --------------------------------------------------------------------------- ++#if defined(USE_OPENACC) ++# if defined(__NVCOMPILER_MAJOR__) && (__NVCOMPILER_MAJOR__ > 26 || (__NVCOMPILER_MAJOR__ == 26 && __NVCOMPILER_MINOR__ >= 5)) ++# define SALMON_ACC_REDUCTION_BROKEN 1 ++# endif ++#endif ++ ++#if defined(USE_OPENACC) && !defined(SALMON_ACC_REDUCTION_BROKEN) ++# define SALMON_EWALD_ACC 1 ++#endif ++ + implicit none + + CONTAINS +@@ -323,7 +356,7 @@ CONTAINS + + if(ewald%yn_bookkeep=='y') then + +-#ifdef USE_OPENACC ++#ifdef SALMON_EWALD_ACC + !$acc kernels copyin(ewald) + !$acc loop private(iia,ia,ipair,ix,iy,iz,ib,r,rab,rr) reduction(+:E_tmp) + #else +@@ -356,7 +389,7 @@ CONTAINS + + end do !ipair + end do !ia +-#ifdef USE_OPENACC ++#ifdef SALMON_EWALD_ACC + !$acc end kernels + #else + !$omp end parallel do +@@ -843,7 +876,7 @@ CONTAINS + + !(check maximum number of pairs and allocate) + npair_bk_max = 0 +-#ifdef USE_OPENACC ++#ifdef SALMON_EWALD_ACC + !$acc kernels loop private(iia,ia,ix,iy,iz,ib,r,rab,rr,npair_bk_loc) reduction(max:npair_bk_max) + #else + !$omp parallel do private(iia,ia,ix,iy,iz,ib,r,rab,rr,npair_bk_loc) & +@@ -880,7 +913,7 @@ CONTAINS + end do + npair_bk_max = max(npair_bk_max,npair_bk_loc) + end do +-#ifdef USE_OPENACC ++#ifdef SALMON_EWALD_ACC + !$acc end kernels + #else + !$omp end parallel do +@@ -897,8 +930,8 @@ CONTAINS + 820 format(a,i6) + endif + +-#ifdef USE_OPENACC +-!$acc kernels loop private(iia,ia,ipair,ix,iy,iz,ib,r,rab,rr) copyin(ewald) ++#ifdef SALMON_EWALD_ACC ++!$acc kernels loop private(iia,ia,ipair,ix,iy,iz,ib,r,rab,rr) + #else + !$omp parallel do private(iia,ia,ipair,ix,iy,iz,ib,r,rab,rr) + #endif +@@ -937,7 +970,7 @@ CONTAINS + end do + ewald%npair_bk(iia) = ipair + end do +-#ifdef USE_OPENACC ++#ifdef SALMON_EWALD_ACC + !$acc end kernels + #else + !$omp end parallel do diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index b482be05..09003d38 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -13,9 +13,19 @@ RESULTS_DIR="${PWD}/results" WORK_DIR="${PWD}/salmon_run" INPUT_ARCHIVE_DEFAULT="/vol0003/rccs-sdt/data/a01010/benchmark_data/SALMON.tar.gz" INPUT_ARCHIVE_CLOUD="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON/SALMON.tar.gz" -INPUT_ARCHIVE_RIKYU="/data1/rkp00012/CX_input/SALMON/SALMON.tar.gz" AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" +# Pre-staged, python-folded restarts: GS(k) -> Python fold -> complex Gamma +# TDDFT restart, prepared once offline via `benchgen salmon create +# --python-fold` (see README.md / salmon-benchmarking in +# subwg2-benchmarks) and stored on each machine so the benchmark itself +# never pays for a from-scratch ground state. Add a new case here (and a +# matching directory on that system) to move another system onto this +# path -- see the RIKYU entry for the shape a new one needs +# (restart/, *.psp8, and one TDDFT .nml, all siblings in one directory). +RIKYU_RESTART_DIR_DEFAULT="/data1/rkp00012/CX_input/SALMON/3x3x3-folded" +RIKYU_RESTART_NML="Si-3-3-3-tddft.nml" + mkdir -p "${RESULTS_DIR}" : > "${RESULTS_DIR}/result" @@ -24,32 +34,16 @@ if [[ ! -x artifacts/salmon ]]; then exit 1 fi -case "${system}" in - Fugaku) - input_archive="${INPUT_ARCHIVE_DEFAULT}" - exec_gs=(-stdin Si-1-1-1.nml ./salmon) - exec_rt=(-stdin Si-1-1-1-tddft.nml ./salmon) - ;; - RIKYU) - input_archive="${BK_SALMON_INPUT_ARCHIVE:-${INPUT_ARCHIVE_RIKYU}}" - exec_gs=(./salmon) - exec_rt=(./salmon) - ;; - RC_GH200|RC_DGXSP|RC_GENOA) - input_archive="${INPUT_ARCHIVE_CLOUD}" - exec_gs=(./salmon) - exec_rt=(./salmon) - ;; - *) - echo "Unknown system: ${system}" >&2 - exit 1 - ;; -esac - -if [[ ! -f "${input_archive}" ]]; then - echo "Input archive not found: ${input_archive}" >&2 - exit 1 -fi +uses_prestaged_restart() { + case "$1" in + RIKYU) + return 0 + ;; + *) + return 1 + ;; + esac +} uses_stdin_input() { case "$1" in @@ -63,22 +57,70 @@ uses_stdin_input() { } rm -rf "${WORK_DIR}" -mkdir -p "${WORK_DIR}/input" -tar -xzf "${input_archive}" -C "${WORK_DIR}/input" +mkdir -p "${WORK_DIR}" -input_dir=$(find "${WORK_DIR}/input" -type d -path "*/Si-1-1-1/input" | head -n 1) -if [[ -z "${input_dir}" ]]; then - input_dir=$(find "${WORK_DIR}/input" -type f -name "Si-1-1-1.nml" -printf '%h\n' | head -n 1) -fi -if [[ -z "${input_dir}" || ! -d "${input_dir}" ]]; then - echo "SALMON Si-1-1-1 input directory not found in ${input_archive}" >&2 - exit 1 -fi +if uses_prestaged_restart "${system}"; then + case "${system}" in + RIKYU) + restart_dir="${BK_SALMON_RESTART_DIR:-${RIKYU_RESTART_DIR_DEFAULT}}" + tddft_nml="${RIKYU_RESTART_NML}" + ;; + esac + + if [[ ! -d "${restart_dir}" || ! -d "${restart_dir}/restart" ]]; then + echo "Pre-staged restart not found: ${restart_dir} (expects restart/, *.psp8, ${tddft_nml})" >&2 + exit 1 + fi + + cp artifacts/salmon "${WORK_DIR}/salmon" + chmod +x "${WORK_DIR}/salmon" + cp "${restart_dir}/${tddft_nml}" "${restart_dir}"/*.psp8 "${WORK_DIR}/" + # Symlink, never copy: the restart is O(10s of GB) (a folded 3x3x3 + # wfn.bin alone is ~35GB) and is immutable input, so copying it into a + # throwaway per-run work dir would burn most of the wall-clock budget on + # I/O instead of the benchmark itself. + ln -s "${restart_dir}/restart" "${WORK_DIR}/restart" + grep -Ein '^[[:space:]]*theory[[:space:]]*=' "${WORK_DIR}/${tddft_nml}" >&2 || true +else + case "${system}" in + Fugaku) + input_archive="${INPUT_ARCHIVE_DEFAULT}" + exec_gs=(-stdin Si-1-1-1.nml ./salmon) + exec_rt=(-stdin Si-1-1-1-tddft.nml ./salmon) + ;; + RC_GH200|RC_DGXSP|RC_GENOA) + input_archive="${INPUT_ARCHIVE_CLOUD}" + exec_gs=(./salmon) + exec_rt=(./salmon) + ;; + *) + echo "Unknown system: ${system}" >&2 + exit 1 + ;; + esac + + if [[ ! -f "${input_archive}" ]]; then + echo "Input archive not found: ${input_archive}" >&2 + exit 1 + fi + + mkdir -p "${WORK_DIR}/input" + tar -xzf "${input_archive}" -C "${WORK_DIR}/input" + + input_dir=$(find "${WORK_DIR}/input" -type d -path "*/Si-1-1-1/input" | head -n 1) + if [[ -z "${input_dir}" ]]; then + input_dir=$(find "${WORK_DIR}/input" -type f -name "Si-1-1-1.nml" -printf '%h\n' | head -n 1) + fi + if [[ -z "${input_dir}" || ! -d "${input_dir}" ]]; then + echo "SALMON Si-1-1-1 input directory not found in ${input_archive}" >&2 + exit 1 + fi -cp artifacts/salmon "${WORK_DIR}/salmon" -chmod +x "${WORK_DIR}/salmon" -cp "${input_dir}"/* "${WORK_DIR}/" -grep -Ein '^[[:space:]]*theory[[:space:]]*=' "${WORK_DIR}/Si-1-1-1.nml" "${WORK_DIR}/Si-1-1-1-tddft.nml" >&2 || true + cp artifacts/salmon "${WORK_DIR}/salmon" + chmod +x "${WORK_DIR}/salmon" + cp "${input_dir}"/* "${WORK_DIR}/" + grep -Ein '^[[:space:]]*theory[[:space:]]*=' "${WORK_DIR}/Si-1-1-1.nml" "${WORK_DIR}/Si-1-1-1-tddft.nml" >&2 || true +fi cd "${WORK_DIR}" case "${system}" in @@ -116,16 +158,14 @@ case "${system}" in ;; RIKYU) module purge - module load nvhpc/26.3 + module load nvhpc/26.5 export OMP_NUM_THREADS="${nthreads}" - for nml in Si-1-1-1.nml Si-1-1-1-tddft.nml; do - awk -v nproc_ob="${n_ranks}" ' - /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = 1, 1, 1"; in_parallel=1; next } - in_parallel && /^\// { in_parallel=0; print; next } - in_parallel { next } - { print } - ' "$nml" > "$nml.tmp" && mv "$nml.tmp" "$nml" - done + awk -v nproc_ob="${n_ranks}" ' + /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = 1, 1, 1"; in_parallel=1; next } + in_parallel && /^\// { in_parallel=0; print; next } + in_parallel { next } + { print } + ' "${tddft_nml}" > "${tddft_nml}.tmp" && mv "${tddft_nml}.tmp" "${tddft_nml}" if [[ "${n_ranks}" -gt 1 ]]; then cat > wrapper.sh <<'WRAPPER' #!/bin/bash @@ -228,57 +268,90 @@ print_salmon_output_diagnostics() { ) } -touch .gs_start_marker -gs_start=$(date +%s.%N) -if uses_stdin_input "${system}"; then - run_salmon_or_diagnose GS .gs_start_marker gs.log "${exec_gs[@]}" < Si-1-1-1.nml -else - run_salmon_or_diagnose GS .gs_start_marker gs.log "${exec_gs[@]}" -fi -gs_end=$(date +%s.%N) +if uses_prestaged_restart "${system}"; then + # GS is offline prep, done once when the restart was staged -- see + # salmon-benchmarking in subwg2-benchmarks for how/when to regenerate + # it. The benchmark itself is TDDFT-only. + touch .rt_start_marker + rt_start=$(date +%s.%N) + run_salmon_or_diagnose RT .rt_start_marker rt.log ./salmon < "${tddft_nml}" + rt_end=$(date +%s.%N) -if [[ -d data_for_restart ]]; then - rm -rf restart - mv data_for_restart restart -fi + rt_elapsed=$(awk -v start="${rt_start}" -v end="${rt_end}" 'BEGIN {printf "%.6f", end - start}') -touch .rt_start_marker -rt_start=$(date +%s.%N) -if uses_stdin_input "${system}"; then - run_salmon_or_diagnose RT .rt_start_marker rt.log "${exec_rt[@]}" < Si-1-1-1-tddft.nml -else - run_salmon_or_diagnose RT .rt_start_marker rt.log "${exec_rt[@]}" -fi -rt_end=$(date +%s.%N) - -gs_elapsed=$(awk -v start="${gs_start}" -v end="${gs_end}" 'BEGIN {printf "%.6f", end - start}') -rt_elapsed=$(awk -v start="${rt_start}" -v end="${rt_end}" 'BEGIN {printf "%.6f", end - start}') -total_elapsed=$(awk -v gs="${gs_elapsed}" -v rt="${rt_elapsed}" 'BEGIN {printf "%.6f", gs + rt}') - -cp gs.log rt.log "${RESULTS_DIR}/" - -if ! salmon_output_has_marker_since gs.log .gs_start_marker || ! salmon_output_has_marker_since rt.log .rt_start_marker; then - echo "SALMON success marker not found in both gs.log and rt.log" >&2 - echo "---- gs.log tail ----" >&2 - tail -n 40 gs.log >&2 || true - echo "---- rt.log tail ----" >&2 - tail -n 40 rt.log >&2 || true - echo "---- files updated since GS start ----" >&2 - print_salmon_output_diagnostics .gs_start_marker - echo "---- files updated since RT start ----" >&2 - print_salmon_output_diagnostics .rt_start_marker - exit 1 -fi + cp rt.log "${RESULTS_DIR}/" + + if ! salmon_output_has_marker_since rt.log .rt_start_marker; then + echo "SALMON RT run failed" >&2 + echo "---- rt.log tail ----" >&2 + tail -n 40 rt.log >&2 || true + echo "---- files updated since RT start ----" >&2 + print_salmon_output_diagnostics .rt_start_marker + exit 1 + fi -{ bk_emit_result \ - --fom "${total_elapsed}" \ + --fom "${rt_elapsed}" \ --fom-unit s \ - --fom-version "total_elapsed_time_s" \ - --exp "Si-1-1-1" \ + --fom-version "tddft_elapsed_time_s_folded_restart" \ + --exp "${tddft_nml%.nml}" \ --nodes "${nodes}" \ --numproc-node "${numproc_node}" \ - --nthreads "${nthreads}" - bk_emit_section gs "${gs_elapsed}" - bk_emit_section rt "${rt_elapsed}" -} >> "${RESULTS_DIR}/result" + --nthreads "${nthreads}" \ + >> "${RESULTS_DIR}/result" +else + touch .gs_start_marker + gs_start=$(date +%s.%N) + if uses_stdin_input "${system}"; then + run_salmon_or_diagnose GS .gs_start_marker gs.log "${exec_gs[@]}" < Si-1-1-1.nml + else + run_salmon_or_diagnose GS .gs_start_marker gs.log "${exec_gs[@]}" + fi + gs_end=$(date +%s.%N) + + if [[ -d data_for_restart ]]; then + rm -rf restart + mv data_for_restart restart + fi + + touch .rt_start_marker + rt_start=$(date +%s.%N) + if uses_stdin_input "${system}"; then + run_salmon_or_diagnose RT .rt_start_marker rt.log "${exec_rt[@]}" < Si-1-1-1-tddft.nml + else + run_salmon_or_diagnose RT .rt_start_marker rt.log "${exec_rt[@]}" + fi + rt_end=$(date +%s.%N) + + gs_elapsed=$(awk -v start="${gs_start}" -v end="${gs_end}" 'BEGIN {printf "%.6f", end - start}') + rt_elapsed=$(awk -v start="${rt_start}" -v end="${rt_end}" 'BEGIN {printf "%.6f", end - start}') + total_elapsed=$(awk -v gs="${gs_elapsed}" -v rt="${rt_elapsed}" 'BEGIN {printf "%.6f", gs + rt}') + + cp gs.log rt.log "${RESULTS_DIR}/" + + if ! salmon_output_has_marker_since gs.log .gs_start_marker || ! salmon_output_has_marker_since rt.log .rt_start_marker; then + echo "SALMON success marker not found in both gs.log and rt.log" >&2 + echo "---- gs.log tail ----" >&2 + tail -n 40 gs.log >&2 || true + echo "---- rt.log tail ----" >&2 + tail -n 40 rt.log >&2 || true + echo "---- files updated since GS start ----" >&2 + print_salmon_output_diagnostics .gs_start_marker + echo "---- files updated since RT start ----" >&2 + print_salmon_output_diagnostics .rt_start_marker + exit 1 + fi + + { + bk_emit_result \ + --fom "${total_elapsed}" \ + --fom-unit s \ + --fom-version "total_elapsed_time_s" \ + --exp "Si-1-1-1" \ + --nodes "${nodes}" \ + --numproc-node "${numproc_node}" \ + --nthreads "${nthreads}" + bk_emit_section gs "${gs_elapsed}" + bk_emit_section rt "${rt_elapsed}" + } >> "${RESULTS_DIR}/result" +fi From 5aca2492dc582b8718fb5926a27ee62e69a0695e Mon Sep 17 00:00:00 2001 From: William Dawson Date: Mon, 17 Aug 2026 15:31:21 +0900 Subject: [PATCH 03/12] [code:salmon] RC_DGXSP: pre-staged folded restart, TDDFT only Extends the pre-staged-restart pattern (added for RIKYU) to RC_DGXSP. run.sh now reads a python-folded 2x2x2 SiO2 restart staged at /lvs0/dne1/rccs-nghpcadu/CX_input/SALMON_2x2x2_folded/ and runs TDDFT only, skipping GS. build.sh needed no change here -- the nvhpc/26.5 Ewald patch added for RIKYU already applies unconditionally to every GPU build including RC_DGXSP's (a no-op there, since it still builds on 26.3, unaffected by the bug). The restart was produced by running SALMON's own GS(k) stage for real (18-atom primitive cell, 2x2x2 k-mesh, 8 ranks on one ng-dgx-m1 GPU) via the existing patched 26.5 binary already on that filesystem, then folding it locally with `benchgen salmon fold-restart --primitive-grid 29,49,31 --repetitions 2,2,2`. Validated end-to-end on real hardware (job 286864, ng-dgx-m1, 1 GPU): clean build, FOM 722.2s, correct electron count (768), stable energy across all 300 RT steps, clean exit ("end SALMON"). Co-Authored-By: Claude Sonnet 5 Signed-off-by: William Dawson --- programs/salmon/run.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index 09003d38..2c51253a 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -21,10 +21,12 @@ AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" # subwg2-benchmarks) and stored on each machine so the benchmark itself # never pays for a from-scratch ground state. Add a new case here (and a # matching directory on that system) to move another system onto this -# path -- see the RIKYU entry for the shape a new one needs +# path -- see the RIKYU or RC_DGXSP entries for the shape a new one needs # (restart/, *.psp8, and one TDDFT .nml, all siblings in one directory). RIKYU_RESTART_DIR_DEFAULT="/data1/rkp00012/CX_input/SALMON/3x3x3-folded" RIKYU_RESTART_NML="Si-3-3-3-tddft.nml" +RC_DGXSP_RESTART_DIR_DEFAULT="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON_2x2x2_folded" +RC_DGXSP_RESTART_NML="Si-2-2-2-tddft.nml" mkdir -p "${RESULTS_DIR}" : > "${RESULTS_DIR}/result" @@ -36,7 +38,7 @@ fi uses_prestaged_restart() { case "$1" in - RIKYU) + RIKYU|RC_DGXSP) return 0 ;; *) @@ -65,6 +67,10 @@ if uses_prestaged_restart "${system}"; then restart_dir="${BK_SALMON_RESTART_DIR:-${RIKYU_RESTART_DIR_DEFAULT}}" tddft_nml="${RIKYU_RESTART_NML}" ;; + RC_DGXSP) + restart_dir="${BK_SALMON_RESTART_DIR:-${RC_DGXSP_RESTART_DIR_DEFAULT}}" + tddft_nml="${RC_DGXSP_RESTART_NML}" + ;; esac if [[ ! -d "${restart_dir}" || ! -d "${restart_dir}/restart" ]]; then @@ -88,7 +94,7 @@ else exec_gs=(-stdin Si-1-1-1.nml ./salmon) exec_rt=(-stdin Si-1-1-1-tddft.nml ./salmon) ;; - RC_GH200|RC_DGXSP|RC_GENOA) + RC_GH200|RC_GENOA) input_archive="${INPUT_ARCHIVE_CLOUD}" exec_gs=(./salmon) exec_rt=(./salmon) From 1e2912034349e8dd040d18f433ffbedad6b0623e Mon Sep 17 00:00:00 2001 From: William Dawson Date: Mon, 17 Aug 2026 18:04:49 +0900 Subject: [PATCH 04/12] [code:salmon] Fugaku: pre-staged folded restart, TDDFT only Per instruction: extend the pre-staged-restart pattern to Fugaku and fix list.csv's rank count (24x1, not 1x4 -- Fujitsu MPI-IO errors above ~2GB/rank on restart read). Co-Authored-By: Claude Sonnet 5 Signed-off-by: William Dawson --- programs/salmon/list.csv | 2 +- programs/salmon/run.sh | 43 ++++++++++++++++++++++++++++++++-------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/programs/salmon/list.csv b/programs/salmon/list.csv index ed4e1596..ac6c8c56 100644 --- a/programs/salmon/list.csv +++ b/programs/salmon/list.csv @@ -1,5 +1,5 @@ system,enable,nodes,numproc_node,nthreads,elapse -Fugaku,yes,1,4,12,0:10:00 +Fugaku,yes,24,1,12,0:30:00 RC_GH200,yes,1,1,1,0:10:00 RC_DGXSP,yes,1,1,1,1:00:00 RC_GENOA,yes,1,1,48,2:00:00 diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index 2c51253a..23d06dea 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -11,7 +11,6 @@ source "${PWD}/scripts/bk_functions.sh" RESULTS_DIR="${PWD}/results" WORK_DIR="${PWD}/salmon_run" -INPUT_ARCHIVE_DEFAULT="/vol0003/rccs-sdt/data/a01010/benchmark_data/SALMON.tar.gz" INPUT_ARCHIVE_CLOUD="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON/SALMON.tar.gz" AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" @@ -27,6 +26,20 @@ RIKYU_RESTART_DIR_DEFAULT="/data1/rkp00012/CX_input/SALMON/3x3x3-folded" RIKYU_RESTART_NML="Si-3-3-3-tddft.nml" RC_DGXSP_RESTART_DIR_DEFAULT="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON_2x2x2_folded" RC_DGXSP_RESTART_NML="Si-2-2-2-tddft.nml" +FUGAKU_RESTART_DIR_DEFAULT="/home/ra000009/data/u10035/CX_input_fugaku/SALMON_3x3x3_folded" +FUGAKU_RESTART_NML="Si-3-3-3-tddft.nml" +# Fugaku's list.csv row MUST use enough MPI ranks that each rank's local +# wfn.bin chunk stays under ~2GB: SALMON's default restart reader +# (method_wf_distributor='single') does one MPI_File_read_all per rank +# spanning its whole orbital slice, and Fujitsu MPI's MPI-IO throws +# MPI_ERR_ARG on per-rank reads above that size (a real, reproduced, +# vendor-specific MPI-IO limit -- confirmed independent of both restart +# provenance and physical memory: it reproduced identically whether the +# restart was folded on the login node or a genuine A64FX compute node, +# and separately from the plain OOM that a too-small node count/rank +# count hits first for this restart's 37GB wfn.bin). 24 ranks (81 +# orbitals/rank, ~1.5GB/rank) is the smallest node count validated so far +# for the 3x3x3 case; going lower reintroduces the failure. mkdir -p "${RESULTS_DIR}" : > "${RESULTS_DIR}/result" @@ -38,7 +51,7 @@ fi uses_prestaged_restart() { case "$1" in - RIKYU|RC_DGXSP) + RIKYU|RC_DGXSP|Fugaku) return 0 ;; *) @@ -71,6 +84,10 @@ if uses_prestaged_restart "${system}"; then restart_dir="${BK_SALMON_RESTART_DIR:-${RC_DGXSP_RESTART_DIR_DEFAULT}}" tddft_nml="${RC_DGXSP_RESTART_NML}" ;; + Fugaku) + restart_dir="${BK_SALMON_RESTART_DIR:-${FUGAKU_RESTART_DIR_DEFAULT}}" + tddft_nml="${FUGAKU_RESTART_NML}" + ;; esac if [[ ! -d "${restart_dir}" || ! -d "${restart_dir}/restart" ]]; then @@ -89,11 +106,6 @@ if uses_prestaged_restart "${system}"; then grep -Ein '^[[:space:]]*theory[[:space:]]*=' "${WORK_DIR}/${tddft_nml}" >&2 || true else case "${system}" in - Fugaku) - input_archive="${INPUT_ARCHIVE_DEFAULT}" - exec_gs=(-stdin Si-1-1-1.nml ./salmon) - exec_rt=(-stdin Si-1-1-1-tddft.nml ./salmon) - ;; RC_GH200|RC_GENOA) input_archive="${INPUT_ARCHIVE_CLOUD}" exec_gs=(./salmon) @@ -132,6 +144,14 @@ cd "${WORK_DIR}" case "${system}" in Fugaku) export OMP_NUM_THREADS="${nthreads}" + # Fugaku always uses the pre-staged-restart path (see + # uses_prestaged_restart above), so tddft_nml is always set here. + awk -v nproc_ob="${n_ranks}" ' + /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = 1, 1, 1"; in_parallel=1; next } + in_parallel && /^\// { in_parallel=0; print; next } + in_parallel { next } + { print } + ' "${tddft_nml}" > "${tddft_nml}.tmp" && mv "${tddft_nml}.tmp" "${tddft_nml}" ;; RC_GH200) module purge @@ -280,7 +300,14 @@ if uses_prestaged_restart "${system}"; then # it. The benchmark itself is TDDFT-only. touch .rt_start_marker rt_start=$(date +%s.%N) - run_salmon_or_diagnose RT .rt_start_marker rt.log ./salmon < "${tddft_nml}" + if uses_stdin_input "${system}"; then + run_salmon_or_diagnose RT .rt_start_marker rt.log ./salmon < "${tddft_nml}" + else + # Fugaku (Fujitsu MPI): -stdin FILE must be an mpiexec-level argument, + # not shell redirection -- plain `< file` only feeds rank 0's stdin + # under pjsub's mpiexec, not all ranks. + run_salmon_or_diagnose RT .rt_start_marker rt.log -stdin "${tddft_nml}" ./salmon + fi rt_end=$(date +%s.%N) rt_elapsed=$(awk -v start="${rt_start}" -v end="${rt_end}" 'BEGIN {printf "%.6f", end - start}') From 6d9e87bedf58ec796e8cd342563af1c91972e929 Mon Sep 17 00:00:00 2001 From: William Dawson Date: Tue, 18 Aug 2026 00:04:06 +0900 Subject: [PATCH 05/12] [code:salmon] RIKYU: pin to optimized+265-fix branch, drop broken USE_CUDA v.2.2.2 (BenchKit's global VERSION_TAG) is over a year stale and was never actually what RIKYU's prior GPU-decomposition numbers were measured against -- those came from a hand-patched checkout ~100 commits ahead (develop-2.0.0@9b93a8c4 + PR #1276's OpenACC tuning), never landed here. Also, -DUSE_CUDA=ON was actively wrong for RIKYU: it selects SALMON's old hand-written CUDA kernels (zpseudo.cu, stencil_current.cu), which measured net *slower* than plain OpenACC at this problem size (pseudo-pt alone is 3.1-3.5x slower under CUDA), and stencil_current.cu's host-side wrapper has a real, deterministic device-buffer bug: it sizes its idx/idy/idz buffers by each rank's LOCAL grid extent but indexes them with the RAW/global grid coordinate. That only happens to fit when a rank owns index 1 on that axis (single GPU, or orbital-only decomposition) -- any real-space (nproc_rgrid>1) decomposition puts a non-first rank out of bounds and reproduces a deterministic CUDA_ERROR_ILLEGAL_ADDRESS in calc_current. Reproduced on every axis and Po x Pg combination tried; root-caused by reading stencil_current.cu directly. Fix: pin RIKYU specifically (not the other systems, which stay on stock v.2.2.2 pending their own re-validation) to a new tag, v.2.2.2-rikyu-optimized-265 on william-dawson/SALMON2 -- PR #1276's stencil/current/pseudo-pt OpenACC optimizations (batched cuBLAS GEMM for pseudo-pt via -DUSE_GEMM, not USE_CUDA) cherry-picked cleanly onto develop-2.0.0@9b93a8c4, combined with the nvhpc/26.5 Ewald-reduction fix already used here. This sidesteps the crash structurally (the buggy CUDA kernels aren't even compiled) while also being the actually-faster path. Verified on real hardware: build succeeds, and the three configs that previously crashed (y-axis and z-axis real-space decomposition, and a mixed Po x Pg case) all run correctly with matching energies -- see the subwg2-benchmarks salmon-gpu-optimization-ideas skill (Open item 5) for the full trail and numbers. Signed-off-by: William Dawson --- programs/salmon/build.sh | 46 +++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 93ea40af..57dd05af 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -7,6 +7,22 @@ REPO_URL="https://github.com/SALMON-TDDFT/SALMON2" REPO_DIR="SALMON2" VERSION_TAG="v.2.2.2" BUILD_DIR="build-benchkit" + +# RIKYU pin: v.2.2.2 is over a year stale (2025-06-06) and measured +# GPU-decomposition numbers on this repo were never actually run against it -- +# they came from a much newer, hand-patched checkout (~100 commits ahead, +# including PR #1276's OpenACC tuning). Rather than let RIKYU silently build +# something nobody has benchmarked, pin it to a tag that IS what was +# benchmarked: develop-2.0.0@9b93a8c4 (2026-08-16) + PR #1276's stencil/ +# current/pseudo-pt OpenACC optimizations (batched cuBLAS GEMM for pseudo-pt, +# not the old USE_CUDA hand-written kernels -- see below) + the nvhpc/26.5 +# Ewald-reduction fix, combined into one branch on william-dawson/SALMON2. +# See .claude/skills/salmon-gpu-optimization-ideas and salmon-build in +# subwg2-benchmarks for how each piece was measured/root-caused. +if [[ "${system}" == "RIKYU" ]]; then + REPO_URL="https://github.com/william-dawson/SALMON2" + VERSION_TAG="v.2.2.2-rikyu-optimized-265" +fi ARTIFACT_DIR="${PWD}/artifacts" RESULTS_DIR="${PWD}/results" BUILD_LOG_DIR="${RESULTS_DIR}/salmon_build_logs" @@ -198,18 +214,38 @@ case "${system}" in -DCMAKE_C_COMPILER=mpicc -DOPENMP_FLAGS=-Mnoopenmp -DUSE_OPENACC=ON - -DUSE_CUDA=ON -DUSE_MPI_DEFAULT=ON -DCMAKE_SYSTEM_PROCESSOR=openacc - -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas -cuda -Minfo=accel -DUSE_OPENACC -DUSE_CUDA" - -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas -cuda -Minfo=accel -DUSE_OPENACC -DUSE_CUDA" - -DCMAKE_CUDA_ARCHITECTURES=100 - -DCMAKE_CUDA_FLAGS=-arch=sm_100 + -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM" + -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM" # nvhpc/26.5, native MPI3 ON (no FORTRAN_COMPILER_HAS_MPI_VERSION3 # override): the HPC-X libnbc bug that forced MPI3 off is 26.3-only. # Confirmed bit-exact and faster than the old 26.3+MPI3-off build on # a 2-GPU domain-decomposition case -- see nvhpc265-ewald-reduction # patch header and subwg2-benchmarks' salmon-build skill. + # + # NOT -DUSE_CUDA -- that flag controls a completely different, OLDER + # optimization path (src/common/{zpseudo,stencil_current}.cu, hand- + # written CUDA kernels) that this pinned source (see VERSION_TAG + # above) replaces with PR #1276's tuned pure-OpenACC kernels instead: + # a batched cuBLAS GEMM rewrite of pseudo-pt (-DUSE_GEMM, needs + # cusolver linked in) and an inlined OpenACC current-density kernel. + # That's where the real speedup comes from, not USE_CUDA -- measured + # data only ever showed the OLD CUDA kernels net *losing* time + # (pseudo-pt 3.1-3.5x slower under USE_CUDA than plain OpenACC; the + # 1.4-3x win on current-density wasn't enough to make up for it). + # USE_CUDA also has a real, deterministic bug independent of any of + # this: stencil_current.cu's host wrapper sizes its device idx/idy/idz + # buffers by each rank's LOCAL grid extent but indexes them with the + # RAW/global grid coordinate, which only happens to fit when a rank's + # is()=1 on that axis (single GPU, or pure orbital decomposition, + # where every rank owns the full box). Any real-space (nproc_rgrid>1) + # decomposition puts a non-first rank at is()>1 on the split axis and + # overruns the buffer -- reproduced as a deterministic Accelerator + # Fatal Error / CUDA_ERROR_ILLEGAL_ADDRESS in calc_current + # (density_matrix.f90) on every axis and Po x Pg combination tried. + # See subwg2-benchmarks' salmon-gpu-optimization-ideas skill (Open + # item 5) and salmon-build skill for the full trail on both points. ) ;; *) From 016bf59c58e50955b0744550bd76a66f7040b6d6 Mon Sep 17 00:00:00 2001 From: William Dawson Date: Tue, 18 Aug 2026 08:38:23 +0900 Subject: [PATCH 06/12] [code:salmon] RIKYU: switch to nvhpc/26.3 + MPI3-off, fixes 8-GPU hang native MPI3 on nvhpc/26.5 (the previous pin) is faster for single-node and domain-decomposition cases, but genuinely hangs 8-GPU (2+ node) orbital decomposition: it dies right after init_ps with a UCC inter-node protocol error (cannot find remote protocol for: UCC_UCP_CONTEXT inter-node cfg#N | tag_send from cuda-managed/GPU0), then spins at ~99% CPU with zero further output. Confirmed this is not the previously-documented version-mixing artifact (that combo mixed a 26.3 binary with a 26.5 runtime; this hang reproduces on a clean, matched, native 26.5 build) and not a GPU-binding bug (verified correct per-rank CUDA_VISIBLE_DEVICES assignment via an instrumented wrapper.sh, no collisions). It's specific to native MPI3 at 2+ nodes on 26.5, which had never actually been exercised at multi-node scale on this project before. nvhpc/26.3 + FORTRAN_COMPILER_HAS_MPI_VERSION3=OFF is the one combination proven correct and hang-free at every scale BenchKit actually needs: 1/2/4/8-GPU all bit-exact on this pinned source (8-GPU orbital: 50.13s, matching -164703.595116277 eV), and 16/32-GPU proven on an earlier source revision (see rikyu-32gpu-scaling skill). The nvhpc/26.5 Ewald-reduction patch already applied here is a no-op on 26.3 (its version gate only fires for nvhpc>=26.5), so nothing needs to change about how it's applied. list.csv only exercises orbital decomposition, so the domain- decomposition speed edge nvhpc/26.5+MPI3-on had is not something BenchKit's own benchmark rows would ever have used anyway -- this trades it for actually working at the node counts list.csv covers. Signed-off-by: William Dawson --- programs/salmon/build.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 57dd05af..cb56383e 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -206,7 +206,7 @@ case "${system}" in # ;; RIKYU) module purge - module load nvhpc/26.5 + module load nvhpc/26.3 apply_ewald_265_patch cmake_args=( "${common_cmake_args[@]}" @@ -216,13 +216,27 @@ case "${system}" in -DUSE_OPENACC=ON -DUSE_MPI_DEFAULT=ON -DCMAKE_SYSTEM_PROCESSOR=openacc + -DFORTRAN_COMPILER_HAS_MPI_VERSION3=OFF -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM" -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM" - # nvhpc/26.5, native MPI3 ON (no FORTRAN_COMPILER_HAS_MPI_VERSION3 - # override): the HPC-X libnbc bug that forced MPI3 off is 26.3-only. - # Confirmed bit-exact and faster than the old 26.3+MPI3-off build on - # a 2-GPU domain-decomposition case -- see nvhpc265-ewald-reduction - # patch header and subwg2-benchmarks' salmon-build skill. + # nvhpc/26.3, not 26.5, and FORTRAN_COMPILER_HAS_MPI_VERSION3=OFF: + # native MPI3 on nvhpc/26.5 is faster for single-node/domain- + # decomposition (see nvhpc265-ewald-reduction patch header and + # subwg2-benchmarks' salmon-build skill) but genuinely HANGS 8-GPU + # (2+ node) orbital decomposition -- reproducibly dies right after + # init_ps with a UCC inter-node protocol error + # (`cannot find remote protocol for: UCC_UCP_CONTEXT inter-node + # cfg#N | tag_send from cuda-managed/GPU0`), then spins at ~99% CPU + # producing zero further output. Confirmed NOT a version-mixing + # artifact (this build mixes nothing -- 26.3 compiler, 26.3 runtime) + # and NOT a GPU-binding bug (verified correct per-rank binding via + # an instrumented wrapper.sh) -- it's specific to native MPI3 at + # 2+ nodes. 26.3+MPI3-off is the one combination proven correct and + # hang-free at every scale BenchKit actually needs here (1, 2, 4, 8 + # GPU all bit-exact on this pinned source; 16/32 GPU proven on an + # earlier source revision -- see rikyu-32gpu-scaling skill). The + # Ewald patch above is a no-op on 26.3 (its version gate only fires + # for nvhpc>=26.5) -- harmless to keep applying regardless. # # NOT -DUSE_CUDA -- that flag controls a completely different, OLDER # optimization path (src/common/{zpseudo,stencil_current}.cu, hand- From 3de701ac2e21798f11c7a4a40b0ee5b0a30ddee9 Mon Sep 17 00:00:00 2001 From: William Dawson Date: Tue, 18 Aug 2026 11:21:29 +0900 Subject: [PATCH 07/12] [code:salmon] RIKYU: pin to FugakuNEXT-v1, back to native 26.5 MPI3-on FugakuNEXT-v1 (also open upstream as SALMON-TDDFT/SALMON2#1276) is the single consolidated branch replacing the ad-hoc chain this pin has gone through (v.2.2.2-rikyu-optimized-265 -> the nvhpc/26.3+MPI3-off workaround -> this): develop-2.0.0@9b93a8c4 plus four commits, one per concern -- the OpenACC tuning, the cmake wiring for it, the nvhpc/26.5 Ewald-reduction-bug guard, and (new) a real fix for the 2+ node UCC hang that previously forced the 26.3+MPI3-off downgrade. That fix (acc_init(acc_device_nvidia) before MPI_Init_thread, so UCC's CUDA-aware protocol probe during MPI_Init's team bootstrap doesn't run contextless and cache a broken config) means the downgrade is no longer needed: back to nvhpc/26.5, native MPI3 ON, no FORTRAN_COMPILER_HAS_MPI_VERSION3 override. This is the actually-tested combination now -- orbital decomposition 1-8 nodes and domain decomposition 1-2 nodes, all bit-exact, all previously-hanging configurations included. Signed-off-by: William Dawson --- programs/salmon/build.sh | 55 ++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index cb56383e..6519d668 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -12,16 +12,23 @@ BUILD_DIR="build-benchkit" # GPU-decomposition numbers on this repo were never actually run against it -- # they came from a much newer, hand-patched checkout (~100 commits ahead, # including PR #1276's OpenACC tuning). Rather than let RIKYU silently build -# something nobody has benchmarked, pin it to a tag that IS what was -# benchmarked: develop-2.0.0@9b93a8c4 (2026-08-16) + PR #1276's stencil/ -# current/pseudo-pt OpenACC optimizations (batched cuBLAS GEMM for pseudo-pt, -# not the old USE_CUDA hand-written kernels -- see below) + the nvhpc/26.5 -# Ewald-reduction fix, combined into one branch on william-dawson/SALMON2. +# something nobody has benchmarked, pin it to FugakuNEXT-v1 on +# william-dawson/SALMON2 (also open as SALMON-TDDFT/SALMON2#1276) -- +# develop-2.0.0@9b93a8c4 (2026-08-16) plus four commits, one per concern: +# 1. PR #1276's stencil/current/pseudo-pt OpenACC tuning (batched cuBLAS +# GEMM for pseudo-pt, not the old USE_CUDA hand-written kernels -- +# see below) +# 2. the nvhpc-openacc-gemm.cmake platform file that wires it up +# 3. the nvhpc/26.5 Ewald-reduction compiler-bug workaround +# 4. a fix for a real, 100%-reproducible 2+ node hang on native +# nvhpc/26.5 (forces CUDA context creation before MPI_Init_thread -- +# see below, and salmon-gpu-optimization-ideas' Open item 5 in +# subwg2-benchmarks for the full root-cause trail) # See .claude/skills/salmon-gpu-optimization-ideas and salmon-build in # subwg2-benchmarks for how each piece was measured/root-caused. if [[ "${system}" == "RIKYU" ]]; then REPO_URL="https://github.com/william-dawson/SALMON2" - VERSION_TAG="v.2.2.2-rikyu-optimized-265" + VERSION_TAG="FugakuNEXT-v1" fi ARTIFACT_DIR="${PWD}/artifacts" RESULTS_DIR="${PWD}/results" @@ -206,7 +213,7 @@ case "${system}" in # ;; RIKYU) module purge - module load nvhpc/26.3 + module load nvhpc/26.5 apply_ewald_265_patch cmake_args=( "${common_cmake_args[@]}" @@ -216,27 +223,25 @@ case "${system}" in -DUSE_OPENACC=ON -DUSE_MPI_DEFAULT=ON -DCMAKE_SYSTEM_PROCESSOR=openacc - -DFORTRAN_COMPILER_HAS_MPI_VERSION3=OFF -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM" -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM" - # nvhpc/26.3, not 26.5, and FORTRAN_COMPILER_HAS_MPI_VERSION3=OFF: - # native MPI3 on nvhpc/26.5 is faster for single-node/domain- - # decomposition (see nvhpc265-ewald-reduction patch header and - # subwg2-benchmarks' salmon-build skill) but genuinely HANGS 8-GPU - # (2+ node) orbital decomposition -- reproducibly dies right after - # init_ps with a UCC inter-node protocol error + # nvhpc/26.5, native MPI3 ON (no FORTRAN_COMPILER_HAS_MPI_VERSION3 + # override needed): this used to hang 2+ node orbital decomposition + # -- died right after init_ps with a UCC inter-node protocol error # (`cannot find remote protocol for: UCC_UCP_CONTEXT inter-node - # cfg#N | tag_send from cuda-managed/GPU0`), then spins at ~99% CPU - # producing zero further output. Confirmed NOT a version-mixing - # artifact (this build mixes nothing -- 26.3 compiler, 26.3 runtime) - # and NOT a GPU-binding bug (verified correct per-rank binding via - # an instrumented wrapper.sh) -- it's specific to native MPI3 at - # 2+ nodes. 26.3+MPI3-off is the one combination proven correct and - # hang-free at every scale BenchKit actually needs here (1, 2, 4, 8 - # GPU all bit-exact on this pinned source; 16/32 GPU proven on an - # earlier source revision -- see rikyu-32gpu-scaling skill). The - # Ewald patch above is a no-op on 26.3 (its version gate only fires - # for nvhpc>=26.5) -- harmless to keep applying regardless. + # cfg#N | tag_send from cuda-managed/GPU0`), then spun at ~99% CPU + # producing zero further output -- but that's now fixed at the + # source (FugakuNEXT-v1's 4th commit: acc_init(acc_device_nvidia) + # before MPI_Init_thread, so UCC's CUDA-aware protocol probe during + # MPI_Init's team bootstrap doesn't run with no CUDA context and + # cache a broken config). Root-caused via a live gdb backtrace on + # the hung process and compute-sanitizer on the real binary -- see + # subwg2-benchmarks' salmon-gpu-optimization-ideas skill, Open item + # 5, for the full trail, and don't reintroduce the old + # nvhpc/26.3 + FORTRAN_COMPILER_HAS_MPI_VERSION3=OFF workaround this + # replaces: native 26.5 MPI3 is faster and this pin is what's + # actually been verified end-to-end (orbital 1-8 nodes, domain 1-2 + # nodes, all bit-exact, on the real build.sh + real artifact). # # NOT -DUSE_CUDA -- that flag controls a completely different, OLDER # optimization path (src/common/{zpseudo,stencil_current}.cu, hand- From 9d7ce8e79caef80667a776ba11e7e78f536e081d Mon Sep 17 00:00:00 2001 From: William Dawson Date: Wed, 19 Aug 2026 23:25:10 +0900 Subject: [PATCH 08/12] [code:salmon] RIKYU: pin FugakuNEXT-v2, mixed decomposition, NCCL reduce Pin to FugakuNEXT-v2, which carries every fix RIKYU needs, and build it unpatched -- the Ewald workaround is in the branch, so patching on top of it was fragile (the reverse-apply check broke as soon as the branch reworded a comment). Non-RIKYU systems still patch stock v.2.2.2, and that helper now detects the fix by its macro rather than by patch context. Switch to nvhpc-hpcx-cuda13/26.5 (HPC-X 2.50 / OpenMPI 5), the stack all of this was measured on, and add -DUSE_NCCL: MPI_Allreduce does its reduction arithmetic on the host even given a device pointer, so the pseudo-pt reduce left NVLink. NCCL takes it from 18.2s to 1.7s at 4 GPUs. Run with mixed decomposition -- grid split 2x2x1 inside a node, orbitals across nodes, process_allocation='grid_sequential' so each icomm_r is one node's 4 ranks. Only the grid split reduces calculating-curr, which does not scale under orbital decomposition. rt iterations vs pure orbital: 8 GPUs 56.2 -> 48.0 s, 16 GPUs 44.6 -> 31.0 s, energies bit-exact. Add --mca fcoll individual, which is required: the restart read goes into managed memory and OMPIO's two-phase collective I/O redistributes it through cuMemcpyAsync, deadlocking whenever the nproc_rgrid product exceeds 2. Drop UCX_IB_GPU_DIRECT_RDMA=no, which was a 26.3 fix and is not needed on 26.5 (ablation-tested at 4, 8, 16 and 32 GPUs). Verified with the real build.sh/run.sh on RIKYU: 1 node FOM 133.18 s, 2 nodes FOM 97.73 s. Co-Authored-By: Claude Opus 5 Signed-off-by: William Dawson --- programs/salmon/build.sh | 57 ++++++++++++++++++++++------------------ programs/salmon/run.sh | 38 ++++++++++++++++++++++----- 2 files changed, 62 insertions(+), 33 deletions(-) diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 6519d668..97497f58 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -28,7 +28,7 @@ BUILD_DIR="build-benchkit" # subwg2-benchmarks for how each piece was measured/root-caused. if [[ "${system}" == "RIKYU" ]]; then REPO_URL="https://github.com/william-dawson/SALMON2" - VERSION_TAG="FugakuNEXT-v1" + VERSION_TAG="FugakuNEXT-v2" fi ARTIFACT_DIR="${PWD}/artifacts" RESULTS_DIR="${PWD}/results" @@ -44,7 +44,12 @@ mkdir -p "${BUILD_LOG_DIR}" bk_fetch_source "${REPO_URL}" "${REPO_DIR}" "${VERSION_TAG}" cd "${REPO_DIR}" -if git apply --check "${FJMPI_PATCH}"; then +# RIKYU builds a pinned branch that already carries every fix it needs +# (see VERSION_TAG above), so it is used as-is -- no patching. The patches +# below exist for the systems that build stock v.2.2.2. +if [[ "${system}" == "RIKYU" ]]; then + echo "RIKYU: building ${VERSION_TAG} unpatched (all fixes are in the branch)" +elif git apply --check "${FJMPI_PATCH}"; then git apply "${FJMPI_PATCH}" elif git apply --reverse --check "${FJMPI_PATCH}" >/dev/null 2>&1; then echo "SALMON Fujitsu MPI topology patch is already applied" @@ -60,10 +65,11 @@ apply_ewald_265_patch() { # Safe to apply on every OpenACC/GPU build regardless of nvhpc version: # the fix gates itself on __NVCOMPILER_MAJOR__/__NVCOMPILER_MINOR__ at # compile time, so it's a no-op on unaffected compilers (<26.5). - if git apply --check "${EWALD_265_PATCH}"; then + # (RIKYU does not call this -- its pinned branch already contains the fix.) + if grep -q "SALMON_EWALD_ACC" src/common/total_energy.f90; then + echo "SALMON nvhpc/26.5 Ewald-reduction workaround already present" + elif git apply --check "${EWALD_265_PATCH}"; then git apply "${EWALD_265_PATCH}" - elif git apply --reverse --check "${EWALD_265_PATCH}" >/dev/null 2>&1; then - echo "SALMON nvhpc/26.5 Ewald-reduction patch is already applied" else echo "SALMON nvhpc/26.5 Ewald-reduction patch does not apply to ${VERSION_TAG}" >&2 exit 1 @@ -213,8 +219,7 @@ case "${system}" in # ;; RIKYU) module purge - module load nvhpc/26.5 - apply_ewald_265_patch + module load nvhpc-hpcx-cuda13/26.5 cmake_args=( "${common_cmake_args[@]}" -DCMAKE_Fortran_COMPILER=mpif90 @@ -223,25 +228,25 @@ case "${system}" in -DUSE_OPENACC=ON -DUSE_MPI_DEFAULT=ON -DCMAKE_SYSTEM_PROCESSOR=openacc - -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM" - -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM" - # nvhpc/26.5, native MPI3 ON (no FORTRAN_COMPILER_HAS_MPI_VERSION3 - # override needed): this used to hang 2+ node orbital decomposition - # -- died right after init_ps with a UCC inter-node protocol error - # (`cannot find remote protocol for: UCC_UCP_CONTEXT inter-node - # cfg#N | tag_send from cuda-managed/GPU0`), then spun at ~99% CPU - # producing zero further output -- but that's now fixed at the - # source (FugakuNEXT-v1's 4th commit: acc_init(acc_device_nvidia) - # before MPI_Init_thread, so UCC's CUDA-aware protocol probe during - # MPI_Init's team bootstrap doesn't run with no CUDA context and - # cache a broken config). Root-caused via a live gdb backtrace on - # the hung process and compute-sanitizer on the real binary -- see - # subwg2-benchmarks' salmon-gpu-optimization-ideas skill, Open item - # 5, for the full trail, and don't reintroduce the old - # nvhpc/26.3 + FORTRAN_COMPILER_HAS_MPI_VERSION3=OFF workaround this - # replaces: native 26.5 MPI3 is faster and this pin is what's - # actually been verified end-to-end (orbital 1-8 nodes, domain 1-2 - # nodes, all bit-exact, on the real build.sh + real artifact). + -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver,nccl -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM -DUSE_NCCL" + -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver,nccl -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM -DUSE_NCCL" + # Module is nvhpc-hpcx-cuda13/26.5 (HPC-X 2.50 / OpenMPI 5), not + # plain nvhpc/26.5: that is the stack the domain-decomposition and + # NCCL numbers were measured on. MPI3 is left at its auto-detected + # ON -- no FORTRAN_COMPILER_HAS_MPI_VERSION3 override -- because the + # GEMM path skips calc_uVpsi_rdivided entirely, so the HPC-X libnbc + # bug that motivated the old 26.3 + MPI3=OFF workaround is + # unreachable. Do not reintroduce that workaround. + # + # -DUSE_NCCL routes the pseudo-pt domain-decomposition reduce through + # ncclAllReduce instead of MPI_Allreduce. MPI_Allreduce performs the + # reduction arithmetic on the host even when handed a device pointer, + # so it leaves NVLink: measured 5.6 GB/s vs NCCL's 196-521 GB/s on the + # same 222 MB device buffer. In SALMON that is pseudo-pt comm 18.2s -> + # 1.7s at 4 GPUs (rt iterations 99.1 -> 80.5), energies bit-exact. + # It needs nccl in -cudalib. If the MPI stack ever grows a working + # GPU-side allreduce (UCC ships a TL_NCCL that currently never + # registers), drop -DUSE_NCCL and this becomes plain MPI again. # # NOT -DUSE_CUDA -- that flag controls a completely different, OLDER # optimization path (src/common/{zpseudo,stencil_current}.cu, hand- diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index 23d06dea..bdf0579d 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -184,14 +184,31 @@ case "${system}" in ;; RIKYU) module purge - module load nvhpc/26.5 + module load nvhpc-hpcx-cuda13/26.5 export OMP_NUM_THREADS="${nthreads}" - awk -v nproc_ob="${n_ranks}" ' - /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = 1, 1, 1"; in_parallel=1; next } + # Mixed decomposition: split the real-space grid 2x2x1 *within* a node + # (4 GPUs, NVLink) and distribute orbitals across nodes. Only the grid + # split reduces calculating-curr, which does not scale under orbital + # decomposition at all (109.5 -> 44.2 ms/iter at 4 GPUs); with the NCCL + # reduce the pseudo-pt cost of splitting is down to ~5 ms/iter. + # Measured on 3x3x3 SiO2, rt iterations vs pure orbital: 8 GPUs + # 56.2 -> 48.0 s, 16 GPUs 44.6 -> 31.0 s, all bit-exact. + # + # process_allocation='grid_sequential' is REQUIRED: it orders ranks + # grid-fastest so each icomm_r is one node's 4 ranks. The default + # 'orbital_sequential' strides icomm_r across nodes, which puts the + # halo exchange on InfiniBand instead of NVLink. + if [[ "${numproc_node}" -eq 4 ]]; then + salmon_nproc_ob="${nodes}"; salmon_rgrid="2, 2, 1"; salmon_alloc="grid_sequential" + else + salmon_nproc_ob="${n_ranks}"; salmon_rgrid="1, 1, 1"; salmon_alloc="orbital_sequential" + fi + awk -v nproc_ob="${salmon_nproc_ob}" -v rgrid="${salmon_rgrid}" -v alloc="${salmon_alloc}" ' + /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = " rgrid; print " process_allocation = " q alloc q; in_parallel=1; next } in_parallel && /^\// { in_parallel=0; print; next } in_parallel { next } { print } - ' "${tddft_nml}" > "${tddft_nml}.tmp" && mv "${tddft_nml}.tmp" "${tddft_nml}" + ' q="'" "${tddft_nml}" > "${tddft_nml}.tmp" && mv "${tddft_nml}.tmp" "${tddft_nml}" if [[ "${n_ranks}" -gt 1 ]]; then cat > wrapper.sh <<'WRAPPER' #!/bin/bash @@ -206,8 +223,10 @@ export CUDA_VISIBLE_DEVICES=$((${OMPI_COMM_WORLD_LOCAL_RANK} % ${NCUDA_GPUS})) exec "$@" WRAPPER chmod +x wrapper.sh - export UCX_IB_GPU_DIRECT_RDMA=no fi + # UCX_IB_GPU_DIRECT_RDMA=no was needed on 26.3 to stop a multi-node + # hang. Ablation-tested on 26.5 (4-GPU domain, 8-GPU mixed across 2 + # nodes, and 16/32 GPUs): not needed, and neither is UCX_TLS=^cma. ;; # RC_FX700) # FX700 currently fails during GS initialization even with the Fujitsu @@ -224,10 +243,15 @@ run_salmon() { shift case "${system}" in RIKYU) + # --mca fcoll individual is REQUIRED. MPI_File_read_all reads the + # restart straight into managed memory and OMPIO's default two-phase + # collective I/O redistributes it through cuMemcpyAsync: that + # deadlocks whenever the nproc_rgrid product exceeds 2, and even when + # it does not it costs 66.9s vs 41.7s on the restart read. if [[ "${n_ranks}" -gt 1 ]]; then - mpirun -n "${n_ranks}" ./wrapper.sh "$@" > "${logfile}" 2>&1 + mpirun -n "${n_ranks}" --mca fcoll individual ./wrapper.sh "$@" > "${logfile}" 2>&1 else - mpirun -n "${n_ranks}" "$@" > "${logfile}" 2>&1 + mpirun -n "${n_ranks}" --mca fcoll individual "$@" > "${logfile}" 2>&1 fi ;; RC_GENOA) From f89c71febfb4d25b0c7ed499ead0931fd900b7c6 Mon Sep 17 00:00:00 2001 From: William Dawson Date: Thu, 20 Aug 2026 01:04:06 +0900 Subject: [PATCH 09/12] [code:salmon] RIKYU: benchmark both decompositions, FOM from rt iterations Benchmark orbital and mixed decomposition rather than picking one. They trade places depending on the problem and on which kernels have been optimised -- orbital is currently faster at every node count, but mixed is what a problem with less orbital parallelism falls back on -- so a benchmark should track both. Each emits its own result line, Exp ...-orbital and ...-mixed. Take the FOM from SALMON's rt-iterations timer instead of wall clock. The one-time restart read is 40-150 s depending on whether Lustre is cold, which both swamps the RT loop and, with two runs in one job, makes the second look far faster than the first purely from page cache. Production runs do very many steps, so the RT loop is the quantity of interest. fom-version becomes rt_iterations_s_folded_restart. Also bump the pin to FugakuNEXT-v3, which computes the nonlocal current density with a batched GEMM: 29.5 -> 3.6 s on that kernel and 44.6 -> 18.7 s on rt iterations at 16 GPUs orbital. Verified on RIKYU, 1 node: orbital 81.84 s, mixed 80.59 s (v2 binary), matching standalone measurements of the same layouts. Co-Authored-By: Claude Opus 5 Signed-off-by: William Dawson --- programs/salmon/build.sh | 8 ++- programs/salmon/run.sh | 121 +++++++++++++++++++++------------------ 2 files changed, 72 insertions(+), 57 deletions(-) diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 97497f58..4adb3728 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -12,7 +12,7 @@ BUILD_DIR="build-benchkit" # GPU-decomposition numbers on this repo were never actually run against it -- # they came from a much newer, hand-patched checkout (~100 commits ahead, # including PR #1276's OpenACC tuning). Rather than let RIKYU silently build -# something nobody has benchmarked, pin it to FugakuNEXT-v1 on +# something nobody has benchmarked, pin it to FugakuNEXT-v3 on # william-dawson/SALMON2 (also open as SALMON-TDDFT/SALMON2#1276) -- # develop-2.0.0@9b93a8c4 (2026-08-16) plus four commits, one per concern: # 1. PR #1276's stencil/current/pseudo-pt OpenACC tuning (batched cuBLAS @@ -28,7 +28,7 @@ BUILD_DIR="build-benchkit" # subwg2-benchmarks for how each piece was measured/root-caused. if [[ "${system}" == "RIKYU" ]]; then REPO_URL="https://github.com/william-dawson/SALMON2" - VERSION_TAG="FugakuNEXT-v2" + VERSION_TAG="FugakuNEXT-v3" fi ARTIFACT_DIR="${PWD}/artifacts" RESULTS_DIR="${PWD}/results" @@ -238,6 +238,10 @@ case "${system}" in # bug that motivated the old 26.3 + MPI3=OFF workaround is # unreachable. Do not reintroduce that workaround. # + # FugakuNEXT-v3 also computes the nonlocal current density with a + # batched GEMM (it was 82% of calculating-curr): current density + # 29.5 -> 3.6 s and rt iterations 44.6 -> 18.7 s at 16 GPUs orbital. + # # -DUSE_NCCL routes the pseudo-pt domain-decomposition reduce through # ncclAllReduce instead of MPI_Allreduce. MPI_Allreduce performs the # reduction arithmetic on the host even when handed a device pointer, diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index bdf0579d..719cff0e 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -186,29 +186,7 @@ case "${system}" in module purge module load nvhpc-hpcx-cuda13/26.5 export OMP_NUM_THREADS="${nthreads}" - # Mixed decomposition: split the real-space grid 2x2x1 *within* a node - # (4 GPUs, NVLink) and distribute orbitals across nodes. Only the grid - # split reduces calculating-curr, which does not scale under orbital - # decomposition at all (109.5 -> 44.2 ms/iter at 4 GPUs); with the NCCL - # reduce the pseudo-pt cost of splitting is down to ~5 ms/iter. - # Measured on 3x3x3 SiO2, rt iterations vs pure orbital: 8 GPUs - # 56.2 -> 48.0 s, 16 GPUs 44.6 -> 31.0 s, all bit-exact. - # - # process_allocation='grid_sequential' is REQUIRED: it orders ranks - # grid-fastest so each icomm_r is one node's 4 ranks. The default - # 'orbital_sequential' strides icomm_r across nodes, which puts the - # halo exchange on InfiniBand instead of NVLink. - if [[ "${numproc_node}" -eq 4 ]]; then - salmon_nproc_ob="${nodes}"; salmon_rgrid="2, 2, 1"; salmon_alloc="grid_sequential" - else - salmon_nproc_ob="${n_ranks}"; salmon_rgrid="1, 1, 1"; salmon_alloc="orbital_sequential" - fi - awk -v nproc_ob="${salmon_nproc_ob}" -v rgrid="${salmon_rgrid}" -v alloc="${salmon_alloc}" ' - /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = " rgrid; print " process_allocation = " q alloc q; in_parallel=1; next } - in_parallel && /^\// { in_parallel=0; print; next } - in_parallel { next } - { print } - ' q="'" "${tddft_nml}" > "${tddft_nml}.tmp" && mv "${tddft_nml}.tmp" "${tddft_nml}" + # Layout is set per RT run below (RIKYU benchmarks two of them). if [[ "${n_ranks}" -gt 1 ]]; then cat > wrapper.sh <<'WRAPPER' #!/bin/bash @@ -322,40 +300,73 @@ if uses_prestaged_restart "${system}"; then # GS is offline prep, done once when the restart was staged -- see # salmon-benchmarking in subwg2-benchmarks for how/when to regenerate # it. The benchmark itself is TDDFT-only. - touch .rt_start_marker - rt_start=$(date +%s.%N) - if uses_stdin_input "${system}"; then - run_salmon_or_diagnose RT .rt_start_marker rt.log ./salmon < "${tddft_nml}" + # Rewrite the ¶llel block in place. + salmon_set_parallel () { + awk -v nproc_ob="$1" -v rgrid="$2" -v alloc="$3" ' + /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = " rgrid; print " process_allocation = " q alloc q; in_parallel=1; next } + in_parallel && /^\// { in_parallel=0; print; next } + in_parallel { next } + { print } + ' q="'" "${tddft_nml}" > "${tddft_nml}.tmp" && mv "${tddft_nml}.tmp" "${tddft_nml}" + } + + # One timed RT run at the current layout, emitting one result line. + run_rt_once () { + local label="$1" logfile="rt_$1.log" + touch .rt_start_marker + local t0 t1 + t0=$(date +%s.%N) + if uses_stdin_input "${system}"; then + run_salmon_or_diagnose RT .rt_start_marker "${logfile}" ./salmon < "${tddft_nml}" + else + # Fugaku (Fujitsu MPI): -stdin FILE must be an mpiexec-level argument, + # not shell redirection -- plain `< file` only feeds rank 0's stdin + # under pjsub's mpiexec, not all ranks. + run_salmon_or_diagnose RT .rt_start_marker "${logfile}" -stdin "${tddft_nml}" ./salmon + fi + t1=$(date +%s.%N) + cp "${logfile}" "${RESULTS_DIR}/" + if ! salmon_output_has_marker_since "${logfile}" .rt_start_marker; then + echo "SALMON RT run failed (${label})" >&2 + tail -n 40 "${logfile}" >&2 || true + print_salmon_output_diagnostics .rt_start_marker + exit 1 + fi + # FOM is SALMON's own rt-iterations timer, not wall clock: the one-time + # restart read is ~40-150 s depending on whether Lustre is cold, which + # swamps and reorders the comparison when two layouts run in one job. + # Production runs do very many steps, so the RT loop is what matters. + local rt_s + rt_s=$(awk '/^rt iterations/ {print $(NF-3); exit}' "${logfile}") + if [[ -z "${rt_s}" ]]; then + echo "could not read 'rt iterations' from ${logfile}" >&2 + exit 1 + fi + bk_emit_result \ + --fom "${rt_s}" \ + --fom-unit s \ + --fom-version "rt_iterations_s_folded_restart" \ + --exp "${tddft_nml%.nml}-${label}" \ + --nodes "${nodes}" \ + --numproc-node "${numproc_node}" \ + --nthreads "${nthreads}" \ + >> "${RESULTS_DIR}/result" + } + + if [[ "${system}" == "RIKYU" && "${numproc_node}" -eq 4 ]]; then + # Benchmark both decompositions. Orbital is currently the faster of the + # two at every node count measured, but mixed is what a problem with less + # orbital parallelism has to fall back on, so both are worth tracking. + # Mixed keeps the 2x2x1 grid split inside a node (NVLink) and spreads + # orbitals across nodes; process_allocation='grid_sequential' is required + # for that, since the default strides icomm_r across nodes. + salmon_set_parallel "${n_ranks}" "1, 1, 1" "orbital_sequential" + run_rt_once orbital + salmon_set_parallel "${nodes}" "2, 2, 1" "grid_sequential" + run_rt_once mixed else - # Fugaku (Fujitsu MPI): -stdin FILE must be an mpiexec-level argument, - # not shell redirection -- plain `< file` only feeds rank 0's stdin - # under pjsub's mpiexec, not all ranks. - run_salmon_or_diagnose RT .rt_start_marker rt.log -stdin "${tddft_nml}" ./salmon - fi - rt_end=$(date +%s.%N) - - rt_elapsed=$(awk -v start="${rt_start}" -v end="${rt_end}" 'BEGIN {printf "%.6f", end - start}') - - cp rt.log "${RESULTS_DIR}/" - - if ! salmon_output_has_marker_since rt.log .rt_start_marker; then - echo "SALMON RT run failed" >&2 - echo "---- rt.log tail ----" >&2 - tail -n 40 rt.log >&2 || true - echo "---- files updated since RT start ----" >&2 - print_salmon_output_diagnostics .rt_start_marker - exit 1 + run_rt_once default fi - - bk_emit_result \ - --fom "${rt_elapsed}" \ - --fom-unit s \ - --fom-version "tddft_elapsed_time_s_folded_restart" \ - --exp "${tddft_nml%.nml}" \ - --nodes "${nodes}" \ - --numproc-node "${numproc_node}" \ - --nthreads "${nthreads}" \ - >> "${RESULTS_DIR}/result" else touch .gs_start_marker gs_start=$(date +%s.%N) From f7f1a7b2abb08cc98e7e4d37dc20991e4956fabe Mon Sep 17 00:00:00 2001 From: William Dawson Date: Wed, 26 Aug 2026 09:42:30 +0900 Subject: [PATCH 10/12] Simplify comments and drop internal-repo references [code:salmon] The comments pointed at skills and documents in a separate, private repository that a reader of this repo cannot open, and explained root-cause history rather than what the code does. Rewrite them to be shorter and self-contained: state the reason a workaround exists, not the trail that found it. No functional change. Signed-off-by: William Dawson Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PEzS2ybD8VEnQa9Sq5qYUM --- programs/salmon/build.sh | 16 +++++----------- programs/salmon/run.sh | 15 ++++++--------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 4adb3728..77a3dc8b 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -20,12 +20,9 @@ BUILD_DIR="build-benchkit" # see below) # 2. the nvhpc-openacc-gemm.cmake platform file that wires it up # 3. the nvhpc/26.5 Ewald-reduction compiler-bug workaround -# 4. a fix for a real, 100%-reproducible 2+ node hang on native -# nvhpc/26.5 (forces CUDA context creation before MPI_Init_thread -- -# see below, and salmon-gpu-optimization-ideas' Open item 5 in -# subwg2-benchmarks for the full root-cause trail) -# See .claude/skills/salmon-gpu-optimization-ideas and salmon-build in -# subwg2-benchmarks for how each piece was measured/root-caused. +# 4. a fix for a 2+ node hang on nvhpc/26.5: the CUDA context is created +# before MPI_Init_thread, so the MPI layer's CUDA-awareness probe does +# not cache a transfer protocol that deadlocks on device buffers if [[ "${system}" == "RIKYU" ]]; then REPO_URL="https://github.com/william-dawson/SALMON2" VERSION_TAG="FugakuNEXT-v3" @@ -60,9 +57,8 @@ fi apply_ewald_265_patch() { # Works around a silent nvfortran 26.5 OpenACC reduction-codegen bug that - # gives a wrong (but plausible) total energy -- see the patch file itself - # and .claude/skills/salmon-build in subwg2-benchmarks for the full trail. - # Safe to apply on every OpenACC/GPU build regardless of nvhpc version: + # gives a wrong (but plausible) total energy -- see the patch file for + # details. Safe to apply on any OpenACC/GPU build regardless of version: # the fix gates itself on __NVCOMPILER_MAJOR__/__NVCOMPILER_MINOR__ at # compile time, so it's a no-op on unaffected compilers (<26.5). # (RIKYU does not call this -- its pinned branch already contains the fix.) @@ -272,8 +268,6 @@ case "${system}" in # overruns the buffer -- reproduced as a deterministic Accelerator # Fatal Error / CUDA_ERROR_ILLEGAL_ADDRESS in calc_current # (density_matrix.f90) on every axis and Po x Pg combination tried. - # See subwg2-benchmarks' salmon-gpu-optimization-ideas skill (Open - # item 5) and salmon-build skill for the full trail on both points. ) ;; *) diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index 719cff0e..623d3ef4 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -14,11 +14,10 @@ WORK_DIR="${PWD}/salmon_run" INPUT_ARCHIVE_CLOUD="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON/SALMON.tar.gz" AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" -# Pre-staged, python-folded restarts: GS(k) -> Python fold -> complex Gamma -# TDDFT restart, prepared once offline via `benchgen salmon create -# --python-fold` (see README.md / salmon-benchmarking in -# subwg2-benchmarks) and stored on each machine so the benchmark itself -# never pays for a from-scratch ground state. Add a new case here (and a +# Pre-staged folded restarts. The ground state is computed once offline and +# folded to a complex Gamma-point TDDFT restart (see README.md), then stored +# on each machine, so the benchmark runs TDDFT only and never pays for a +# from-scratch ground state. Add a new case here (and a # matching directory on that system) to move another system onto this # path -- see the RIKYU or RC_DGXSP entries for the shape a new one needs # (restart/, *.psp8, and one TDDFT .nml, all siblings in one directory). @@ -297,10 +296,8 @@ print_salmon_output_diagnostics() { } if uses_prestaged_restart "${system}"; then - # GS is offline prep, done once when the restart was staged -- see - # salmon-benchmarking in subwg2-benchmarks for how/when to regenerate - # it. The benchmark itself is TDDFT-only. - # Rewrite the ¶llel block in place. + # The ground state was computed offline when the restart was staged; the + # benchmark is TDDFT-only. Rewrite the ¶llel block in place. salmon_set_parallel () { awk -v nproc_ob="$1" -v rgrid="$2" -v alloc="$3" ' /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = " rgrid; print " process_allocation = " q alloc q; in_parallel=1; next } From 9ec29251d78d3880b473d06a64b735adf7059f09 Mon Sep 17 00:00:00 2001 From: William Dawson Date: Wed, 26 Aug 2026 10:16:15 +0900 Subject: [PATCH 11/12] Pin FugakuNEXT-v4 and drop the Ewald patch [code:salmon] The nvhpc/26.5 Ewald problem was misdiagnosed as a compiler bug. It is the documented 26.5 change to evaluating an OpenACC loop's tripcount on the device: three loops in total_energy.f90 took their bound from a derived-type member that is not device-resident, so they ran zero times and their reductions returned the identity. FugakuNEXT-v4 fixes it at the source by hoisting the bound into a local scalar; the loops stay on the GPU and 26.5 matches 26.3 bit-for-bit. Bump VERSION_TAG to FugakuNEXT-v4 and remove the patch, its apply function and both call sites. Removal is required rather than tidy: the function detected "already patched" by grepping for SALMON_EWALD_ACC, a macro v4 deletes, so it would have failed to match, then failed to apply against the fixed source, and exited non-zero. RC_GH200 and RC_DGXSP built stock v.2.2.2 with this patch applied. Both use compilers below 26.5 (25.7 and 26.3), where the patch was already a no-op, so dropping it changes nothing for them today -- but they will need the source fix if either is moved to 26.5 or later. Signed-off-by: William Dawson Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PEzS2ybD8VEnQa9Sq5qYUM --- programs/salmon/build.sh | 28 +---- .../patches/nvhpc265-ewald-reduction.patch | 100 ------------------ 2 files changed, 5 insertions(+), 123 deletions(-) delete mode 100644 programs/salmon/patches/nvhpc265-ewald-reduction.patch diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 77a3dc8b..3d900a90 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -12,27 +12,28 @@ BUILD_DIR="build-benchkit" # GPU-decomposition numbers on this repo were never actually run against it -- # they came from a much newer, hand-patched checkout (~100 commits ahead, # including PR #1276's OpenACC tuning). Rather than let RIKYU silently build -# something nobody has benchmarked, pin it to FugakuNEXT-v3 on +# something nobody has benchmarked, pin it to FugakuNEXT-v4 on # william-dawson/SALMON2 (also open as SALMON-TDDFT/SALMON2#1276) -- # develop-2.0.0@9b93a8c4 (2026-08-16) plus four commits, one per concern: # 1. PR #1276's stencil/current/pseudo-pt OpenACC tuning (batched cuBLAS # GEMM for pseudo-pt, not the old USE_CUDA hand-written kernels -- # see below) # 2. the nvhpc-openacc-gemm.cmake platform file that wires it up -# 3. the nvhpc/26.5 Ewald-reduction compiler-bug workaround +# 3. a fix for the Ewald loop tripcounts: nvfortran >= 26.5 evaluates +# an OpenACC tripcount on the device, where the derived-type member +# the bound came from is not resident, so the loops ran zero times # 4. a fix for a 2+ node hang on nvhpc/26.5: the CUDA context is created # before MPI_Init_thread, so the MPI layer's CUDA-awareness probe does # not cache a transfer protocol that deadlocks on device buffers if [[ "${system}" == "RIKYU" ]]; then REPO_URL="https://github.com/william-dawson/SALMON2" - VERSION_TAG="FugakuNEXT-v3" + VERSION_TAG="FugakuNEXT-v4" fi ARTIFACT_DIR="${PWD}/artifacts" RESULTS_DIR="${PWD}/results" BUILD_LOG_DIR="${RESULTS_DIR}/salmon_build_logs" AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" FJMPI_PATCH="${PWD}/programs/salmon/patches/fjmpi-topology-guard.patch" -EWALD_265_PATCH="${PWD}/programs/salmon/patches/nvhpc265-ewald-reduction.patch" source scripts/bk_functions.sh @@ -55,23 +56,6 @@ else exit 1 fi -apply_ewald_265_patch() { - # Works around a silent nvfortran 26.5 OpenACC reduction-codegen bug that - # gives a wrong (but plausible) total energy -- see the patch file for - # details. Safe to apply on any OpenACC/GPU build regardless of version: - # the fix gates itself on __NVCOMPILER_MAJOR__/__NVCOMPILER_MINOR__ at - # compile time, so it's a no-op on unaffected compilers (<26.5). - # (RIKYU does not call this -- its pinned branch already contains the fix.) - if grep -q "SALMON_EWALD_ACC" src/common/total_energy.f90; then - echo "SALMON nvhpc/26.5 Ewald-reduction workaround already present" - elif git apply --check "${EWALD_265_PATCH}"; then - git apply "${EWALD_265_PATCH}" - else - echo "SALMON nvhpc/26.5 Ewald-reduction patch does not apply to ${VERSION_TAG}" >&2 - exit 1 - fi -} - rm -rf "${BUILD_DIR}" mkdir -p "${BUILD_DIR}" @@ -127,7 +111,6 @@ case "${system}" in RC_GH200) module purge module load system/qc-gh200 nvhpc-hpcx-cuda12/25.7 - apply_ewald_265_patch cmake_args=( "${common_cmake_args[@]}" -DCMAKE_Fortran_COMPILER=mpif90 @@ -147,7 +130,6 @@ case "${system}" in source /etc/profile.d/modules.sh module purge module load system/ng-dgx nvhpc-hpcx-cuda13/26.3 - apply_ewald_265_patch cmake_args=( "${common_cmake_args[@]}" -DCMAKE_Fortran_COMPILER=mpif90 diff --git a/programs/salmon/patches/nvhpc265-ewald-reduction.patch b/programs/salmon/patches/nvhpc265-ewald-reduction.patch deleted file mode 100644 index 609e9691..00000000 --- a/programs/salmon/patches/nvhpc265-ewald-reduction.patch +++ /dev/null @@ -1,100 +0,0 @@ -diff --git a/src/common/total_energy.f90 b/src/common/total_energy.f90 -index cc8fa5bd..cde5ef1d 100644 ---- a/src/common/total_energy.f90 -+++ b/src/common/total_energy.f90 -@@ -15,6 +15,39 @@ - ! - !--------10--------20--------30--------40--------50--------60--------70--------80--------90--------100-------110-------120-------130 - MODULE Total_Energy -+ -+! --------------------------------------------------------------------------- -+! nvfortran 26.5 OpenACC reduction workaround. -+! -+! Under nvhpc 26.5 the OpenACC reductions in init_ewald (reduction(max:)) and -+! calc_Total_Energy_periodic (reduction(+:)) silently return the reduction -+! IDENTITY (0) instead of the computed value. Verified by building the very -+! same loops as hand-written CUDA kernels on the same GPU with the same data: -+! CUDA gives 74 / 0.4478233939199485 (correct), OpenACC gives 0 / 0. -+! nvhpc 26.3 is unaffected. Both loops are one-time / per-ion-move setup, so -+! running them on the host costs essentially nothing. -+! -+! The book-keeping fill loop in init_ewald is affected the same way (its -+! per-atom counter comes back as 1 instead of ~74), so it is gated too. -+! -+! Unrelated to the compiler, that same fill loop also declared copyin(ewald) -+! -- host->device ONLY -- while writing ewald%bk / ewald%npair_bk inside the -+! region, so those device writes were semantically discarded. The wrong clause -+! is removed here so the GPU path is correct once the gate is relaxed. (That -+! fix on its own does not cure 26.5; the codegen bug above is the blocker.) -+! -+! When NVIDIA fixes this, drop the version test below (or raise the bound). -+! --------------------------------------------------------------------------- -+#if defined(USE_OPENACC) -+# if defined(__NVCOMPILER_MAJOR__) && (__NVCOMPILER_MAJOR__ > 26 || (__NVCOMPILER_MAJOR__ == 26 && __NVCOMPILER_MINOR__ >= 5)) -+# define SALMON_ACC_REDUCTION_BROKEN 1 -+# endif -+#endif -+ -+#if defined(USE_OPENACC) && !defined(SALMON_ACC_REDUCTION_BROKEN) -+# define SALMON_EWALD_ACC 1 -+#endif -+ - implicit none - - CONTAINS -@@ -323,7 +356,7 @@ CONTAINS - - if(ewald%yn_bookkeep=='y') then - --#ifdef USE_OPENACC -+#ifdef SALMON_EWALD_ACC - !$acc kernels copyin(ewald) - !$acc loop private(iia,ia,ipair,ix,iy,iz,ib,r,rab,rr) reduction(+:E_tmp) - #else -@@ -356,7 +389,7 @@ CONTAINS - - end do !ipair - end do !ia --#ifdef USE_OPENACC -+#ifdef SALMON_EWALD_ACC - !$acc end kernels - #else - !$omp end parallel do -@@ -843,7 +876,7 @@ CONTAINS - - !(check maximum number of pairs and allocate) - npair_bk_max = 0 --#ifdef USE_OPENACC -+#ifdef SALMON_EWALD_ACC - !$acc kernels loop private(iia,ia,ix,iy,iz,ib,r,rab,rr,npair_bk_loc) reduction(max:npair_bk_max) - #else - !$omp parallel do private(iia,ia,ix,iy,iz,ib,r,rab,rr,npair_bk_loc) & -@@ -880,7 +913,7 @@ CONTAINS - end do - npair_bk_max = max(npair_bk_max,npair_bk_loc) - end do --#ifdef USE_OPENACC -+#ifdef SALMON_EWALD_ACC - !$acc end kernels - #else - !$omp end parallel do -@@ -897,8 +930,8 @@ CONTAINS - 820 format(a,i6) - endif - --#ifdef USE_OPENACC --!$acc kernels loop private(iia,ia,ipair,ix,iy,iz,ib,r,rab,rr) copyin(ewald) -+#ifdef SALMON_EWALD_ACC -+!$acc kernels loop private(iia,ia,ipair,ix,iy,iz,ib,r,rab,rr) - #else - !$omp parallel do private(iia,ia,ipair,ix,iy,iz,ib,r,rab,rr) - #endif -@@ -937,7 +970,7 @@ CONTAINS - end do - ewald%npair_bk(iia) = ipair - end do --#ifdef USE_OPENACC -+#ifdef SALMON_EWALD_ACC - !$acc end kernels - #else - !$omp end parallel do From 06a9f037718015676f0b1d37bb8754bb00a28e5d Mon Sep 17 00:00:00 2001 From: William Dawson Date: Wed, 26 Aug 2026 10:23:13 +0900 Subject: [PATCH 12/12] Gate the FJMPI patch on Fugaku and trim comments [code:salmon] The patch chain had a RIKYU branch whose only job was to print that it was not patching. Only Fugaku compiles the Fujitsu MPI path, so apply the patch there and drop the special case. Cut the comments back. The measurement history behind each flag does not belong in a build script. No functional change. Signed-off-by: William Dawson Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PEzS2ybD8VEnQa9Sq5qYUM --- programs/salmon/build.sh | 81 +++++----------------------------------- 1 file changed, 10 insertions(+), 71 deletions(-) diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 3d900a90..1c04aad2 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -8,23 +8,8 @@ REPO_DIR="SALMON2" VERSION_TAG="v.2.2.2" BUILD_DIR="build-benchkit" -# RIKYU pin: v.2.2.2 is over a year stale (2025-06-06) and measured -# GPU-decomposition numbers on this repo were never actually run against it -- -# they came from a much newer, hand-patched checkout (~100 commits ahead, -# including PR #1276's OpenACC tuning). Rather than let RIKYU silently build -# something nobody has benchmarked, pin it to FugakuNEXT-v4 on -# william-dawson/SALMON2 (also open as SALMON-TDDFT/SALMON2#1276) -- -# develop-2.0.0@9b93a8c4 (2026-08-16) plus four commits, one per concern: -# 1. PR #1276's stencil/current/pseudo-pt OpenACC tuning (batched cuBLAS -# GEMM for pseudo-pt, not the old USE_CUDA hand-written kernels -- -# see below) -# 2. the nvhpc-openacc-gemm.cmake platform file that wires it up -# 3. a fix for the Ewald loop tripcounts: nvfortran >= 26.5 evaluates -# an OpenACC tripcount on the device, where the derived-type member -# the bound came from is not resident, so the loops ran zero times -# 4. a fix for a 2+ node hang on nvhpc/26.5: the CUDA context is created -# before MPI_Init_thread, so the MPI layer's CUDA-awareness probe does -# not cache a transfer protocol that deadlocks on device buffers +# v.2.2.2 is over a year stale and predates the OpenACC tuning, Ewald +# tripcount fix and multi-node CUDA-context fix that RIKYU needs. if [[ "${system}" == "RIKYU" ]]; then REPO_URL="https://github.com/william-dawson/SALMON2" VERSION_TAG="FugakuNEXT-v4" @@ -42,18 +27,14 @@ mkdir -p "${BUILD_LOG_DIR}" bk_fetch_source "${REPO_URL}" "${REPO_DIR}" "${VERSION_TAG}" cd "${REPO_DIR}" -# RIKYU builds a pinned branch that already carries every fix it needs -# (see VERSION_TAG above), so it is used as-is -- no patching. The patches -# below exist for the systems that build stock v.2.2.2. -if [[ "${system}" == "RIKYU" ]]; then - echo "RIKYU: building ${VERSION_TAG} unpatched (all fixes are in the branch)" -elif git apply --check "${FJMPI_PATCH}"; then - git apply "${FJMPI_PATCH}" -elif git apply --reverse --check "${FJMPI_PATCH}" >/dev/null 2>&1; then - echo "SALMON Fujitsu MPI topology patch is already applied" -else - echo "SALMON Fujitsu MPI topology patch does not apply to ${VERSION_TAG}" >&2 - exit 1 +# Only Fugaku compiles the Fujitsu MPI path. +if [[ "${system}" == "Fugaku" || "${system}" == "FugakuCN" ]]; then + if git apply --check "${FJMPI_PATCH}"; then + git apply "${FJMPI_PATCH}" + elif ! git apply --reverse --check "${FJMPI_PATCH}" >/dev/null 2>&1; then + echo "SALMON Fujitsu MPI topology patch does not apply to ${VERSION_TAG}" >&2 + exit 1 + fi fi rm -rf "${BUILD_DIR}" @@ -208,48 +189,6 @@ case "${system}" in -DCMAKE_SYSTEM_PROCESSOR=openacc -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver,nccl -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM -DUSE_NCCL" -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver,nccl -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM -DUSE_NCCL" - # Module is nvhpc-hpcx-cuda13/26.5 (HPC-X 2.50 / OpenMPI 5), not - # plain nvhpc/26.5: that is the stack the domain-decomposition and - # NCCL numbers were measured on. MPI3 is left at its auto-detected - # ON -- no FORTRAN_COMPILER_HAS_MPI_VERSION3 override -- because the - # GEMM path skips calc_uVpsi_rdivided entirely, so the HPC-X libnbc - # bug that motivated the old 26.3 + MPI3=OFF workaround is - # unreachable. Do not reintroduce that workaround. - # - # FugakuNEXT-v3 also computes the nonlocal current density with a - # batched GEMM (it was 82% of calculating-curr): current density - # 29.5 -> 3.6 s and rt iterations 44.6 -> 18.7 s at 16 GPUs orbital. - # - # -DUSE_NCCL routes the pseudo-pt domain-decomposition reduce through - # ncclAllReduce instead of MPI_Allreduce. MPI_Allreduce performs the - # reduction arithmetic on the host even when handed a device pointer, - # so it leaves NVLink: measured 5.6 GB/s vs NCCL's 196-521 GB/s on the - # same 222 MB device buffer. In SALMON that is pseudo-pt comm 18.2s -> - # 1.7s at 4 GPUs (rt iterations 99.1 -> 80.5), energies bit-exact. - # It needs nccl in -cudalib. If the MPI stack ever grows a working - # GPU-side allreduce (UCC ships a TL_NCCL that currently never - # registers), drop -DUSE_NCCL and this becomes plain MPI again. - # - # NOT -DUSE_CUDA -- that flag controls a completely different, OLDER - # optimization path (src/common/{zpseudo,stencil_current}.cu, hand- - # written CUDA kernels) that this pinned source (see VERSION_TAG - # above) replaces with PR #1276's tuned pure-OpenACC kernels instead: - # a batched cuBLAS GEMM rewrite of pseudo-pt (-DUSE_GEMM, needs - # cusolver linked in) and an inlined OpenACC current-density kernel. - # That's where the real speedup comes from, not USE_CUDA -- measured - # data only ever showed the OLD CUDA kernels net *losing* time - # (pseudo-pt 3.1-3.5x slower under USE_CUDA than plain OpenACC; the - # 1.4-3x win on current-density wasn't enough to make up for it). - # USE_CUDA also has a real, deterministic bug independent of any of - # this: stencil_current.cu's host wrapper sizes its device idx/idy/idz - # buffers by each rank's LOCAL grid extent but indexes them with the - # RAW/global grid coordinate, which only happens to fit when a rank's - # is()=1 on that axis (single GPU, or pure orbital decomposition, - # where every rank owns the full box). Any real-space (nproc_rgrid>1) - # decomposition puts a non-first rank at is()>1 on the split axis and - # overruns the buffer -- reproduced as a deterministic Accelerator - # Fatal Error / CUDA_ERROR_ILLEGAL_ADDRESS in calc_current - # (density_matrix.f90) on every axis and Po x Pg combination tried. ) ;; *)