diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 793cf0a5..1c04aad2 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -7,6 +7,13 @@ REPO_URL="https://github.com/SALMON-TDDFT/SALMON2" REPO_DIR="SALMON2" VERSION_TAG="v.2.2.2" BUILD_DIR="build-benchkit" + +# v.2.2.2 is over a year stale and predates the OpenACC tuning, Ewald +# tripcount fix and multi-node CUDA-context fix that RIKYU needs. +if [[ "${system}" == "RIKYU" ]]; then + REPO_URL="https://github.com/william-dawson/SALMON2" + VERSION_TAG="FugakuNEXT-v4" +fi ARTIFACT_DIR="${PWD}/artifacts" RESULTS_DIR="${PWD}/results" BUILD_LOG_DIR="${RESULTS_DIR}/salmon_build_logs" @@ -20,14 +27,16 @@ mkdir -p "${BUILD_LOG_DIR}" bk_fetch_source "${REPO_URL}" "${REPO_DIR}" "${VERSION_TAG}" cd "${REPO_DIR}" -if git apply --check "${FJMPI_PATCH}"; then - git apply "${FJMPI_PATCH}" -elif git apply --reverse --check "${FJMPI_PATCH}" >/dev/null 2>&1; then - echo "SALMON Fujitsu MPI topology patch is already applied" -else - echo "SALMON Fujitsu MPI topology patch does not apply to ${VERSION_TAG}" >&2 - exit 1 +# Only Fugaku compiles the Fujitsu MPI path. +if [[ "${system}" == "Fugaku" || "${system}" == "FugakuCN" ]]; then + if git apply --check "${FJMPI_PATCH}"; then + git apply "${FJMPI_PATCH}" + elif ! git apply --reverse --check "${FJMPI_PATCH}" >/dev/null 2>&1; then + echo "SALMON Fujitsu MPI topology patch does not apply to ${VERSION_TAG}" >&2 + exit 1 + fi fi + rm -rf "${BUILD_DIR}" mkdir -p "${BUILD_DIR}" @@ -167,6 +176,21 @@ case "${system}" in # -DUSE_FJMPI=OFF # ) # ;; + RIKYU) + module purge + module load nvhpc-hpcx-cuda13/26.5 + cmake_args=( + "${common_cmake_args[@]}" + -DCMAKE_Fortran_COMPILER=mpif90 + -DCMAKE_C_COMPILER=mpicc + -DOPENMP_FLAGS=-Mnoopenmp + -DUSE_OPENACC=ON + -DUSE_MPI_DEFAULT=ON + -DCMAKE_SYSTEM_PROCESSOR=openacc + -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver,nccl -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM -DUSE_NCCL" + -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas,cusolver,nccl -cuda -Minfo=accel -DUSE_OPENACC -DUSE_GEMM -DUSE_NCCL" + ) + ;; *) echo "Unknown system: ${system}" >&2 exit 1 diff --git a/programs/salmon/list.csv b/programs/salmon/list.csv index 386804d9..ac6c8c56 100644 --- a/programs/salmon/list.csv +++ b/programs/salmon/list.csv @@ -1,6 +1,10 @@ system,enable,nodes,numproc_node,nthreads,elapse -Fugaku,yes,1,4,12,0:10:00 +Fugaku,yes,24,1,12,0:30:00 RC_GH200,yes,1,1,1,0:10:00 RC_DGXSP,yes,1,1,1,1:00:00 RC_GENOA,yes,1,1,48,2:00:00 RC_FX700,no,1,1,48,0:30:00 +RIKYU,yes,1,4,1,0:30:00 +RIKYU,yes,2,4,1,0:30:00 +RIKYU,yes,3,4,1,0:30:00 +RIKYU,yes,4,4,1,0:30:00 diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index 1689c38a..623d3ef4 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -11,10 +11,35 @@ source "${PWD}/scripts/bk_functions.sh" RESULTS_DIR="${PWD}/results" WORK_DIR="${PWD}/salmon_run" -INPUT_ARCHIVE_DEFAULT="/vol0003/rccs-sdt/data/a01010/benchmark_data/SALMON.tar.gz" INPUT_ARCHIVE_CLOUD="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON/SALMON.tar.gz" AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" +# Pre-staged folded restarts. The ground state is computed once offline and +# folded to a complex Gamma-point TDDFT restart (see README.md), then stored +# on each machine, so the benchmark runs TDDFT only and never pays for a +# from-scratch ground state. Add a new case here (and a +# matching directory on that system) to move another system onto this +# path -- see the RIKYU or RC_DGXSP entries for the shape a new one needs +# (restart/, *.psp8, and one TDDFT .nml, all siblings in one directory). +RIKYU_RESTART_DIR_DEFAULT="/data1/rkp00012/CX_input/SALMON/3x3x3-folded" +RIKYU_RESTART_NML="Si-3-3-3-tddft.nml" +RC_DGXSP_RESTART_DIR_DEFAULT="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON_2x2x2_folded" +RC_DGXSP_RESTART_NML="Si-2-2-2-tddft.nml" +FUGAKU_RESTART_DIR_DEFAULT="/home/ra000009/data/u10035/CX_input_fugaku/SALMON_3x3x3_folded" +FUGAKU_RESTART_NML="Si-3-3-3-tddft.nml" +# Fugaku's list.csv row MUST use enough MPI ranks that each rank's local +# wfn.bin chunk stays under ~2GB: SALMON's default restart reader +# (method_wf_distributor='single') does one MPI_File_read_all per rank +# spanning its whole orbital slice, and Fujitsu MPI's MPI-IO throws +# MPI_ERR_ARG on per-rank reads above that size (a real, reproduced, +# vendor-specific MPI-IO limit -- confirmed independent of both restart +# provenance and physical memory: it reproduced identically whether the +# restart was folded on the login node or a genuine A64FX compute node, +# and separately from the plain OOM that a too-small node count/rank +# count hits first for this restart's 37GB wfn.bin). 24 ranks (81 +# orbitals/rank, ~1.5GB/rank) is the smallest node count validated so far +# for the 3x3x3 case; going lower reintroduces the failure. + mkdir -p "${RESULTS_DIR}" : > "${RESULTS_DIR}/result" @@ -23,31 +48,20 @@ if [[ ! -x artifacts/salmon ]]; then exit 1 fi -case "${system}" in - Fugaku) - input_archive="${INPUT_ARCHIVE_DEFAULT}" - exec_gs=(-stdin Si-1-1-1.nml ./salmon) - exec_rt=(-stdin Si-1-1-1-tddft.nml ./salmon) - ;; - RC_GH200|RC_DGXSP|RC_GENOA) - input_archive="${INPUT_ARCHIVE_CLOUD}" - exec_gs=(./salmon) - exec_rt=(./salmon) - ;; - *) - echo "Unknown system: ${system}" >&2 - exit 1 - ;; -esac - -if [[ ! -f "${input_archive}" ]]; then - echo "Input archive not found: ${input_archive}" >&2 - exit 1 -fi +uses_prestaged_restart() { + case "$1" in + RIKYU|RC_DGXSP|Fugaku) + return 0 + ;; + *) + return 1 + ;; + esac +} uses_stdin_input() { case "$1" in - RC_GH200|RC_DGXSP|RC_GENOA) + RIKYU|RC_GH200|RC_DGXSP|RC_GENOA) return 0 ;; *) @@ -57,27 +71,86 @@ uses_stdin_input() { } rm -rf "${WORK_DIR}" -mkdir -p "${WORK_DIR}/input" -tar -xzf "${input_archive}" -C "${WORK_DIR}/input" +mkdir -p "${WORK_DIR}" -input_dir=$(find "${WORK_DIR}/input" -type d -path "*/Si-1-1-1/input" | head -n 1) -if [[ -z "${input_dir}" ]]; then - input_dir=$(find "${WORK_DIR}/input" -type f -name "Si-1-1-1.nml" -printf '%h\n' | head -n 1) -fi -if [[ -z "${input_dir}" || ! -d "${input_dir}" ]]; then - echo "SALMON Si-1-1-1 input directory not found in ${input_archive}" >&2 - exit 1 -fi +if uses_prestaged_restart "${system}"; then + case "${system}" in + RIKYU) + restart_dir="${BK_SALMON_RESTART_DIR:-${RIKYU_RESTART_DIR_DEFAULT}}" + tddft_nml="${RIKYU_RESTART_NML}" + ;; + RC_DGXSP) + restart_dir="${BK_SALMON_RESTART_DIR:-${RC_DGXSP_RESTART_DIR_DEFAULT}}" + tddft_nml="${RC_DGXSP_RESTART_NML}" + ;; + Fugaku) + restart_dir="${BK_SALMON_RESTART_DIR:-${FUGAKU_RESTART_DIR_DEFAULT}}" + tddft_nml="${FUGAKU_RESTART_NML}" + ;; + esac -cp artifacts/salmon "${WORK_DIR}/salmon" -chmod +x "${WORK_DIR}/salmon" -cp "${input_dir}"/* "${WORK_DIR}/" -grep -Ein '^[[:space:]]*theory[[:space:]]*=' "${WORK_DIR}/Si-1-1-1.nml" "${WORK_DIR}/Si-1-1-1-tddft.nml" >&2 || true + if [[ ! -d "${restart_dir}" || ! -d "${restart_dir}/restart" ]]; then + echo "Pre-staged restart not found: ${restart_dir} (expects restart/, *.psp8, ${tddft_nml})" >&2 + exit 1 + fi + + cp artifacts/salmon "${WORK_DIR}/salmon" + chmod +x "${WORK_DIR}/salmon" + cp "${restart_dir}/${tddft_nml}" "${restart_dir}"/*.psp8 "${WORK_DIR}/" + # Symlink, never copy: the restart is O(10s of GB) (a folded 3x3x3 + # wfn.bin alone is ~35GB) and is immutable input, so copying it into a + # throwaway per-run work dir would burn most of the wall-clock budget on + # I/O instead of the benchmark itself. + ln -s "${restart_dir}/restart" "${WORK_DIR}/restart" + grep -Ein '^[[:space:]]*theory[[:space:]]*=' "${WORK_DIR}/${tddft_nml}" >&2 || true +else + case "${system}" in + RC_GH200|RC_GENOA) + input_archive="${INPUT_ARCHIVE_CLOUD}" + exec_gs=(./salmon) + exec_rt=(./salmon) + ;; + *) + echo "Unknown system: ${system}" >&2 + exit 1 + ;; + esac + + if [[ ! -f "${input_archive}" ]]; then + echo "Input archive not found: ${input_archive}" >&2 + exit 1 + fi + + mkdir -p "${WORK_DIR}/input" + tar -xzf "${input_archive}" -C "${WORK_DIR}/input" + + input_dir=$(find "${WORK_DIR}/input" -type d -path "*/Si-1-1-1/input" | head -n 1) + if [[ -z "${input_dir}" ]]; then + input_dir=$(find "${WORK_DIR}/input" -type f -name "Si-1-1-1.nml" -printf '%h\n' | head -n 1) + fi + if [[ -z "${input_dir}" || ! -d "${input_dir}" ]]; then + echo "SALMON Si-1-1-1 input directory not found in ${input_archive}" >&2 + exit 1 + fi + + cp artifacts/salmon "${WORK_DIR}/salmon" + chmod +x "${WORK_DIR}/salmon" + cp "${input_dir}"/* "${WORK_DIR}/" + grep -Ein '^[[:space:]]*theory[[:space:]]*=' "${WORK_DIR}/Si-1-1-1.nml" "${WORK_DIR}/Si-1-1-1-tddft.nml" >&2 || true +fi cd "${WORK_DIR}" case "${system}" in Fugaku) export OMP_NUM_THREADS="${nthreads}" + # Fugaku always uses the pre-staged-restart path (see + # uses_prestaged_restart above), so tddft_nml is always set here. + awk -v nproc_ob="${n_ranks}" ' + /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = 1, 1, 1"; in_parallel=1; next } + in_parallel && /^\// { in_parallel=0; print; next } + in_parallel { next } + { print } + ' "${tddft_nml}" > "${tddft_nml}.tmp" && mv "${tddft_nml}.tmp" "${tddft_nml}" ;; RC_GH200) module purge @@ -108,6 +181,30 @@ case "${system}" in fi export OMP_NUM_THREADS="${nthreads}" ;; + RIKYU) + module purge + module load nvhpc-hpcx-cuda13/26.5 + export OMP_NUM_THREADS="${nthreads}" + # Layout is set per RT run below (RIKYU benchmarks two of them). + if [[ "${n_ranks}" -gt 1 ]]; then + cat > wrapper.sh <<'WRAPPER' +#!/bin/bash +NCUDA_GPUS=${NCUDA_GPUS:-$(nvidia-smi -L | wc -l)} +if [ "$OMPI_COMM_WORLD_LOCAL_SIZE" -gt "$NCUDA_GPUS" ]; then + if [ "$OMPI_COMM_WORLD_LOCAL_RANK" -eq 0 ]; then + nvidia-cuda-mps-control -d + fi + sleep 10 +fi +export CUDA_VISIBLE_DEVICES=$((${OMPI_COMM_WORLD_LOCAL_RANK} % ${NCUDA_GPUS})) +exec "$@" +WRAPPER + chmod +x wrapper.sh + fi + # UCX_IB_GPU_DIRECT_RDMA=no was needed on 26.3 to stop a multi-node + # hang. Ablation-tested on 26.5 (4-GPU domain, 8-GPU mixed across 2 + # nodes, and 16/32 GPUs): not needed, and neither is UCX_TLS=^cma. + ;; # RC_FX700) # FX700 currently fails during GS initialization even with the Fujitsu # topology guard patch applied. Keep this route disabled until verified. @@ -122,6 +219,18 @@ run_salmon() { local logfile="$1" shift case "${system}" in + RIKYU) + # --mca fcoll individual is REQUIRED. MPI_File_read_all reads the + # restart straight into managed memory and OMPIO's default two-phase + # collective I/O redistributes it through cuMemcpyAsync: that + # deadlocks whenever the nproc_rgrid product exceeds 2, and even when + # it does not it costs 66.9s vs 41.7s on the restart read. + if [[ "${n_ranks}" -gt 1 ]]; then + mpirun -n "${n_ranks}" --mca fcoll individual ./wrapper.sh "$@" > "${logfile}" 2>&1 + else + mpirun -n "${n_ranks}" --mca fcoll individual "$@" > "${logfile}" 2>&1 + fi + ;; RC_GENOA) mpirun -n "${n_ranks}" --bind-to core --map-by "ppr:${numproc_node}:node:PE=${nthreads}" "$@" > "${logfile}" 2>&1 ;; @@ -186,57 +295,128 @@ print_salmon_output_diagnostics() { ) } -touch .gs_start_marker -gs_start=$(date +%s.%N) -if uses_stdin_input "${system}"; then - run_salmon_or_diagnose GS .gs_start_marker gs.log "${exec_gs[@]}" < Si-1-1-1.nml -else - run_salmon_or_diagnose GS .gs_start_marker gs.log "${exec_gs[@]}" -fi -gs_end=$(date +%s.%N) +if uses_prestaged_restart "${system}"; then + # The ground state was computed offline when the restart was staged; the + # benchmark is TDDFT-only. Rewrite the ¶llel block in place. + salmon_set_parallel () { + awk -v nproc_ob="$1" -v rgrid="$2" -v alloc="$3" ' + /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = " rgrid; print " process_allocation = " q alloc q; in_parallel=1; next } + in_parallel && /^\// { in_parallel=0; print; next } + in_parallel { next } + { print } + ' q="'" "${tddft_nml}" > "${tddft_nml}.tmp" && mv "${tddft_nml}.tmp" "${tddft_nml}" + } -if [[ -d data_for_restart ]]; then - rm -rf restart - mv data_for_restart restart -fi + # One timed RT run at the current layout, emitting one result line. + run_rt_once () { + local label="$1" logfile="rt_$1.log" + touch .rt_start_marker + local t0 t1 + t0=$(date +%s.%N) + if uses_stdin_input "${system}"; then + run_salmon_or_diagnose RT .rt_start_marker "${logfile}" ./salmon < "${tddft_nml}" + else + # Fugaku (Fujitsu MPI): -stdin FILE must be an mpiexec-level argument, + # not shell redirection -- plain `< file` only feeds rank 0's stdin + # under pjsub's mpiexec, not all ranks. + run_salmon_or_diagnose RT .rt_start_marker "${logfile}" -stdin "${tddft_nml}" ./salmon + fi + t1=$(date +%s.%N) + cp "${logfile}" "${RESULTS_DIR}/" + if ! salmon_output_has_marker_since "${logfile}" .rt_start_marker; then + echo "SALMON RT run failed (${label})" >&2 + tail -n 40 "${logfile}" >&2 || true + print_salmon_output_diagnostics .rt_start_marker + exit 1 + fi + # FOM is SALMON's own rt-iterations timer, not wall clock: the one-time + # restart read is ~40-150 s depending on whether Lustre is cold, which + # swamps and reorders the comparison when two layouts run in one job. + # Production runs do very many steps, so the RT loop is what matters. + local rt_s + rt_s=$(awk '/^rt iterations/ {print $(NF-3); exit}' "${logfile}") + if [[ -z "${rt_s}" ]]; then + echo "could not read 'rt iterations' from ${logfile}" >&2 + exit 1 + fi + bk_emit_result \ + --fom "${rt_s}" \ + --fom-unit s \ + --fom-version "rt_iterations_s_folded_restart" \ + --exp "${tddft_nml%.nml}-${label}" \ + --nodes "${nodes}" \ + --numproc-node "${numproc_node}" \ + --nthreads "${nthreads}" \ + >> "${RESULTS_DIR}/result" + } -touch .rt_start_marker -rt_start=$(date +%s.%N) -if uses_stdin_input "${system}"; then - run_salmon_or_diagnose RT .rt_start_marker rt.log "${exec_rt[@]}" < Si-1-1-1-tddft.nml + if [[ "${system}" == "RIKYU" && "${numproc_node}" -eq 4 ]]; then + # Benchmark both decompositions. Orbital is currently the faster of the + # two at every node count measured, but mixed is what a problem with less + # orbital parallelism has to fall back on, so both are worth tracking. + # Mixed keeps the 2x2x1 grid split inside a node (NVLink) and spreads + # orbitals across nodes; process_allocation='grid_sequential' is required + # for that, since the default strides icomm_r across nodes. + salmon_set_parallel "${n_ranks}" "1, 1, 1" "orbital_sequential" + run_rt_once orbital + salmon_set_parallel "${nodes}" "2, 2, 1" "grid_sequential" + run_rt_once mixed + else + run_rt_once default + fi else - run_salmon_or_diagnose RT .rt_start_marker rt.log "${exec_rt[@]}" -fi -rt_end=$(date +%s.%N) - -gs_elapsed=$(awk -v start="${gs_start}" -v end="${gs_end}" 'BEGIN {printf "%.6f", end - start}') -rt_elapsed=$(awk -v start="${rt_start}" -v end="${rt_end}" 'BEGIN {printf "%.6f", end - start}') -total_elapsed=$(awk -v gs="${gs_elapsed}" -v rt="${rt_elapsed}" 'BEGIN {printf "%.6f", gs + rt}') - -cp gs.log rt.log "${RESULTS_DIR}/" - -if ! salmon_output_has_marker_since gs.log .gs_start_marker || ! salmon_output_has_marker_since rt.log .rt_start_marker; then - echo "SALMON success marker not found in both gs.log and rt.log" >&2 - echo "---- gs.log tail ----" >&2 - tail -n 40 gs.log >&2 || true - echo "---- rt.log tail ----" >&2 - tail -n 40 rt.log >&2 || true - echo "---- files updated since GS start ----" >&2 - print_salmon_output_diagnostics .gs_start_marker - echo "---- files updated since RT start ----" >&2 - print_salmon_output_diagnostics .rt_start_marker - exit 1 -fi + touch .gs_start_marker + gs_start=$(date +%s.%N) + if uses_stdin_input "${system}"; then + run_salmon_or_diagnose GS .gs_start_marker gs.log "${exec_gs[@]}" < Si-1-1-1.nml + else + run_salmon_or_diagnose GS .gs_start_marker gs.log "${exec_gs[@]}" + fi + gs_end=$(date +%s.%N) + + if [[ -d data_for_restart ]]; then + rm -rf restart + mv data_for_restart restart + fi + + touch .rt_start_marker + rt_start=$(date +%s.%N) + if uses_stdin_input "${system}"; then + run_salmon_or_diagnose RT .rt_start_marker rt.log "${exec_rt[@]}" < Si-1-1-1-tddft.nml + else + run_salmon_or_diagnose RT .rt_start_marker rt.log "${exec_rt[@]}" + fi + rt_end=$(date +%s.%N) + + gs_elapsed=$(awk -v start="${gs_start}" -v end="${gs_end}" 'BEGIN {printf "%.6f", end - start}') + rt_elapsed=$(awk -v start="${rt_start}" -v end="${rt_end}" 'BEGIN {printf "%.6f", end - start}') + total_elapsed=$(awk -v gs="${gs_elapsed}" -v rt="${rt_elapsed}" 'BEGIN {printf "%.6f", gs + rt}') + + cp gs.log rt.log "${RESULTS_DIR}/" -{ - bk_emit_result \ - --fom "${total_elapsed}" \ - --fom-unit s \ - --fom-version "total_elapsed_time_s" \ - --exp "Si-1-1-1" \ - --nodes "${nodes}" \ - --numproc-node "${numproc_node}" \ - --nthreads "${nthreads}" - bk_emit_section gs "${gs_elapsed}" - bk_emit_section rt "${rt_elapsed}" -} >> "${RESULTS_DIR}/result" + if ! salmon_output_has_marker_since gs.log .gs_start_marker || ! salmon_output_has_marker_since rt.log .rt_start_marker; then + echo "SALMON success marker not found in both gs.log and rt.log" >&2 + echo "---- gs.log tail ----" >&2 + tail -n 40 gs.log >&2 || true + echo "---- rt.log tail ----" >&2 + tail -n 40 rt.log >&2 || true + echo "---- files updated since GS start ----" >&2 + print_salmon_output_diagnostics .gs_start_marker + echo "---- files updated since RT start ----" >&2 + print_salmon_output_diagnostics .rt_start_marker + exit 1 + fi + + { + bk_emit_result \ + --fom "${total_elapsed}" \ + --fom-unit s \ + --fom-version "total_elapsed_time_s" \ + --exp "Si-1-1-1" \ + --nodes "${nodes}" \ + --numproc-node "${numproc_node}" \ + --nthreads "${nthreads}" + bk_emit_section gs "${gs_elapsed}" + bk_emit_section rt "${rt_elapsed}" + } >> "${RESULTS_DIR}/result" +fi