diff --git a/docs/ci.md b/docs/ci.md index 3485c43..b08763a 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -266,18 +266,19 @@ cache is safe to use by comparing the current build input hash and the cached Git sources are rechecked with `git ls-remote`; file/archive sources are rechecked by SHA-256. If a container image hash is recorded, that image hash is -also verified. Host-build cache restore is conservative: it requires -`BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE=true` and a non-empty -`BK_BUILD_CACHE_ENV_KEY` so site operators can invalidate cache entries when -compiler/module stacks change. Cache misses fall back to the normal -`programs//build.sh` path and store a fresh cache after a successful -build. The generated child pipeline does not declare a GitLab `cache:` stanza; -the cache directory must be a site-managed persistent path such as the custom -runner's `CUSTOM_DIR`, not a per-job cleanup directory. +also verified. Host builds that go through the common `make` / `cmake` / +`ninja` wrappers are matched by a build-environment fingerprint collected just +before the build tool runs. The fingerprint includes loaded modules, selected +build environment variables, tool real paths, versions, and binary SHA-256 +hashes. Cache misses fall back to the normal `programs//build.sh` path +and store a fresh cache after a successful build. The generated child pipeline +does not declare a GitLab `cache:` stanza; the cache directory must be a +site-managed persistent path such as the custom runner's `CUSTOM_DIR`, not a +per-job cleanup directory. Git source は `git ls-remote` で再確認し、file/archive source は SHA-256 を再計算します。 container image hash が記録されている場合は image hash も確認します。 -host build の cache restore は保守的に扱い、compiler/module stack 変更時に site 運用側が cache を無効化できるよう、`BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE=true` と非空の `BK_BUILD_CACHE_ENV_KEY` を要求します。 +common の `make` / `cmake` / `ninja` wrapper を通る host build は、build tool 実行直前に収集した build environment fingerprint で照合します。この fingerprint には loaded modules、選択された build 環境変数、tool の real path、version、binary SHA-256 hash が含まれます。 cache miss の場合は通常の `programs//build.sh` 経路に戻り、成功後に新しい cache を保存します。 生成された child pipeline は GitLab の `cache:` stanza を出しません。cache directory は job ごとの cleanup 対象ではなく、custom runner の `CUSTOM_DIR` など site 側で管理する永続パスにしてください。 diff --git a/docs/guides/add-app.md b/docs/guides/add-app.md index 9add282..03a8800 100644 --- a/docs/guides/add-app.md +++ b/docs/guides/add-app.md @@ -148,7 +148,9 @@ cache hit は、少なくとも現在の app build input hash と source provena Git source では cache 内の `repo_url` / `branch` / `commit_hash` に対し、現在の branch commit を `git ls-remote` で再解決します。 file/archive source では SHA-256 を再計算します。 container image SHA-256 が source_info に入っている場合は container image も再検証します。 -container ではない host build は toolchain 更新で stale になり得るため、restore には site 側で `BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE=true` と非空の `BK_BUILD_CACHE_ENV_KEY` を明示する必要があります。 +container ではない host build でも、common の `make` / `cmake` / `ninja` wrapper を通る場合は build tool 実行直前の build environment fingerprint で照合します。 +この fingerprint には loaded modules、選択された build 環境変数、tool の real path、version、binary SHA-256 hash が含まれます。 +app 側で cache API を呼ぶ必要はありません。通常どおり `module load` して `make` / `cmake` / `ninja` を呼ぶだけで、common wrapper が fingerprint を記録します。 --- diff --git a/scripts/build_tool_wrappers/bk_build_tool_wrapper.sh b/scripts/build_tool_wrappers/bk_build_tool_wrapper.sh index 96720de..7afdee3 100755 --- a/scripts/build_tool_wrappers/bk_build_tool_wrapper.sh +++ b/scripts/build_tool_wrappers/bk_build_tool_wrapper.sh @@ -72,6 +72,24 @@ fallback_resolved_path() { printf '%s' "$path" } +fallback_command_sha256() { + local path="$1" + + if [ -z "$path" ] || [ ! -f "$path" ] || [ ! -r "$path" ]; then + printf '%s' "" + return 0 + fi + if PATH="$path_without_wrapper" command -v sha256sum >/dev/null 2>&1; then + PATH="$path_without_wrapper" sha256sum "$path" | awk '{print $1}' + return 0 + fi + if PATH="$path_without_wrapper" command -v openssl >/dev/null 2>&1; then + PATH="$path_without_wrapper" openssl dgst -sha256 -r "$path" | awk '{print $1}' + return 0 + fi + printf '%s' "" +} + fallback_command_version() { local cmd="$1" local version_args=("--version") @@ -122,7 +140,7 @@ fallback_commands_json() { local commands="${BK_SNAPSHOT_TOOL_COMMANDS:-$default_commands}" local first=true - local cmd path real_path version + local cmd path real_path version sha256 printf '{' for cmd in $commands; do @@ -132,16 +150,18 @@ fallback_commands_json() { fi real_path=$(fallback_resolved_path "$path") version=$(fallback_command_version "$cmd") + sha256=$(fallback_command_sha256 "$real_path") if [ "$first" = true ]; then first=false else printf ',' fi - printf '%s:{"path":%s,"real_path":%s,"version":%s}' \ + printf '%s:{"path":%s,"real_path":%s,"version":%s,"sha256":%s}' \ "$(json_string "$cmd")" \ "$(json_string "$path")" \ "$(json_string "$real_path")" \ - "$(json_string "$version")" + "$(json_string "$version")" \ + "$(json_string "$sha256")" done printf '}' } @@ -335,6 +355,15 @@ if [ "${BK_BUILD_TOOL_WRAPPER_SNAPSHOT:-true}" != "false" ]; then echo "bk_build_tool_wrapper: cannot collect build environment snapshot" >&2 exit 1 fi + if [ "$snapshot_written" = true ] && [ "${BK_BUILD_CACHE_LATE_RESTORE_PROBE:-false}" = "true" ]; then + late_restore_status_file="${BK_BUILD_CACHE_LATE_RESTORE_STATUS_FILE:-${repo_root}/results/build_cache_late_restore.env}" + mkdir -p "$(dirname "$late_restore_status_file")" + { + printf 'BK_BUILD_CACHE_LATE_RESTORE_TOOL=%s\n' "$tool_name" + printf 'BK_BUILD_CACHE_LATE_RESTORE_SNAPSHOT=%s\n' "$snapshot_file" + } > "$late_restore_status_file" + exit "${BK_BUILD_CACHE_LATE_RESTORE_EXIT_CODE:-86}" + fi fi exec "$real_tool" "$@" diff --git a/scripts/build_with_cache.sh b/scripts/build_with_cache.sh index f7f26bd..2ab40a6 100755 --- a/scripts/build_with_cache.sh +++ b/scripts/build_with_cache.sh @@ -29,6 +29,8 @@ if [ -n "$cache_root" ]; then cache_manifest="${cache_dir}/manifest.env" fi status_file="${repo_root}/results/build_cache.env" +late_restore_exit_code=86 +late_restore_status_file="${repo_root}/results/build_cache_late_restore.env" source "${repo_root}/scripts/bk_functions.sh" @@ -178,6 +180,50 @@ build_inputs_hash() { rm -f "$file_list" "$hash_list" } +host_environment_fingerprint() { + local snapshot_file="${repo_root}/results/environment_snapshot_build_actual.json" + local fingerprint_input + local fingerprint_hash + + if [ ! -f "$snapshot_file" ]; then + return 1 + fi + if ! command -v jq >/dev/null 2>&1; then + return 1 + fi + + fingerprint_input=$(mktemp) + if ! jq -S -c \ + --arg code "$code" \ + --arg system "$system" \ + '{ + schema: "benchkit-host-build-env-v1", + code: $code, + system: $system, + toolchain: { + modules: (.toolchain.modules // []), + commands: ( + .toolchain.commands // {} + | with_entries( + .value |= { + real_path: (.real_path // ""), + version: (.version // ""), + sha256: (.sha256 // "") + } + ) + ), + environment: (.toolchain.environment // {}) + } + }' "$snapshot_file" > "$fingerprint_input"; then + rm -f "$fingerprint_input" + return 1 + fi + fingerprint_hash=$(bk_sha256_file "$fingerprint_input") + rm -f "$fingerprint_input" + [ -n "$fingerprint_hash" ] || return 1 + printf '%s\n' "$fingerprint_hash" +} + resolve_git_branch_commit() { local repo_url="$1" local branch="$2" @@ -208,6 +254,8 @@ validate_cached_source() { local container_path local cached_container_sha256 local current_container_sha256 + local cached_host_fingerprint + local current_host_fingerprint source_type=$(env_file_value "$source_info_file" BK_SOURCE_TYPE) case "$source_type" in @@ -258,6 +306,25 @@ validate_cached_source() { return 0 fi + cached_host_fingerprint=$(manifest_value BK_CACHE_HOST_ENV_FINGERPRINT) + if [ -n "$cached_host_fingerprint" ]; then + if ! current_host_fingerprint=$(host_environment_fingerprint); then + echo "host build cache restore requires actual environment fingerprint" + return 1 + fi + if [ "$current_host_fingerprint" != "$cached_host_fingerprint" ]; then + echo "host build environment fingerprint changed" + return 1 + fi + if [ -n "$(manifest_value BK_CACHE_ENV_KEY_B64)" ] || [ -n "${BK_BUILD_CACHE_ENV_KEY:-}" ]; then + if [ "$(manifest_value BK_CACHE_ENV_KEY_B64 | decode_base64_value 2>/dev/null || true)" != "${BK_BUILD_CACHE_ENV_KEY:-}" ]; then + echo "host build cache environment key changed" + return 1 + fi + fi + return 0 + fi + if [ "${BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE:-false}" != "true" ]; then echo "host build cache restore requires BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE=true" return 1 @@ -323,6 +390,7 @@ store_cache() { local source_info_sha256="" local source_info_file="${repo_root}/results/source_info.env" local container_sha256="" + local host_fingerprint="" if [ "${BK_BUILD_CACHE_ENABLED:-true}" != "true" ]; then write_status disabled "BK_BUILD_CACHE_ENABLED is not true" @@ -337,9 +405,12 @@ store_cache() { return 0 fi container_sha256=$(env_file_value "$source_info_file" BK_CONTAINER_IMAGE_SHA256SUM) - if [ -z "$container_sha256" ] && [ -z "${BK_BUILD_CACHE_ENV_KEY:-}" ]; then - write_status miss "host build cache store requires non-empty BK_BUILD_CACHE_ENV_KEY" - return 0 + if [ -z "$container_sha256" ]; then + host_fingerprint=$(host_environment_fingerprint || true) + if [ -z "$host_fingerprint" ] && [ -z "${BK_BUILD_CACHE_ENV_KEY:-}" ]; then + write_status miss "host build cache store requires environment fingerprint or non-empty BK_BUILD_CACHE_ENV_KEY" + return 0 + fi fi inputs_hash=$(build_inputs_hash) @@ -359,6 +430,7 @@ store_cache() { printf 'BK_CACHE_SYSTEM_B64=%s\n' "$(bk_base64_encode_value "$system")" printf 'BK_CACHE_BUILD_INPUTS_SHA256=%s\n' "$inputs_hash" printf 'BK_CACHE_SOURCE_INFO_SHA256=%s\n' "$source_info_sha256" + printf 'BK_CACHE_HOST_ENV_FINGERPRINT=%s\n' "$host_fingerprint" printf 'BK_CACHE_ENV_KEY_B64=%s\n' "$(bk_base64_encode_value "${BK_BUILD_CACHE_ENV_KEY:-}")" printf 'BK_CACHE_CREATED_AT=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" } > "${tmp_dir}/manifest.env" @@ -370,6 +442,56 @@ store_cache() { echo "build cache: stored ${code}/${system}" } +should_attempt_late_host_restore() { + if [ "${BK_BUILD_CACHE_LATE_HOST_RESTORE:-true}" != "true" ]; then + return 1 + fi + if [ "${BK_BUILD_CACHE_ENABLED:-true}" != "true" ]; then + return 1 + fi + cache_dir_available || return 1 + [ -f "$cache_manifest" ] || return 1 + [ -d "${cache_dir}/artifacts" ] || return 1 + [ -f "${cache_dir}/results/source_info.env" ] || return 1 + [ -n "$(manifest_value BK_CACHE_HOST_ENV_FINGERPRINT)" ] || return 1 +} + +late_restore_hit=false +run_build_with_optional_late_host_restore() { + local build_status=0 + + if ! should_attempt_late_host_restore; then + bash "${program_path}/build.sh" "$system" + return $? + fi + + echo "build cache: probing host build environment for ${code}/${system}" + rm -f "$late_restore_status_file" + set +e + BK_BUILD_CACHE_LATE_RESTORE_PROBE=true \ + BK_BUILD_CACHE_LATE_RESTORE_EXIT_CODE="$late_restore_exit_code" \ + BK_BUILD_CACHE_LATE_RESTORE_STATUS_FILE="$late_restore_status_file" \ + bash "${program_path}/build.sh" "$system" + build_status=$? + set -e + + if [ "$build_status" -eq "$late_restore_exit_code" ] && [ -f "$late_restore_status_file" ]; then + if restore_cache; then + late_restore_hit=true + return 0 + fi + echo "build cache: late host restore miss for ${code}/${system}; running build.sh" + rm -f "$late_restore_status_file" + bash "${program_path}/build.sh" "$system" + return $? + fi + + if [ "$build_status" -ne 0 ]; then + return "$build_status" + fi + return 0 +} + validate_path_component "$code" code validate_path_component "$system" system @@ -392,5 +514,8 @@ if restore_cache; then fi echo "build cache: miss for ${code}/${system}; running build.sh" -bash "${program_path}/build.sh" "$system" +run_build_with_optional_late_host_restore +if [ "$late_restore_hit" = true ]; then + exit 0 +fi store_cache diff --git a/scripts/collect_environment_snapshot.sh b/scripts/collect_environment_snapshot.sh index ee080ce..05b4856 100755 --- a/scripts/collect_environment_snapshot.sh +++ b/scripts/collect_environment_snapshot.sh @@ -34,6 +34,24 @@ resolved_command_path() { printf '%s' "$path" } +command_sha256() { + local path="$1" + + if [ -z "$path" ] || [ ! -f "$path" ] || [ ! -r "$path" ]; then + printf '%s' "" + return 0 + fi + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + return 0 + fi + if command -v openssl >/dev/null 2>&1; then + openssl dgst -sha256 -r "$path" | awk '{print $1}' + return 0 + fi + printf '%s' "" +} + command_version() { local cmd="$1" shift @@ -90,7 +108,7 @@ snapshot_tool_commands_json() { default_commands+="cmake make ninja ld ar pkg-config python3 apptainer singularity ncu nsys" local commands="${BK_SNAPSHOT_TOOL_COMMANDS:-$default_commands}" - local cmd path real_path version + local cmd path real_path version sha256 for cmd in $commands; do path=$(command_path "$cmd") @@ -99,12 +117,14 @@ snapshot_tool_commands_json() { fi real_path=$(resolved_command_path "$path") version=$(snapshot_command_version "$cmd") + sha256=$(command_sha256 "$real_path") jq -n -c \ --arg name "$cmd" \ --arg path "$path" \ --arg real_path "$real_path" \ --arg version "$version" \ - '{key: $name, value: {path: $path, real_path: $real_path, version: $version}}' + --arg sha256 "$sha256" \ + '{key: $name, value: {path: $path, real_path: $real_path, version: $version, sha256: $sha256}}' done | jq -s -c 'from_entries' } diff --git a/scripts/tests/test_build_cache.sh b/scripts/tests/test_build_cache.sh index 82d51fb..1e6947f 100755 --- a/scripts/tests/test_build_cache.sh +++ b/scripts/tests/test_build_cache.sh @@ -9,9 +9,11 @@ trap 'rm -rf "${TMP_DIR}"' EXIT mkdir -p \ "${TMP_DIR}/project/programs/app" \ + "${TMP_DIR}/project/programs/toolapp" \ "${TMP_DIR}/project/scripts" \ "${TMP_DIR}/project/scripts/build_tool_wrappers" \ - "${TMP_DIR}/source" + "${TMP_DIR}/source" \ + "${TMP_DIR}/tools" cp "${REPO_DIR}/scripts/bk_functions.sh" "${TMP_DIR}/project/scripts/bk_functions.sh" cp "${REPO_DIR}/scripts/build_with_cache.sh" "${TMP_DIR}/project/scripts/build_with_cache.sh" @@ -48,6 +50,38 @@ printf 'artifact %s %s\n' "$system" "$BK_COMMIT_HASH" > artifacts/app.bin EOF chmod +x "${TMP_DIR}/project/programs/app/build.sh" +cat > "${TMP_DIR}/project/programs/toolapp/build.sh" <<'EOF' +#!/bin/bash +set -euo pipefail + +system="$1" +source scripts/bk_functions.sh +mkdir -p artifacts +bk_fetch_source "${BK_TEST_SOURCE_REPO}" toolsrc main +cd toolsrc +make "$system" +EOF +chmod +x "${TMP_DIR}/project/programs/toolapp/build.sh" + +cat > "${TMP_DIR}/tools/make" <<'EOF' +#!/bin/bash +set -euo pipefail + +if [ "${1:-}" = "--version" ]; then + echo "GNU Make fake-test" + exit 0 +fi + +count=0 +if [ -f "${BK_TEST_TOOL_BUILD_COUNT}" ]; then + count=$(cat "${BK_TEST_TOOL_BUILD_COUNT}") +fi +count=$((count + 1)) +printf '%s\n' "$count" > "${BK_TEST_TOOL_BUILD_COUNT}" +printf 'tool artifact %s %s\n' "$*" "${BK_COMMIT_HASH:-missing}" > ../artifacts/toolapp.bin +EOF +chmod +x "${TMP_DIR}/tools/make" + app_build_hash=$(sha256sum "${TMP_DIR}/project/programs/app/build.sh" | awk '{print $1}') run_build_with_cache_for_root() { @@ -84,6 +118,23 @@ run_build_without_host_restore_opt_in() { run_build_without_host_restore_opt_in_for_root "${TMP_DIR}/project" } +run_tool_build_with_cache_for_root() { + local project_root="$1" + + pushd "$project_root" >/dev/null + PATH="${TMP_DIR}/tools:${PATH}" \ + BK_BENCHKIT_ROOT="$project_root" \ + BK_BUILD_CACHE_DIR="${TMP_DIR}/tool-cache" \ + BK_TEST_SOURCE_REPO="${TMP_DIR}/source/.git" \ + BK_TEST_TOOL_BUILD_COUNT="${TMP_DIR}/tool-build-count" \ + bash scripts/build_with_cache.sh toolapp TestSystem programs/toolapp + popd >/dev/null +} + +run_tool_build_with_cache() { + run_tool_build_with_cache_for_root "${TMP_DIR}/project" +} + pushd "${TMP_DIR}/project" >/dev/null BK_BENCHKIT_ROOT="${TMP_DIR}/project" \ BK_TEST_SOURCE_REPO="${TMP_DIR}/source/.git" \ @@ -170,6 +221,24 @@ test "$(cat "${TMP_DIR}/build-count")" = "2" grep -q "$first_commit" "${TMP_DIR}/project/artifacts/app.bin" grep -q '^BK_BUILD_CACHE_STATUS=hit$' "${TMP_DIR}/project/results/build_cache.env" +rm -rf "${TMP_DIR}/project/artifacts" "${TMP_DIR}/project/results" "${TMP_DIR}/project/toolsrc" +rm -f "${TMP_DIR}/tool-build-count" +run_tool_build_with_cache +test "$(cat "${TMP_DIR}/tool-build-count")" = "1" +grep -q "$first_commit" "${TMP_DIR}/project/artifacts/toolapp.bin" +grep -q '^BK_BUILD_CACHE_STORED=true$' "${TMP_DIR}/project/results/build_cache.env" +test -f "${TMP_DIR}/tool-cache/toolapp/TestSystem/manifest.env" +awk -F= '$1 == "BK_CACHE_HOST_ENV_FINGERPRINT" && length($2) == 64 {found=1} END {exit(found ? 0 : 1)}' \ + "${TMP_DIR}/tool-cache/toolapp/TestSystem/manifest.env" +jq -e '.toolchain.commands.make.sha256 | length == 64' \ + "${TMP_DIR}/project/results/environment_snapshot_build_actual.json" >/dev/null + +rm -rf "${TMP_DIR}/project/artifacts" "${TMP_DIR}/project/results" "${TMP_DIR}/project/toolsrc" +run_tool_build_with_cache +test "$(cat "${TMP_DIR}/tool-build-count")" = "1" +grep -q "$first_commit" "${TMP_DIR}/project/artifacts/toolapp.bin" +grep -q '^BK_BUILD_CACHE_STATUS=hit$' "${TMP_DIR}/project/results/build_cache.env" + pushd "${TMP_DIR}/source" >/dev/null printf 'two\n' > source.txt git add source.txt