Add host build cache fingerprint restore

docs/ci.md

@@ -266,18 +266,19 @@ cache is safe to use by comparing the current build input hash and the cached
 
 Git sources are rechecked with `git ls-remote`; file/archive sources are
 rechecked by SHA-256. If a container image hash is recorded, that image hash is
-also verified. Host-build cache restore is conservative: it requires
-`BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE=true` and a non-empty
-`BK_BUILD_CACHE_ENV_KEY` so site operators can invalidate cache entries when
-compiler/module stacks change. Cache misses fall back to the normal
-`programs/<code>/build.sh` path and store a fresh cache after a successful
-build. The generated child pipeline does not declare a GitLab `cache:` stanza;
-the cache directory must be a site-managed persistent path such as the custom
-runner's `CUSTOM_DIR`, not a per-job cleanup directory.
+also verified. Host builds that go through the common `make` / `cmake` /
+`ninja` wrappers are matched by a build-environment fingerprint collected just
+before the build tool runs. The fingerprint includes loaded modules, selected
+build environment variables, tool real paths, versions, and binary SHA-256
+hashes. Cache misses fall back to the normal `programs/<code>/build.sh` path
+and store a fresh cache after a successful build. The generated child pipeline
+does not declare a GitLab `cache:` stanza; the cache directory must be a
+site-managed persistent path such as the custom runner's `CUSTOM_DIR`, not a
+per-job cleanup directory.
 
 Git source は `git ls-remote` で再確認し、file/archive source は SHA-256 を再計算します。
 container image hash が記録されている場合は image hash も確認します。
-host build の cache restore は保守的に扱い、compiler/module stack 変更時に site 運用側が cache を無効化できるよう、`BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE=true` と非空の `BK_BUILD_CACHE_ENV_KEY` を要求します。
+common の `make` / `cmake` / `ninja` wrapper を通る host build は、build tool 実行直前に収集した build environment fingerprint で照合します。この fingerprint には loaded modules、選択された build 環境変数、tool の real path、version、binary SHA-256 hash が含まれます。
 cache miss の場合は通常の `programs/<code>/build.sh` 経路に戻り、成功後に新しい cache を保存します。
 生成された child pipeline は GitLab の `cache:` stanza を出しません。cache directory は job ごとの cleanup 対象ではなく、custom runner の `CUSTOM_DIR` など site 側で管理する永続パスにしてください。
 

docs/guides/add-app.md

@@ -148,7 +148,9 @@ cache hit は、少なくとも現在の app build input hash と source provena
 Git source では cache 内の `repo_url` / `branch` / `commit_hash` に対し、現在の branch commit を `git ls-remote` で再解決します。
 file/archive source では SHA-256 を再計算します。
 container image SHA-256 が source_info に入っている場合は container image も再検証します。
-container ではない host build は toolchain 更新で stale になり得るため、restore には site 側で `BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE=true` と非空の `BK_BUILD_CACHE_ENV_KEY` を明示する必要があります。
+container ではない host build でも、common の `make` / `cmake` / `ninja` wrapper を通る場合は build tool 実行直前の build environment fingerprint で照合します。
+この fingerprint には loaded modules、選択された build 環境変数、tool の real path、version、binary SHA-256 hash が含まれます。
+app 側で cache API を呼ぶ必要はありません。通常どおり `module load` して `make` / `cmake` / `ninja` を呼ぶだけで、common wrapper が fingerprint を記録します。
 
 ---
 

scripts/build_tool_wrappers/bk_build_tool_wrapper.sh

@@ -72,6 +72,24 @@ fallback_resolved_path() {
   printf '%s' "$path"
 }
 
+fallback_command_sha256() {
+  local path="$1"
+
+  if [ -z "$path" ] || [ ! -f "$path" ] || [ ! -r "$path" ]; then
+    printf '%s' ""
+    return 0
+  fi
+  if PATH="$path_without_wrapper" command -v sha256sum >/dev/null 2>&1; then
+    PATH="$path_without_wrapper" sha256sum "$path" | awk '{print $1}'
+    return 0
+  fi
+  if PATH="$path_without_wrapper" command -v openssl >/dev/null 2>&1; then
+    PATH="$path_without_wrapper" openssl dgst -sha256 -r "$path" | awk '{print $1}'
+    return 0
+  fi
+  printf '%s' ""
+}
+
 fallback_command_version() {
   local cmd="$1"
   local version_args=("--version")
@@ -122,7 +140,7 @@ fallback_commands_json() {
 
   local commands="${BK_SNAPSHOT_TOOL_COMMANDS:-$default_commands}"
   local first=true
-  local cmd path real_path version
+  local cmd path real_path version sha256
 
   printf '{'
   for cmd in $commands; do
@@ -132,16 +150,18 @@ fallback_commands_json() {
     fi
     real_path=$(fallback_resolved_path "$path")
     version=$(fallback_command_version "$cmd")
+    sha256=$(fallback_command_sha256 "$real_path")
     if [ "$first" = true ]; then
       first=false
     else
       printf ','
     fi
-    printf '%s:{"path":%s,"real_path":%s,"version":%s}' \
+    printf '%s:{"path":%s,"real_path":%s,"version":%s,"sha256":%s}' \
       "$(json_string "$cmd")" \
       "$(json_string "$path")" \
       "$(json_string "$real_path")" \
-      "$(json_string "$version")"
+      "$(json_string "$version")" \
+      "$(json_string "$sha256")"
   done
   printf '}'
 }
@@ -335,6 +355,15 @@ if [ "${BK_BUILD_TOOL_WRAPPER_SNAPSHOT:-true}" != "false" ]; then
     echo "bk_build_tool_wrapper: cannot collect build environment snapshot" >&2
     exit 1
   fi
+  if [ "$snapshot_written" = true ] && [ "${BK_BUILD_CACHE_LATE_RESTORE_PROBE:-false}" = "true" ]; then
+    late_restore_status_file="${BK_BUILD_CACHE_LATE_RESTORE_STATUS_FILE:-${repo_root}/results/build_cache_late_restore.env}"
+    mkdir -p "$(dirname "$late_restore_status_file")"
+    {
+      printf 'BK_BUILD_CACHE_LATE_RESTORE_TOOL=%s\n' "$tool_name"
+      printf 'BK_BUILD_CACHE_LATE_RESTORE_SNAPSHOT=%s\n' "$snapshot_file"
+    } > "$late_restore_status_file"
+    exit "${BK_BUILD_CACHE_LATE_RESTORE_EXIT_CODE:-86}"
+  fi
 fi
 
 exec "$real_tool" "$@"

scripts/build_with_cache.sh

@@ -29,6 +29,8 @@ if [ -n "$cache_root" ]; then
   cache_manifest="${cache_dir}/manifest.env"
 fi
 status_file="${repo_root}/results/build_cache.env"
+late_restore_exit_code=86
+late_restore_status_file="${repo_root}/results/build_cache_late_restore.env"
 
 source "${repo_root}/scripts/bk_functions.sh"
 
@@ -178,6 +180,50 @@ build_inputs_hash() {
   rm -f "$file_list" "$hash_list"
 }
 
+host_environment_fingerprint() {
+  local snapshot_file="${repo_root}/results/environment_snapshot_build_actual.json"
+  local fingerprint_input
+  local fingerprint_hash
+
+  if [ ! -f "$snapshot_file" ]; then
+    return 1
+  fi
+  if ! command -v jq >/dev/null 2>&1; then
+    return 1
+  fi
+
+  fingerprint_input=$(mktemp)
+  if ! jq -S -c \
+    --arg code "$code" \
+    --arg system "$system" \
+    '{
+      schema: "benchkit-host-build-env-v1",
+      code: $code,
+      system: $system,
+      toolchain: {
+        modules: (.toolchain.modules // []),
+        commands: (
+          .toolchain.commands // {}
+          | with_entries(
+              .value |= {
+                real_path: (.real_path // ""),
+                version: (.version // ""),
+                sha256: (.sha256 // "")
+              }
+            )
+        ),
+        environment: (.toolchain.environment // {})
+      }
+    }' "$snapshot_file" > "$fingerprint_input"; then
+    rm -f "$fingerprint_input"
+    return 1
+  fi
+  fingerprint_hash=$(bk_sha256_file "$fingerprint_input")
+  rm -f "$fingerprint_input"
+  [ -n "$fingerprint_hash" ] || return 1
+  printf '%s\n' "$fingerprint_hash"
+}
+
 resolve_git_branch_commit() {
   local repo_url="$1"
   local branch="$2"
@@ -208,6 +254,8 @@ validate_cached_source() {
   local container_path
   local cached_container_sha256
   local current_container_sha256
+  local cached_host_fingerprint
+  local current_host_fingerprint
 
   source_type=$(env_file_value "$source_info_file" BK_SOURCE_TYPE)
   case "$source_type" in
@@ -258,6 +306,25 @@ validate_cached_source() {
     return 0
   fi
 
+  cached_host_fingerprint=$(manifest_value BK_CACHE_HOST_ENV_FINGERPRINT)
+  if [ -n "$cached_host_fingerprint" ]; then
+    if ! current_host_fingerprint=$(host_environment_fingerprint); then
+      echo "host build cache restore requires actual environment fingerprint"
+      return 1
+    fi
+    if [ "$current_host_fingerprint" != "$cached_host_fingerprint" ]; then
+      echo "host build environment fingerprint changed"
+      return 1
+    fi
+    if [ -n "$(manifest_value BK_CACHE_ENV_KEY_B64)" ] || [ -n "${BK_BUILD_CACHE_ENV_KEY:-}" ]; then
+      if [ "$(manifest_value BK_CACHE_ENV_KEY_B64 | decode_base64_value 2>/dev/null || true)" != "${BK_BUILD_CACHE_ENV_KEY:-}" ]; then
+        echo "host build cache environment key changed"
+        return 1
+      fi
+    fi
+    return 0
+  fi
+
   if [ "${BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE:-false}" != "true" ]; then
     echo "host build cache restore requires BK_BUILD_CACHE_ALLOW_HOST_ENV_CACHE=true"
     return 1
@@ -323,6 +390,7 @@ store_cache() {
   local source_info_sha256=""
   local source_info_file="${repo_root}/results/source_info.env"
   local container_sha256=""
+  local host_fingerprint=""
 
   if [ "${BK_BUILD_CACHE_ENABLED:-true}" != "true" ]; then
     write_status disabled "BK_BUILD_CACHE_ENABLED is not true"
@@ -337,9 +405,12 @@ store_cache() {
     return 0
   fi
   container_sha256=$(env_file_value "$source_info_file" BK_CONTAINER_IMAGE_SHA256SUM)
-  if [ -z "$container_sha256" ] && [ -z "${BK_BUILD_CACHE_ENV_KEY:-}" ]; then
-    write_status miss "host build cache store requires non-empty BK_BUILD_CACHE_ENV_KEY"
-    return 0
+  if [ -z "$container_sha256" ]; then
+    host_fingerprint=$(host_environment_fingerprint || true)
+    if [ -z "$host_fingerprint" ] && [ -z "${BK_BUILD_CACHE_ENV_KEY:-}" ]; then
+      write_status miss "host build cache store requires environment fingerprint or non-empty BK_BUILD_CACHE_ENV_KEY"
+      return 0
+    fi
   fi
 
   inputs_hash=$(build_inputs_hash)
@@ -359,6 +430,7 @@ store_cache() {
     printf 'BK_CACHE_SYSTEM_B64=%s\n' "$(bk_base64_encode_value "$system")"
     printf 'BK_CACHE_BUILD_INPUTS_SHA256=%s\n' "$inputs_hash"
     printf 'BK_CACHE_SOURCE_INFO_SHA256=%s\n' "$source_info_sha256"
+    printf 'BK_CACHE_HOST_ENV_FINGERPRINT=%s\n' "$host_fingerprint"
     printf 'BK_CACHE_ENV_KEY_B64=%s\n' "$(bk_base64_encode_value "${BK_BUILD_CACHE_ENV_KEY:-}")"
     printf 'BK_CACHE_CREATED_AT=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
   } > "${tmp_dir}/manifest.env"
@@ -370,6 +442,56 @@ store_cache() {
   echo "build cache: stored ${code}/${system}"
 }
 
+should_attempt_late_host_restore() {
+  if [ "${BK_BUILD_CACHE_LATE_HOST_RESTORE:-true}" != "true" ]; then
+    return 1
+  fi
+  if [ "${BK_BUILD_CACHE_ENABLED:-true}" != "true" ]; then
+    return 1
+  fi
+  cache_dir_available || return 1
+  [ -f "$cache_manifest" ] || return 1
+  [ -d "${cache_dir}/artifacts" ] || return 1
+  [ -f "${cache_dir}/results/source_info.env" ] || return 1
+  [ -n "$(manifest_value BK_CACHE_HOST_ENV_FINGERPRINT)" ] || return 1
+}
+
+late_restore_hit=false
+run_build_with_optional_late_host_restore() {
+  local build_status=0
+
+  if ! should_attempt_late_host_restore; then
+    bash "${program_path}/build.sh" "$system"
+    return $?
+  fi
+
+  echo "build cache: probing host build environment for ${code}/${system}"
+  rm -f "$late_restore_status_file"
+  set +e
+  BK_BUILD_CACHE_LATE_RESTORE_PROBE=true \
+    BK_BUILD_CACHE_LATE_RESTORE_EXIT_CODE="$late_restore_exit_code" \
+    BK_BUILD_CACHE_LATE_RESTORE_STATUS_FILE="$late_restore_status_file" \
+    bash "${program_path}/build.sh" "$system"
+  build_status=$?
+  set -e
+
+  if [ "$build_status" -eq "$late_restore_exit_code" ] && [ -f "$late_restore_status_file" ]; then
+    if restore_cache; then
+      late_restore_hit=true
+      return 0
+    fi
+    echo "build cache: late host restore miss for ${code}/${system}; running build.sh"
+    rm -f "$late_restore_status_file"
+    bash "${program_path}/build.sh" "$system"
+    return $?
+  fi
+
+  if [ "$build_status" -ne 0 ]; then
+    return "$build_status"
+  fi
+  return 0
+}
+
 validate_path_component "$code" code
 validate_path_component "$system" system
 
@@ -392,5 +514,8 @@ if restore_cache; then
 fi
 
 echo "build cache: miss for ${code}/${system}; running build.sh"
-bash "${program_path}/build.sh" "$system"
+run_build_with_optional_late_host_restore
+if [ "$late_restore_hit" = true ]; then
+  exit 0
+fi
 store_cache

scripts/collect_environment_snapshot.sh

@@ -34,6 +34,24 @@ resolved_command_path() {
   printf '%s' "$path"
 }
 
+command_sha256() {
+  local path="$1"
+
+  if [ -z "$path" ] || [ ! -f "$path" ] || [ ! -r "$path" ]; then
+    printf '%s' ""
+    return 0
+  fi
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum "$path" | awk '{print $1}'
+    return 0
+  fi
+  if command -v openssl >/dev/null 2>&1; then
+    openssl dgst -sha256 -r "$path" | awk '{print $1}'
+    return 0
+  fi
+  printf '%s' ""
+}
+
 command_version() {
   local cmd="$1"
   shift
@@ -90,7 +108,7 @@ snapshot_tool_commands_json() {
   default_commands+="cmake make ninja ld ar pkg-config python3 apptainer singularity ncu nsys"
 
   local commands="${BK_SNAPSHOT_TOOL_COMMANDS:-$default_commands}"
-  local cmd path real_path version
+  local cmd path real_path version sha256
 
   for cmd in $commands; do
     path=$(command_path "$cmd")
@@ -99,12 +117,14 @@ snapshot_tool_commands_json() {
     fi
     real_path=$(resolved_command_path "$path")
     version=$(snapshot_command_version "$cmd")
+    sha256=$(command_sha256 "$real_path")
     jq -n -c \
       --arg name "$cmd" \
       --arg path "$path" \
       --arg real_path "$real_path" \
       --arg version "$version" \
-      '{key: $name, value: {path: $path, real_path: $real_path, version: $version}}'
+      --arg sha256 "$sha256" \
+      '{key: $name, value: {path: $path, real_path: $real_path, version: $version, sha256: $sha256}}'
   done | jq -s -c 'from_entries'
 }