diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c4a4921..92c3ae65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ - **[Improvement]** `621dded` warn on provider version conflict; `4d92e95` point the "package not found" error at the provider mechanism; `ee78182` docs. ## CVMFS layout, merged views & relaxed reuse +- **[Feature]** `--reuse-from |cvmfs` reuses deployed components via their published modulefiles/`init.sh` (sourced in place from `/cvmfs`); `--reuse-policy strict|relaxed`, `::policy` sugar, per-dependency reuse, corrected pkg-config prefixes, and `cvmfs`-resolution from the `cvmfs_modules_template`. +- **[Change]** Removed the legacy `cvmfs://` `--remote-store` reuse: the `build_id` graft (`--reuse-base`/`--reuse-cvmfs`, `CVMFSRemoteSync`, `select_build_id`/`graftable_match`) is retired in favour of `--reuse-from`; a `cvmfs://` `--remote-store` now errors and points at `--reuse-from` (see ADR-0001, superseded). - **[Feature]** `0632f28` / `b70abac` / `b1dfc1d` / `e8d1222` / `ea04075` merged symlink-farm view: one-entry-per-var env, opt-in `enter/setenv --view`, view-aware `load`/`printenv` + age-based GC, path remap (fixes PyROOT). - **[Feature]** `58f13b6` / `c2d99be` / `f59a547` / `53e91d4` / `80b6a08` published per-`build_id` views on CVMFS, `bits publish --view`, per-tree pre-publish primitive, CVMFS layout recorded in `.meta.json`. - **[Feature]** `8f193fa`→`c52f5d7`, `cf19966` ADR-0001 import pipeline: modulefile harvest → classify → closure/`build_id` → overlay → `bits import` (build-sufficient from modulefiles). diff --git a/bits b/bits index 9d51d158..30689662 100755 --- a/bits +++ b/bits @@ -5,6 +5,19 @@ BITSDIR="$(dirname "$0")" +# `.bitscmd` saved-arg profile (see `bits use`): inject the [common] + [] +# tokens right after the action, BEFORE anything is parsed, so both the build +# path and the module commands (q/enter/…, which read ARGV below) see them. +# Runs python only when a profile exists; `use` opts out; explicit args stay last +# (they win on single-value options). Safe no-op if the action isn't first. +if [[ -f .bitscmd ]]; then + _bits_prof=() + while IFS= read -r -d '' _bits_t; do _bits_prof+=("$_bits_t"); done \ + < <(BITS_SELF="${BITSDIR}" python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.bits_use import main; sys.exit(main())' --rewrite0 -- "$@") + [[ ${#_bits_prof[@]} -gt 0 ]] && set -- "${_bits_prof[@]}" + unset _bits_prof _bits_t +fi + ARGV=("$@"); ARGC=$# # ARGC must be a plain integer, not an array # Optional phase timing for diagnosing slow load/enter: set BITS_TIMING=1 to @@ -375,6 +388,23 @@ if [[ "${1:-}" == "store" ]]; then exec "$BITSDIR/bitsStore" "$@" fi +# `bits use …` → save/show/clear a per-command arg profile in ./.bitscmd, so +# repeated commands stay short (the injection at the top of this script consumes +# it). Handled here, before work-dir/module setup: it only touches ./.bitscmd. +if [[ "${1:-}" == "use" ]]; then + shift + BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.bits_use import main; sys.exit(main())' "$@" +fi + +# `bits cvmfs {platforms,show,summary} …` → read-only inspection of a deployed +# CVMFS bits tree: platforms + host compatibility, a package's build/provenance +# from .meta.json (no jq), and a per-build_id summary. Reads the tree only, so it +# needs no work dir; distinct from cvmfs-stage/cvmfs-publish (producer side). +if [[ "${1:-}" == "cvmfs" ]]; then + shift + BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.cvmfs_inspect import main; sys.exit(main())' "$@" +fi + # `bits cvmfs-stage …` → producer-side CVMFS staging (ADR-0011). # Prepares a package into a staging S3 prefix with the canonical publisher and # names the catalog prepub must graft. Handled here, before work-dir/module @@ -400,7 +430,7 @@ fi for arg in "$@" do case $arg in - architecture|brew|build|certify|clean|cleanup|compliance|cvmfs-path|deps|doctor|gc|init|publish|stats|status|store-stats|verify|version|-debug|-d) + architecture|brew|build|certify|clean|cleanup|compliance|cvmfs-path|deps|doctor|gc|import|init|publish|stats|status|store-stats|verify|version|-debug|-d) mkdir -p "$BITS_WORK_DIR" || echo "Cannot create directory: $BITS_WORK_DIR" "$BITSDIR/bitsBuild" "$@" exit $? diff --git a/bits_helpers/args.py b/bits_helpers/args.py index c303fa59..a8169f23 100644 --- a/bits_helpers/args.py +++ b/bits_helpers/args.py @@ -380,8 +380,8 @@ def doParseArgs(): "manifest), closure-check the set, stamp it with one deterministic " "build_id, and generate a per-build_id overlay (build-sufficient bits " "modulefiles + module-side .meta.json + .cvmfscatalog) that " - "'bits build --reuse-policy relaxed --reuse-base ' can graft " - "without recompiling. See ADR-0001." + "'bits build --reuse-from |cvmfs' can reuse without " + "recompiling." ), ) import_parser.add_argument("-w", "--work-dir", dest="workDir", @@ -397,6 +397,15 @@ def doParseArgs(): metavar="FILE", default=None, help="JSON manifest to import instead of harvesting (fallback when " "no modulefiles exist).") + import_parser.add_argument("--trusted", dest="importTrusted", action="store_true", + help="Trusted mode: harvest a bits-built deployment directly, reading " + "its own modulefiles (--modulepath) and re-anchoring them to " + "--install-base, capturing package hashes from the install tree. " + "Deterministic (no modulecmd); publishable strict reuse.") + import_parser.add_argument("--install-base", dest="importInstallBase", + metavar="DIR", default=None, + help="With --trusted, the absolute Packages root the modulefiles' " + "BASEDIR resolves to (and where each package's .meta.json lives).") import_parser.add_argument("--aliases", dest="importAliases", metavar="FILE", default=None, help="JSON name-alias map (foreign -> bits names).") @@ -673,10 +682,6 @@ def doParseArgs(): is set to the same value). May be set to a default store on some architectures; use --no-remote-store to disable it in that case. """) - build_remote.add_argument("--reuse-cvmfs", dest="reuseCvmfs", action="store_true", - help=("Reuse already-deployed components from the CVMFS area declared by the " - "defaults `cvmfs_dir:` field. Sets --remote-store to cvmfs:// " - "when no remote store is given.")) build_remote.add_argument("--sign-manifest", dest="signManifest", default=None, metavar="KEY.pem", help=("After the build, sign the build manifest with this Ed25519 private key " "(PEM) so consumers can verify it for trusted reuse. Public verification " @@ -727,13 +732,14 @@ def doParseArgs(): "build_id) for fast local dev; the result is loose-provenance and is " "refused by the publish path. Falls back to the defaults `reuse_policy:` " "value, else 'strict'.")) - build_remote.add_argument("--reuse-base", dest="reuseBase", metavar="BUILD_ID", default=None, - help=("With --reuse-policy relaxed, the build_id of the deployed release to " - "graft packages from. Falls back to the defaults `reuse_base:` value. " - "Special values auto-select from the cvmfs:// store: 'latest' (newest " - "build_id for the build target) and 'latest-common' (newest build_id " - "shared by all requested packages); the same happens when left empty " - "under relaxed. The chosen build_id is announced.")) + build_remote.add_argument("--reuse-from", dest="reuseFrom", metavar="PATH|cvmfs", default=None, + help=("Reuse deployed components via their published modulefiles at this " + "absolute modules-tree path (distinct from --remote-store, which is the " + "tarball store). The literal 'cvmfs' resolves the exact location from the " + "defaults `system:` layout (module_dir under cvmfs_dir); fails if that is " + "not configured. A trailing '::relaxed' or '::strict' also sets the reuse " + "policy (e.g. 'cvmfs::relaxed'); it must agree with --reuse-policy if both " + "are given.")) build_remote.add_argument("--build-local", dest="buildLocal", metavar="PKG[,PKG...]", default="", help=("Comma-separated packages to always build locally even under " "--reuse-policy relaxed (e.g. a package you need patched), rather than " diff --git a/bits_helpers/bits_use.py b/bits_helpers/bits_use.py new file mode 100644 index 00000000..22c82e03 --- /dev/null +++ b/bits_helpers/bits_use.py @@ -0,0 +1,222 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""`bits use` — save reusable command-line args in ``./.bitscmd`` so repeated +commands stay short. Distinct from ``.bitsrc`` (typed key=value settings): +``.bitscmd`` holds raw CLI tokens, structured by the command they apply to. + +A ``[common]`` section is injected into arch-aware commands (put only broadly +accepted args here, i.e. ``--architecture``, which ``build`` AND ``q``/``enter``/ +``clean`` all take); a per-command section (``[build]``, ``[q]``, …) adds args +that command accepts — e.g. ``--defaults`` belongs in ``[build]``, NOT +``[common]`` (module commands and ``clean``/``import`` don't accept it). Injected +BEFORE the user's own args, so those override single-value options. Example:: + + [common] + --architecture x86_64-el9-gcc14-opt + + [build] + --defaults lcg::release::gcc14::opt --docker --sandbox off --reuse-from cvmfs::relaxed + +Prototype — runs standalone:: + + python3 -m bits_helpers.bits_use common --architecture x86_64-el9-gcc14-opt + python3 -m bits_helpers.bits_use build --docker --sandbox off + python3 -m bits_helpers.bits_use # show all sections + python3 -m bits_helpers.bits_use --clear [SECTION] +""" + +import os +import shlex +import sys + +PROFILE = ".bitscmd" +COMMON = "common" # section injected into every command ('global' alias) + + +def _join(tokens): + try: + return shlex.join(tokens) # Python 3.8+ + except AttributeError: # pragma: no cover + return " ".join(shlex.quote(t) for t in tokens) + + +def read_all(path=PROFILE): + """Parse the profile into an ordered ``{section: [tokens]}`` dict. + + ``[name]`` opens a section; lines before any header belong to ``common``; + ``#`` comments and blank lines are ignored. Each section's lines are joined + and shlex-split into tokens. + """ + sections, cur, buf = {}, COMMON, [] + if not os.path.exists(path): + return sections + + def _flush(): + if buf: + sections.setdefault(cur, []) + sections[cur] += shlex.split(" ".join(buf)) + try: + with open(path) as fh: + for raw in fh: + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("[") and line.endswith("]"): + _flush(); buf = [] + cur = line[1:-1].strip().lower() + cur = COMMON if cur in ("global", COMMON) else cur + sections.setdefault(cur, []) + continue + buf.append(line) + _flush() + except (OSError, ValueError): + # ValueError: a malformed token (e.g. an unbalanced quote) in the + # profile. Fail safe — ignore the profile rather than crash a build. + return {} + return sections + + +def write_section(section, tokens, path=PROFILE): + """Replace *section* with *tokens*, preserving the other sections.""" + section = COMMON if section in ("global", COMMON) else section.lower() + sections = read_all(path) + sections[section] = list(tokens) + _write_all(sections, path) + return path + + +def clear_section(section=None, path=PROFILE): + """Clear one section, or the whole profile when *section* is None.""" + if section is None: + try: + os.unlink(path); return True + except OSError: + return False + sections = read_all(path) + section = COMMON if section in ("global", COMMON) else section.lower() + if section in sections: + del sections[section] + _write_all(sections, path) + return True + return False + + +def _write_all(sections, path=PROFILE): + order = [COMMON] + [s for s in sections if s != COMMON] + with open(path, "w") as fh: + for s in order: + toks = sections.get(s) + if not toks: + continue + fh.write("[%s]\n%s\n\n" % (s, _join(toks))) + + +def merged_argv(command, user_args, path=PROFILE): + """Args to run for *command*: ``[common]`` then ``[command]`` then the + user's own args (which come last and win on single-value options).""" + sec = read_all(path) + return sec.get(COMMON, []) + sec.get((command or "").lower(), []) + list(user_args) + + +# Top-level flags that may precede the action (from the root argparse parser); +# skipped when locating the action token. +TOP_FLAGS = {"-d", "--debug", "-n", "--dry-run"} + +# The profile is injected ONLY into arch-aware commands (they all accept +# `--architecture`, the intended `[common]` content). Meta commands (use, cvmfs, +# store, version, help, cvmfs-stage/publish) and `verify` (accepts neither +# --architecture nor --defaults) take a different option set and are excluded. +# Note: several of these accept --architecture but NOT --defaults, so --defaults +# belongs in per-command sections ([build], …), never in [common]. +INJECT_ACTIONS = { + "build", "deps", "doctor", "status", "clean", "cleanup", "gc", + "import", "publish", "certify", "compliance", + "q", "query", "enter", "setenv", "printenv", "load", "unload", +} + + +def _find_action(argv): + """Index of the action token in *argv* (first non-top-flag word), or None.""" + for i, tok in enumerate(argv): + if tok in TOP_FLAGS: + continue + if tok.startswith("-"): + return None # an option before any action → leave as-is + return i + return None + + +def rewrite_argv(argv, path=PROFILE): + """Return *argv* with the ``.bitscmd`` profile injected right after the action + token: ``[common]`` plus ``[]``, for arch-aware actions only + (``INJECT_ACTIONS``). A no-op when there is no profile, no action, or the + action is a meta command (``use``/``cvmfs``/``store``/…). This is the single + entry point the wrapper calls at startup. + """ + argv = list(argv) + sec = read_all(path) + if not sec: + return argv + ai = _find_action(argv) + if ai is None: + return argv + action = argv[ai].lower() + if action not in INJECT_ACTIONS: + return argv + inject = sec.get(COMMON, []) + sec.get(action, []) + if not inject: + return argv + return argv[:ai + 1] + inject + argv[ai + 1:] + + +# ── CLI ────────────────────────────────────────────────────────────────────── + +def _show(path=PROFILE): + sec = read_all(path) + if not sec: + print("no .bitscmd in %s" % os.getcwd()); return 0 + for s, toks in sec.items(): + if toks: + print("[%s] %s" % (s, _join(toks))) + return 0 + + +def main(argv=None): + argv = list(sys.argv[1:] if argv is None else argv) + # Wrapper hook: emit the profile-injected argv, NUL-separated, for the args + # after '--'. `mapfile -t -d '' arr < <(bits_use --rewrite0 -- "$@")`. + if argv and argv[0] == "--rewrite0": + rest = argv[2:] if len(argv) > 1 and argv[1] == "--" else argv[1:] + out = rewrite_argv(rest) + if out != rest: # injection happened → tell the user + n = len(out) - len(rest) + sys.stderr.write("using .bitscmd (+%d arg%s)\n" % (n, "" if n == 1 else "s")) + # NUL-terminate EVERY token (trailing NUL included) so the wrapper's + # `while read -d ''` loop captures the final token — an unterminated last + # field is assigned but the loop exits before appending it (dropping it). + sys.stdout.write("".join(t + "\0" for t in out)) + return 0 + if not argv: + return _show() + if argv[0] in ("--clear", "clear"): + sect = argv[1] if len(argv) > 1 else None + ok = clear_section(sect) + print(("cleared %s" % (sect or "all")) if ok else "nothing to clear") + return 0 + # Optional leading section name (a bare word); otherwise default to common. + if argv[0] and not argv[0].startswith("-"): + section, rest = argv[0], argv[1:] + else: + section, rest = COMMON, argv + if not rest: + print("no args given for [%s]" % section); return 1 + write_section(section, rest) + print("saved to [%s] in %s: %s" % ( + COMMON if section in ("global", COMMON) else section.lower(), + PROFILE, _join(rest))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index feec667b..2d9226a3 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -819,24 +819,6 @@ def storeHashes(package, specs, considerRelocation): # subsequent calculations. return - # Relaxed CVMFS graft (ADR-0001): a grafted package adopts the *deployed* - # artifact's hash. The existing reuse path (CVMFSRemoteSync.fetch_symlinks + - # the reuse decision) then materialises and symlinks the deployed tree under - # that hash instead of building, and consumers hash against the real deployed - # dependency — so no separate build-skip branch is needed. Only triggers when - # the resolver tagged the spec from_cvmfs (relaxed mode); never in strict. - if spec.get("from_cvmfs") and spec.get("cvmfs_hash"): - _h = spec["cvmfs_hash"] - spec["remote_revision_hash"] = _h - spec["local_revision_hash"] = _h - spec["remote_hashes"] = [_h] - spec["local_hashes"] = [_h] - spec["hash"] = _h - # The grafted package has no followed dependencies; set deps_hash too (the - # normal path always sets it, and DEPS_HASH is read via spec.get downstream). - spec.setdefault("deps_hash", "") - return - # For now, all the hashers share data -- they'll be split below. h_all = Hasher() @@ -1105,7 +1087,8 @@ def _pkg_install_path(workDir, architecture, spec): def generate_initdotsh(package, specs, architecture, workDir="sw", post_build=False, - from_modules=False, cmake_prefix_env=False): + from_modules=False, cmake_prefix_env=False, + reuse_cvmfs_base=None): """Return the contents of the given package's etc/profile/init.sh as a string. If post_build is true, also generate variables pointing to the package @@ -1173,7 +1156,77 @@ def _dep_init_path(dep): package=quote(dep_spec["package"]), ver_rev=quote(ver_rev(dep_spec)), ) - lines.extend(_dep_init_path(dep) for dep in spec.get("requires", ())) + # A dependency satisfied from a reused CVMFS release is set up by sourcing its + # DEPLOYED init.sh from CVMFS — the same mechanism as a local dep, just from + # the deployment. The deployed init.sh resolves paths via "$WORK_DIR/ + # $BITS_ARCH_PREFIX", so we point those at the CVMFS Packages base while + # sourcing (and restore after) so its own and its transitive deps' paths land + # on CVMFS. Per-DEPENDENCY, so a legacy-built package can consume a reused dep. + # Needs /cvmfs mounted in the build container (no modulecmd required). + _reqs = list(spec.get("requires", ())) + _reused_set = {d for d in _reqs + if reuse_cvmfs_base and specs[d].get("reuse_module_id")} + + def _reused_dep_lines(d): + # Point the deployed init.sh's "$WORK_DIR/$BITS_ARCH_PREFIX" at the CVMFS + # Packages base. BITS_ARCH_PREFIX MUST be non-null (the deployed init.sh's + # `: "${BITS_ARCH_PREFIX:=}"` would otherwise re-add the arch); "." is + # a harmless no-op segment (/./ == /). Save/restore so + # locally-built deps keep the local WORK_DIR. + dep_spec = specs[d] + verrev = dep_spec["reuse_module_id"].split("/", 1)[1] + return [ + '_bits_swd="${WORK_DIR:-}"; _bits_sap="${BITS_ARCH_PREFIX:-}"', + 'WORK_DIR="%s"; BITS_ARCH_PREFIX="."' % reuse_cvmfs_base, + '[ -n "${%s_REVISION}" ] || . "%s/%s/%s/etc/profile.d/init.sh"' + % (pkg_to_shell_id(d), reuse_cvmfs_base, dep_spec["package"], verrev), + 'WORK_DIR="${_bits_swd}"; BITS_ARCH_PREFIX="${_bits_sap}"; ' + 'unset _bits_swd _bits_sap', + ] + + if _reused_set: + # Emit deps in topological order (prerequisites first) so a dep set up + # before a reused dep whose deployed init.sh transitively references it — + # e.g. a locally-built bits-recipe-tools before a reused CMake — sets its + # _REVISION first, and the deployed init.sh's guard skips the re-source + # (which would look on CVMFS where a local-only build does not exist). + _req_set = set(_reqs) + _order = [d for d in topological_sort(specs) if d in _req_set] + for d in _order: + if d in _reused_set: + lines.extend(_reused_dep_lines(d)) + else: + lines.append(_dep_init_path(d)) + # A reused CVMFS package may ship a pkg-config .pc whose baked `prefix=` does + # not match its deployed location (publish-time relocation can misplace it), + # breaking find_package via pkg-config for a consumer (e.g. xrootd → Davix). + # The reuse anchoring already resolved each dep's real root into _ROOT, + # so stage corrected .pc copies (prefix rewritten to that root) in a writable + # dir and prepend it to PKG_CONFIG_PATH. Reads from read-only /cvmfs, writes + # under $WORK_DIR; a no-op for reused deps that ship no .pc. + _reused_roots = " ".join('"${%s_ROOT:-}"' % pkg_to_shell_id(d) + for d in _order if d in _reused_set) + lines.extend([ + '_bits_rpc="${WORK_DIR:-.}/reuse-pkgconfig"; mkdir -p "$_bits_rpc"', + 'for _bits_root in %s; do' % _reused_roots, + ' [ -n "$_bits_root" ] || continue', + ' for _bits_pcd in "$_bits_root/lib64/pkgconfig" "$_bits_root/lib/pkgconfig"; do', + ' [ -d "$_bits_pcd" ] || continue', + ' for _bits_pc in "$_bits_pcd"/*.pc; do', + ' [ -e "$_bits_pc" ] || continue', + ' sed "s|^prefix=.*|prefix=$_bits_root|" "$_bits_pc" > "$_bits_rpc/${_bits_pc##*/}"', + ' done', + ' done', + 'done', + # Prepend once — init.sh may be sourced repeatedly; avoid unbounded growth. + 'case ":${PKG_CONFIG_PATH:-}:" in', + ' *":$_bits_rpc:"*) ;;', + ' *) export PKG_CONFIG_PATH="$_bits_rpc${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" ;;', + 'esac', + 'unset _bits_rpc _bits_root _bits_pcd _bits_pc', + ]) + else: + lines.extend(_dep_init_path(dep) for dep in _reqs) if post_build: bigpackage = pkg_to_shell_id(package) @@ -1438,21 +1491,11 @@ def dependency_list(key): from bits_helpers.provenance import ( compute_build_id, compute_abi_tag, recipe_tools_ref, ) - # Contagious provenance (ADR-0001): a locally-built package is "loose" when its - # dependency closure contains a package grafted from CVMFS (adopted by - # name/build_id, not verified hash). Grafted packages are not built, so this - # function only ever runs for local builds. - def _closure_grafted(): - for _key in ("full_build_requires", "full_runtime_requires"): - for _dep in specs[package].get(_key, ()): - _ds = specs.get(_dep) - if isinstance(_ds, dict) and _ds.get("from_cvmfs"): - return True - return False - # A build is also loose if its closure decoupled a dependency via - # untracked_requires: this package, or one below it, was hashed as if that - # dependency never changed, so its identity no longer certifies its full input - # closure. Contagious upward like grafted provenance. + # Contagious provenance: a build is "loose" when its closure decoupled a + # dependency via untracked_requires — this package, or one below it, was hashed + # as if that dependency never changed, so its identity no longer certifies its + # full input closure. (Relaxed --reuse-from builds are kept non-publishable by + # the reuse-policy publish guard, so they never reach a published record.) def _closure_untracked(): if specs[package].get("untracked_requires"): return True @@ -1463,7 +1506,7 @@ def _closure_untracked(): return True return False _untracked = list(specs[package].get("untracked_requires", ())) - _provenance = "loose" if (_closure_grafted() or _closure_untracked()) else "pure" + _provenance = "loose" if _closure_untracked() else "pure" return json.dumps({ "comment": args.annotate.get(package), "bits_version": __version__, @@ -2376,14 +2419,9 @@ def defaultsReader(): # ── CVMFS layout (templated dirs from defaults-release) ──────────────────── # When defaults declare cvmfs_dir / install_dir / module_dir (templates that - # may use %(architecture)s), resolve them and use them to default the build/ - # reuse flags so the whole CVMFS chain can be driven from one declaration: - # * reuse deployed -> --remote-store = cvmfs:// (with --reuse-cvmfs) - # Docker builds are relocatable by default (built in WORK_DIR with padded - # placeholders + relocate-me.sh), so the tarballs can be reused anywhere and are - # relocated into CVMFS on publish. Pass --cvmfs-prefix explicitly only for an - # in-place, non-relocatable build (skips relocation on publish, but the result - # is NOT reusable outside that exact CVMFS path). + # may use %(architecture)s), resolve them so create_provenance_info can record + # the tree paths in each package's .meta.json. Reusing deployed components is + # driven by --reuse-from (module overlay), not the remote/tarball store. from bits_helpers.cvmfs_layout import resolve_cvmfs_layout _cvmfs = resolve_cvmfs_layout(defaultsMeta, args.architecture) # Stash the resolved layout so create_provenance_info can record it in each @@ -2394,9 +2432,62 @@ def defaultsReader(): if _cvmfs: info("CVMFS layout: install=%s modules=%s views=%s", _cvmfs["install_path"], _cvmfs["module_path"], _cvmfs["views_path"]) - if getattr(args, "reuseCvmfs", False) and not args.remoteStore and _cvmfs["cvmfs_dir"]: - args.remoteStore = "cvmfs://" + _cvmfs["cvmfs_dir"] - info("Reusing deployed components: --remote-store %s", args.remoteStore) + + # Resolve --reuse-from into an absolute modules-tree path ('cvmfs' -> the + # defaults system: layout module_path). Nothing consumes it yet (later step). + from bits_helpers.cvmfs_layout import (resolve_reuse_from, split_reuse_policy, + reuse_module_path_from_templates) + # Sugar: a trailing '::relaxed'/'::strict' on --reuse-from sets the reuse + # policy alongside the source (reconciled with --reuse-policy below). + _reuse_src, _reuse_from_policy = split_reuse_policy(getattr(args, "reuseFrom", None)) + # --reuse-from cvmfs prefers the system: layout module_path; if that is not + # declared, fall back to the group's cvmfs_modules_template (one declaration + # drives both publish and reuse). Expanded with raw_architecture — the DEPLOYED + # arch — not the build-qualified family, so it matches the deployment. + _reuse_layout = _cvmfs + if _reuse_src == "cvmfs" and not (_cvmfs and _cvmfs.get("module_path")): + _mp = reuse_module_path_from_templates( + defaultsMeta, raw_architecture, os.environ.get("BITS_CVMFS_PREFIX") or None) + if _mp: + _reuse_layout = dict(_cvmfs or {}, module_path=_mp) + try: + args.reuseFrom = resolve_reuse_from(_reuse_src, _reuse_layout) + except ValueError as exc: + dieOnError(True, str(exc)) + # Build the local, re-anchored overlay from the deployment named by + # --reuse-from, so reused deps can be set up via modules. The install base (the + # deployment's Packages root) comes from the layout — available for + # --reuse-from cvmfs. Dormant until packages are marked reused (next step): the + # overlay is produced and its path threaded to generate_initdotsh, but nothing + # is grafted yet, so the build is unchanged. + args.reuseOverlay = None + args.reuseBuildId = None + args.reuseCvmfsBase = None + if args.reuseFrom: + info("Reuse-from modules: %s", args.reuseFrom) + _install_base = _cvmfs.get("install_path") if _cvmfs else None + if not _install_base and "Modules/modulefiles" in args.reuseFrom: + # Explicit --reuse-from path, no layout: derive the Packages root by the + # deployment convention (the same Modules/modulefiles<->Packages map the + # BASE module uses). + _install_base = args.reuseFrom.replace("Modules/modulefiles", "Packages") + dieOnError(not _install_base, + "--reuse-from needs the deployment's Packages root; declare " + "install_dir / cvmfs_dir in the defaults system: layout, or point " + "--reuse-from at a .../Modules/modulefiles tree.") + from bits_helpers.cvmfs_import import import_trusted_release + _res = import_trusted_release(args.reuseFrom, _install_base, args.architecture, + os.path.join(workDir, "MODULES")) + if _res.get("build_id"): + args.reuseOverlay = _res["overlay_path"] # host path, for reading + args.reuseBuildId = _res["build_id"] + args.reuseCvmfsBase = _install_base # CVMFS Packages base for init.sh + info("Reuse overlay: %d module(s) -> %s", + len(_res["written"]), args.reuseOverlay) + else: + warning("Reuse overlay: release under %s is not closed (missing: %s); " + "no modules imported.", args.reuseFrom, + ", ".join(_res.get("dangling", []))) # Build-host policy knobs live under a single `system:` entry in defaults. # These control *how* the build runs (network, CPU) — not *what* it produces — @@ -2558,45 +2649,22 @@ def _truthy(v): args.criticalPathSchedule = _cp if isinstance(_cp, bool) \ else str(_cp).strip().lower() in ("1", "true", "yes", "on") - # Relaxed CVMFS reuse policy (ADR-0001). Non-hashed build-host policy, like - # the two above. Precedence: explicit --reuse-policy/--reuse-base > defaults - # system.reuse_policy / reuse_base > strict / none. Default strict keeps the - # simple aliBuild case bit-for-bit unchanged. + # Reuse policy (strict/relaxed) for the --reuse-from module overlay. Strict + # reuses only on exact content-hash match (publishable); relaxed matches any + # version in the one-release overlay (loose provenance, non-publishable). + # An explicit --reuse-policy is canonical; the --reuse-from '::policy' suffix + # is sugar. If both are given they must agree; otherwise the suffix fills in. + # Precedence: --reuse-policy > --reuse-from ::policy > defaults > strict. + if getattr(args, "reusePolicy", None) is not None and _reuse_from_policy \ + and args.reusePolicy != _reuse_from_policy: + dieOnError(True, + "--reuse-policy %s conflicts with --reuse-from ...::%s; " + "specify the policy once." % (args.reusePolicy, _reuse_from_policy)) if getattr(args, "reusePolicy", None) is None: - args.reusePolicy = str(_system_opt("reuse_policy", "strict")).strip().lower() + args.reusePolicy = _reuse_from_policy \ + or str(_system_opt("reuse_policy", "strict")).strip().lower() if args.reusePolicy not in ("strict", "relaxed"): args.reusePolicy = "strict" - if getattr(args, "reuseBase", None) is None: - args.reuseBase = _system_opt("reuse_base", "") or "" - # Relaxed reuse: auto-select a build_id from the CVMFS store when none was - # given (or the sentinels "latest" / "latest-common"), so the user need not - # copy an id by hand. Anchors on the build target for "latest"; requires all - # requested packages to share it for "latest-common". Announced loudly; an - # explicit --reuse-base always wins. Uses the (combined) architecture, which - # must match the deployed /Packages dir name. - if args.reusePolicy == "relaxed" \ - and str(args.reuseBase).strip().lower() in ("", "latest", "latest-common"): - _strategy = "latest-common" if str(args.reuseBase).strip().lower() == "latest-common" \ - else "latest" - _store = args.remoteStore or "" - if not _store.startswith("cvmfs://"): - warning("relaxed reuse: auto-select needs a cvmfs:// --remote-store; " - "no build_id selected, packages will be built.") - args.reuseBase = "" - else: - from bits_helpers.cvmfs_reuse import select_build_id - _root = re.sub("^cvmfs://", "", _store) - _bid, _cov = select_build_id(packages, args.architecture, _root, _strategy) - if _bid: - banner("relaxed reuse: auto-selected build_id '%s' (%s)\n" - " from %s/%s/Packages, covering %d/%d requested package(s)", - _bid, _strategy, _root, args.architecture, - len(_cov.get(_bid, ())), len(packages)) - args.reuseBase = _bid - else: - warning("relaxed reuse: no %s build_id found under %s/%s/Packages; " - "packages will be built.", _strategy, _root, args.architecture) - args.reuseBase = "" # Publish guard: relaxed builds are loose-provenance (their closure includes # unverified deployed binaries) and must never reach a write store / publish # pipeline. Refuse early and clearly. @@ -2730,31 +2798,6 @@ def performPreferCheckWithTempDir(pkg, cmd): with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{pkg['package']}_") as temp_dir: return getstatusoutput_docker(cmd, cwd=temp_dir) - # Relaxed CVMFS graft callback (ADR-0001). Active only under --reuse-policy - # relaxed with a cvmfs:// remote store and a --reuse-base build_id; None in - # every other case → strict behaviour, no graft (simple aliBuild path - # unaffected). Uses the combined architecture (args.architecture) — the arch - # recorded in the deployed packages' .meta.json — not raw_architecture. - _cvmfs_match = None - if getattr(args, "reusePolicy", "strict") == "relaxed": - _base = getattr(args, "reuseBase", "") or "" - _store = args.remoteStore or "" - if not _base: - warning("--reuse-policy relaxed needs --reuse-base (or defaults " - "reuse_base:); no packages will be grafted.") - elif not _store.startswith("cvmfs://"): - warning("--reuse-policy relaxed needs a cvmfs:// --remote-store " - "(or --reuse-cvmfs); no packages will be grafted.") - else: - from bits_helpers.cvmfs_reuse import graftable_match - _store_root = re.sub("^cvmfs://", "", _store) - _build_local = set(getattr(args, "buildLocal", []) or []) - def _cvmfs_match(spec, _root=_store_root, _bid=_base, - _arch=args.architecture, _bl=_build_local): - if spec["package"] in _bl: - return None - return graftable_match(spec["package"], _arch, _bid, _root) - systemPackages, ownPackages, failed, validDefaults = \ getPackageList(packages = packages, specs = specs, @@ -2772,8 +2815,7 @@ def _cvmfs_match(spec, _root=_store_root, _bid=_base, taps = taps, log = debug, provider_dirs = provider_dirs, - defaults_meta = defaultsMeta, - performCvmfsMatch = _cvmfs_match) + defaults_meta = defaultsMeta) _bad_defaults, _missing_flavor = incompatibleFlavorDefaults(validDefaults, args.defaults, defaultsMeta) dieOnError(bool(_bad_defaults) or _missing_flavor, @@ -2988,9 +3030,14 @@ def _build_row(pkg): spec["commit_hash"] = "0" if "sources" in spec: + # Expand a templated tag (e.g. "v%(version)s") for tarball sources too. + # The git branch above only resolves it when a `source:` is present, so a + # tarball-only recipe kept the raw tag, which then leaked into commit_hash + # and the SOURCES/// path. No-op for literal tags. + spec["tag"] = resolve_tag(spec, defaultsMeta.get("variables")) for i, s in enumerate(spec["sources"]): resolved = resolveLocalPath(args.configDir, s) - spec["sources"][i] = resolved + spec["sources"][i] = resolved spec["commit_hash"] = spec["tag"] # Version may contain date params like tag, plus %(commit_hash)s, # %(short_hash)s and %(tag)s. @@ -3278,6 +3325,13 @@ def _build_row(pkg): import atexit atexit.register(lambda ex=_prefetch_executor: ex.shutdown(wait=False, cancel_futures=True)) + # Default build_family for the final banner, matching the in-loop formula + # (build.py sets it per-package below). Needed so a reused/skipped MAIN package + # does not leave mainBuildFamily unset; the in-loop assignment overrides it + # whenever the main package is actually built. + _mdp = getattr(args, "develPrefix", develPackageBranch) + mainBuildFamily = ("{}-{}".format(_mdp, "_".join(args.defaults)) if _mdp + else "_".join(args.defaults)) while buildOrder: p = buildOrder.pop(0) spec = specs[p] @@ -3296,6 +3350,40 @@ def _build_row(pkg): debug("Hashes for recipe %s are %s (remote); %s (local)", p, ", ".join(spec["remote_hashes"]), ", ".join(spec["local_hashes"])) + # 4b: if the reuse overlay satisfies this package, set it up from modules + # instead of building. Its consumers 'module load' it (generate_initdotsh); + # it is neither built nor materialized locally, so we skip the whole + # build/unpack path here (this is what avoids the legacy tarball synthesis + + # relocate-me.sh). Strict = same remote hash (byte-identical, publishable); + # relaxed = any version in the one-release overlay. defaults-*, --build-local + # and development packages are never grafted. + if (getattr(args, "reuseOverlay", None) and not spec["is_devel_pkg"] + and not spec["package"].startswith("defaults-")): + _bl_raw = getattr(args, "buildLocal", None) or [] + if isinstance(_bl_raw, str): + _bl_raw = _bl_raw.split(",") + _bl = set(x for x in _bl_raw if x) + _relaxed = getattr(args, "reusePolicy", "strict") == "relaxed" + _want = None if _relaxed else spec.get("remote_revision_hash") + # In strict mode a missing hash must NOT fall through to match-any. + if spec["package"] not in _bl and (_relaxed or _want): + from bits_helpers.cvmfs_import import overlay_reuse_module + _mid = overlay_reuse_module(args.reuseOverlay, spec["package"], want_hash=_want) + if _mid: + # Adopt a consistent identity for the manifest, then skip the build. + spec["reuse_module_id"] = _mid + _verrev = _mid.split("/", 1)[1] + spec["revision"] = (_verrev[len(spec["version"]) + 1:] + if _verrev.startswith(spec["version"] + "-") else _verrev) + spec["hash"] = spec.get("remote_revision_hash") or spec.get("hash", "") + spec["cachedTarball"] = "" + spec.setdefault("deps_hash", "") + info("Reuse: %s from CVMFS overlay as module %s (not built)", p, _mid) + if getattr(args, "manifest", None) is not None: + args.manifest.add_package(spec, "already_installed", + effective_architecture=effective_arch(spec, args.architecture)) + continue + # Warn if a package declares architecture: shared but has arch-specific # deps — the shared label would be misleading in that case because its # hash (and therefore install path) will differ across platforms. @@ -3434,7 +3522,10 @@ def _build_row(pkg): # readlink() succeeds even for dangling symlinks, so we must check # existence explicitly. if not os.path.isfile(symlink_path): - warning("Ignoring dangling symlink in tarball directory: %s", symlink_path) + # Benign and self-healing: a leftover from a failed build or a cleanup + # that removed the store tarball. The scan skips it and the build + # rebuilds, so this is diagnostic noise, not actionable — keep it debug. + debug("Ignoring dangling symlink in tarball directory: %s", symlink_path) continue realPath = readlink(symlink_path) # The revision group is optional ((?:-((?:local)?[0-9]+))?) to handle @@ -3835,6 +3926,10 @@ def _build_row(pkg): ver_rev(spec)) init_workDir = container_workDir if args.docker else args.workDir + # Reused deps are set up by sourcing their deployed init.sh from the CVMFS + # Packages base (an absolute /cvmfs path, identical on host and in the + # container once /cvmfs is mounted). + _reuse_cvmfs_base = getattr(args, "reuseCvmfsBase", None) makedirs(scriptDir, exist_ok=True) # Remember where the resource monitor will write this package's trace so we # can aggregate build stats once the run finishes (P3). @@ -3848,10 +3943,12 @@ def _build_row(pkg): "provenance": create_provenance_info(spec["package"], specs, args), "initdotsh_deps": generate_initdotsh(p, specs, args.architecture, workDir=init_workDir, post_build=False, from_modules=getattr(args, "initdotshFromModules", False), - cmake_prefix_env=_cmake_prefix_env), + cmake_prefix_env=_cmake_prefix_env, + reuse_cvmfs_base=_reuse_cvmfs_base), "initdotsh_full": generate_initdotsh(p, specs, args.architecture, workDir=init_workDir, post_build=True, from_modules=getattr(args, "initdotshFromModules", False), - cmake_prefix_env=_cmake_prefix_env), + cmake_prefix_env=_cmake_prefix_env, + reuse_cvmfs_base=_reuse_cvmfs_base), "develPrefix": develPrefix, "workDir": workDir, "configDir": abspath(args.configDir), @@ -4043,10 +4140,14 @@ def _build_row(pkg): "-v {workdir}:{container_workDir} {roSources}-v{configDir}:/pkgdist.bits:ro " "-v {scriptDir}/build.sh:/build.sh:ro " "-v {bits_dir}:/bits " + "{cvmfsMount}" "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} " "-e HOME=/tmp -e SHELL=/bin/bash -e WORK_DIR_OVERRIDE={container_workDir} -e BITS_CONFIG_DIR_OVERRIDE=/pkgdist.bits {extraArgs} {image} bash -ex /build.sh" ).format( jobLabel=("--label bits-job=%s " % quote(_job_id)) if _job_id else "", + # Mount /cvmfs read-only when reusing deployed components, so a reused + # dep's init.sh (and its files under /cvmfs) resolve inside the container. + cvmfsMount=("-v /cvmfs:/cvmfs:ro " if getattr(args, "reuseCvmfsBase", None) else ""), platformArg="--platform %s " % quote(_docker_platform) if _docker_platform else "", roSources=_ro_sources, image=quote(args.dockerImage), diff --git a/bits_helpers/cvmfs_import.py b/bits_helpers/cvmfs_import.py index efc7512b..58fb90f9 100644 --- a/bits_helpers/cvmfs_import.py +++ b/bits_helpers/cvmfs_import.py @@ -230,6 +230,173 @@ def _shell_id(name): return "".join(c if (c.isalnum() or c == "_") else "_" for c in name) +def rewrite_module_anchor(text, install_base): + """Re-anchor a bits-built modulefile so it no longer needs the shared BASEDIR. + + A deployed bits modulefile derives its install root from ``$::env(BASEDIR)`` + (set by the BASE module), so the first BASE on MODULEPATH pins the location + for every package. When such a modulefile is reused next to a local build + that would misplace it. Replace the ``BASEDIR`` env reference with the + absolute *install_base* so the copy is self-anchoring and order-independent; + everything else (guards, deps, $version) is preserved verbatim. + """ + base = (install_base or "").rstrip("/") + return (text.replace("$::env(BASEDIR)", base) + .replace("$env(BASEDIR)", base)) + + +def _read_meta(path): + """Load a JSON ``.meta.json`` defensively; None on any read/parse error.""" + import json + try: + with open(path) as fh: + return json.load(fh) + except (OSError, ValueError): + return None + + +def _is_base_id(token): + """True if *token* names the BASE module (BASE or BASE/), quotes/braces + tolerated — matched as a whole id so BASECAMP/BASELINE are NOT BASE.""" + t = token.strip("'\"{}") + return t == "BASE" or t.startswith("BASE/") + + +def _line_loads_base(line): + """True if *line* has a ``load`` / ``prereq`` / ``is-loaded`` directive whose + next token is the BASE module id — matched by token so ``BASECAMP`` etc. are + not mis-matched.""" + _directives = ("load", "prereq", "prereq-all", "depends-on", "is-loaded") + toks = line.split() + return any(tok in _directives and i + 1 < len(toks) and _is_base_id(toks[i + 1]) + for i, tok in enumerate(toks)) + + +def strip_base_dep(text): + """Drop the ``BASE`` module dependency from a re-anchored modulefile. + + A deployed bits modulefile loads ``BASE`` only to obtain ``BASEDIR``; once + ``rewrite_module_anchor`` has inlined that as an absolute path, ``BASE`` is + unneeded (and may be absent from the reuse set). Real deps (``CMake``, + ``Python``…) are kept. Handles both the one-line guard and the multi-line + ``if ![ is-loaded 'BASE/1.0' ] {`` / ``module load BASE/1.0`` / ``}`` block: + when the BASE line opens an unbalanced ``{``, consume through its matching + ``}`` so no orphan brace is left behind. + """ + out = [] + lines = text.splitlines() + i, n = 0, len(lines) + while i < n: + line = lines[i] + if _line_loads_base(line): + depth = line.count("{") - line.count("}") + i += 1 + while depth > 0 and i < n: + depth += lines[i].count("{") - lines[i].count("}") + i += 1 + continue + out.append(line) + i += 1 + return "\n".join(out) + ("\n" if text.endswith("\n") else "") + + +def _module_load_deps(text): + """Ordered, de-duplicated ``module load `` targets in *text*, excluding BASE.""" + deps = [] + for line in text.splitlines(): + toks = line.split() + for i, tok in enumerate(toks): + if tok == "load" and i and toks[i - 1] == "module" and i + 1 < len(toks): + dep = toks[i + 1].strip("'\"{}") + if dep and dep != "BASE" and not dep.startswith("BASE/") and dep not in deps: + deps.append(dep) + return deps + + +def harvest_trusted(module_root, install_base): + """Harvest a bits-built deployment into a re-anchored corpus. + + *module_root* is the deployed modulefiles tree (``/`` files, e.g. + ``.../Modules/modulefiles``); *install_base* is the absolute ``Packages`` root + the modulefiles' ``BASEDIR`` should resolve to. For each modulefile: re-anchor + it to *install_base*, strip its ``BASE`` dep, and read the co-located package + ``.meta.json`` (``///.meta.json``) for the content + hash / build_id. Returns ``(corpus, package_hashes, build_id)``; build_id is + the deployment's recorded one (all packages share it), or "" if none carried it. + """ + import json + import os + corpus, hashes, build_id = {}, {}, "" + if not (module_root and os.path.isdir(module_root)): + return corpus, hashes, build_id + for pkg in sorted(os.listdir(module_root)): + pkg_dir = os.path.join(module_root, pkg) + if not os.path.isdir(pkg_dir): + continue + for verrev in sorted(os.listdir(pkg_dir)): + mfile = os.path.join(pkg_dir, verrev) + if not os.path.isfile(mfile): + continue + with open(mfile) as fh: + rendered = strip_base_dep(rewrite_module_anchor(fh.read(), install_base)) + module_id = "%s/%s" % (pkg, verrev) + meta = _read_meta(os.path.join(install_base, pkg, verrev, ".meta.json")) or {} + pkg_info = meta.get("package") if isinstance(meta.get("package"), dict) else {} + hashes[module_id] = pkg_info.get("hash", "") + build_id = build_id or meta.get("build_id", "") + corpus[module_id] = { + "version": pkg_info.get("version"), + "revision": pkg_info.get("revision"), + "deps": _module_load_deps(rendered), + "rendered": rendered, + "base_prefix": install_base, + } + return corpus, hashes, build_id + + +def import_trusted_release(module_root, install_base, arch, out_root, label="reuse", + force=False): + """Import a trusted bits deployment: harvest → build_id → write overlay. + + Uses the deployment's recorded build_id when present, else a corpus-derived + one. Returns ``{"build_id", "written", "dangling"}`` (same shape as + ``import_release``). + """ + import os + corpus, hashes, dep_build_id = harvest_trusted(module_root, install_base) + dangling = closure_check(corpus) + if dangling and not force: + return {"build_id": None, "written": [], "dangling": dangling, + "overlay_path": None} + build_id = dep_build_id or compute_corpus_build_id(corpus, label) + written = write_overlay(corpus, build_id, arch, out_root, package_hashes=hashes) + # Return the overlay path so callers need not reconstruct //. + return {"build_id": build_id, "written": written, "dangling": dangling, + "overlay_path": os.path.join(out_root, build_id, arch)} + + +def overlay_reuse_module(overlay_path, package, want_hash=None): + """Return ``/`` if the overlay satisfies *package*, else None. + + The overlay is ``//`` (modulefile) alongside a + hidden ``..meta.json``. Strict (``want_hash`` given): match a module + whose recorded hash equals it — a byte-identical, publishable reuse. Relaxed + (``want_hash`` None): any module for the package (the overlay is one coherent + release). Defensive: a missing overlay/package yields None. + """ + import os + pkg_dir = os.path.join(overlay_path or "", package) + if not (overlay_path and os.path.isdir(pkg_dir)): + return None + for name in sorted(os.listdir(pkg_dir)): + if name.startswith("."): # skip the hidden ..meta.json + continue + meta = _read_meta(os.path.join(pkg_dir, ".%s.meta.json" % name)) or {} + if want_hash is None or meta.get("hash") == want_hash: + return "%s/%s" % (package, name) + return None + + def build_module_meta(module_id, entry, build_id, package_hash="", abi_tag=""): """Module-side ``.meta.json`` payload for a corpus *entry* (D6 overlay). @@ -471,8 +638,12 @@ def write_overlay(corpus, build_id, arch, out_root, alias=None, continue dest = os.path.join(arch_root, name) os.makedirs(dest, exist_ok=True) + # A trusted-harvest entry carries the deployment's own modulefile, + # already re-anchored (entry["rendered"]); a foreign one is regenerated + # from its parsed ops. One writer, two sources. with open(os.path.join(dest, vfile), "w") as fh: - fh.write(generate_modulefile(bits_id, remapped, build_id)) + fh.write(entry.get("rendered") + or generate_modulefile(bits_id, remapped, build_id)) meta = build_module_meta(bits_id, entry, build_id, package_hash=package_hashes.get(module_id, ""), abi_tag=abi_tag) diff --git a/bits_helpers/cvmfs_import_cmd.py b/bits_helpers/cvmfs_import_cmd.py index d88c3a0f..b5664bff 100644 --- a/bits_helpers/cvmfs_import_cmd.py +++ b/bits_helpers/cvmfs_import_cmd.py @@ -56,6 +56,26 @@ def doImport(args, parser): manifest = getattr(args, "importManifest", None) modulepath = getattr(args, "importModulepath", None) + # Trusted mode: harvest a bits-built deployment directly (no modulecmd) and + # re-anchor its own modulefiles to the absolute Packages root. + if getattr(args, "importTrusted", False): + from bits_helpers.cvmfs_import import import_trusted_release + install_base = getattr(args, "importInstallBase", None) + if not (modulepath and install_base): + error("import --trusted needs --modulepath and " + "--install-base ") + return False + result = import_trusted_release(modulepath, install_base, arch, out_root, + label=label, force=force) + if result["build_id"] is None: + error("import: release is not closed; missing deps: %s", + ", ".join(result["dangling"])) + return False + info("import: build_id %s", result["build_id"]) + info("import: wrote %d module(s) under %s", len(result["written"]), + os.path.join(out_root, result["build_id"], arch)) + return True + if manifest: try: with open(manifest) as fh: diff --git a/bits_helpers/cvmfs_inspect.py b/bits_helpers/cvmfs_inspect.py new file mode 100644 index 00000000..67239425 --- /dev/null +++ b/bits_helpers/cvmfs_inspect.py @@ -0,0 +1,321 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Read-only inspection of a deployed CVMFS bits tree: discover platforms, mark +host compatibility, and present a package's build/provenance from .meta.json +without jq. Prototype — runs standalone as:: + + python3 -m bits_helpers.cvmfs_inspect platforms --cvmfs + python3 -m bits_helpers.cvmfs_inspect show Davix --cvmfs --arch --deps + python3 -m bits_helpers.cvmfs_inspect summary --cvmfs --arch + +`` is the directory holding the per-arch trees, i.e. it contains +``/Packages///.meta.json``. +""" + +import argparse +import json +import os +import sys + +try: + from bits_helpers.utilities import ( + detectArchComponents, arch_machine_token, arch_distro_token) +except Exception: # standalone / partial import fallback + detectArchComponents = None + arch_machine_token = arch_distro_token = lambda s: None + +# Compatibility marks (state -> glyph); ASCII-safe fallbacks are easy to add. +_MARK = {"native": "✓", "container": "⚠", "incompatible": "✗"} + + +# ── discovery ──────────────────────────────────────────────────────────────── + +def list_platforms(cvmfs_root): + """ dirs under *cvmfs_root* that actually hold a Packages/ tree.""" + out = [] + try: + for name in sorted(os.listdir(cvmfs_root)): + if os.path.isdir(os.path.join(cvmfs_root, name, "Packages")): + out.append(name) + except OSError: + pass + return out + + +def _packages_root(cvmfs_root, arch): + return os.path.join(cvmfs_root, arch, "Packages") + + +def list_packages(cvmfs_root, arch): + """{pkg: [verrev, ...]} under /Packages (sorted).""" + root = _packages_root(cvmfs_root, arch) + out = {} + try: + for pkg in sorted(os.listdir(root)): + pdir = os.path.join(root, pkg) + if not os.path.isdir(pdir): + continue + out[pkg] = sorted(v for v in os.listdir(pdir) + if os.path.isdir(os.path.join(pdir, v))) + except OSError: + pass + return out + + +def resolve_verrev(cvmfs_root, arch, pkg, ver=None): + """Pick a verrev: prefix-match *ver* if given, else the newest present.""" + vers = list_packages(cvmfs_root, arch).get(pkg, []) + if not vers: + return None + if ver: + for v in vers: + if v == ver or v.startswith(ver + "-"): + return v + return None + return vers[-1] + + +def read_meta(cvmfs_root, arch, pkg, verrev): + with open(os.path.join(_packages_root(cvmfs_root, arch), pkg, verrev, + ".meta.json")) as fh: + return json.load(fh) + + +def _legacy_meta(arch, pkg, verrev): + """Minimal record inferred from the path for a pre-bits tree with no + .meta.json. Best-effort guess from directory names, marked as legacy; only + name, architecture, and the verrev dir (kept in `version`) are real — + everything else is unknown/blank.""" + return { + "architecture": arch, "abi_tag": "", "build_id": "", + "provenance": "legacy (no .meta.json)", "reuse_policy": "", + "defaults": [], "bits_version": "", "dist": {}, + "package": {"name": pkg, "version": verrev, "revision": "", "hash": ""}, + "dependencies": {"direct": {"build": [], "runtime": []}, + "recursive": {"build": [], "runtime": []}}, + "_legacy": True, + } + + +def meta_or_legacy(cvmfs_root, arch, pkg, verrev): + """Read .meta.json; when it is simply absent, synthesize a legacy record from + the path. `lexists` so only a truly-missing file is legacy: a present-but- + unreadable file, or a dangling symlink, still raises (open fails) and is + surfaced as an error rather than masked as legacy.""" + path = os.path.join(_packages_root(cvmfs_root, arch), pkg, verrev, ".meta.json") + if not os.path.lexists(path): + return _legacy_meta(arch, pkg, verrev) + with open(path) as fh: + return json.load(fh) + + +# ── host compatibility (advisory, three-state) ─────────────────────────────── + +def _norm_machine(tok): + return (tok or "").replace("-", "_") + + +def classify_platform(arch, host=None): + """(state, note) with state in {native, container, incompatible}. + + Honest and layered: machine must match (else incompatible); a differing distro + token means the binaries were built for another OS/toolchain and want a + matching container (the gcc15-in-alma9 lesson), not the bare host. + """ + host = host or (detectArchComponents() if detectArchComponents else {}) + h_machine = _norm_machine(host.get("_machine") or host.get("machine")) + h_os = host.get("os") or "" + p_machine = _norm_machine(arch_machine_token(arch)) + p_os = arch_distro_token(arch) or "" + if p_machine and h_machine and p_machine != h_machine: + return "incompatible", "needs %s (host is %s)" % (p_machine, h_machine) + if p_os and h_os and p_os != h_os: + return "container", ("built for %s; host is %s — run in a matching image" + % (p_os, h_os)) + return "native", "machine/OS match host" + + +# ── presentation ───────────────────────────────────────────────────────────── + +def _verrev(pkg): + """version-revision, dropping the dash when there is no revision (legacy).""" + ver, rev = pkg.get("version", "?"), pkg.get("revision", "") + return "%s-%s" % (ver, rev) if rev else ver + + +def format_meta(meta, deps=False, provenance_only=False): + pkg = meta.get("package", {}) or {} + L = ["%s %s" % (pkg.get("name", "?"), _verrev(pkg))] + L.append(" hash: %s" % pkg.get("hash", "")) + L.append(" architecture: %s" % meta.get("architecture", "")) + L.append(" abi_tag: %s" % meta.get("abi_tag", "")) + L.append(" build_id: %s" % meta.get("build_id", "")) + L.append(" provenance: %s" % meta.get("provenance", "")) + L.append(" reuse_policy: %s" % meta.get("reuse_policy", "")) + L.append(" defaults: %s" % " ".join(meta.get("defaults", []) or [])) + L.append(" bits: %s dist %s" + % (meta.get("bits_version", ""), + ((meta.get("dist") or {}).get("commit", "") or "")[:12])) + if provenance_only: + return "\n".join(L) + if deps: + d = meta.get("dependencies", {}) or {} + for scope in ("direct", "recursive"): + sd = d.get(scope, {}) or {} + for kind in ("build", "runtime"): + items = sd.get(kind, []) or [] + if items: + L.append(" %s %s (%d):" % (scope, kind, len(items))) + for it in items: + L.append(" - %s %s-%s" % (it.get("name"), + it.get("version"), it.get("revision"))) + return "\n".join(L) + + +# ── summary ────────────────────────────────────────────────────────────────── + +def summarize(cvmfs_root, arch): + """(pkgs, build_ids): pkgs={pkg:[verrev]}, build_ids={bid:[pkg/verrev]}.""" + pkgs = list_packages(cvmfs_root, arch) + build_ids = {} + for pkg, vers in pkgs.items(): + for v in vers: + try: + m = read_meta(cvmfs_root, arch, pkg, v) + except (OSError, ValueError): + continue + build_ids.setdefault(m.get("build_id") or "(none)", []).append( + "%s/%s" % (pkg, v)) + return pkgs, build_ids + + +# ── CLI ────────────────────────────────────────────────────────────────────── + +def _cmd_platforms(a): + plats = list_platforms(a.cvmfs) + if a.json: + print(json.dumps([{"arch": p, "state": classify_platform(p)[0]} + for p in plats], indent=2)) + return 0 + if not plats: + print("no platforms found under %s" % a.cvmfs); return 1 + for p in plats: + state, note = classify_platform(p) + print(" %s %-32s %s" % (_MARK.get(state, "?"), p, note)) + print("\n %s native %s needs container %s incompatible" + % (_MARK["native"], _MARK["container"], _MARK["incompatible"])) + return 0 + + +def _show_one(a, arch, pkg, ver, verrev=None): + """Full build/provenance detail for one platform.""" + verrev = verrev or resolve_verrev(a.cvmfs, arch, pkg, ver) + if not verrev: + print("%s%s not found under %s/%s/Packages" + % (pkg, "/" + ver if ver else "", a.cvmfs, arch)); return 1 + try: + meta = meta_or_legacy(a.cvmfs, arch, pkg, verrev) + except (OSError, ValueError) as exc: + print("cannot read %s/%s/%s/.meta.json: %s" % (arch, pkg, verrev, exc)) + return 1 + if a.json: + print(json.dumps(meta, indent=2)); return 0 + print(format_meta(meta, deps=a.deps, provenance_only=a.provenance)) + if meta.get("_legacy"): + print(" (legacy tree: no .meta.json — fields inferred from the path)") + return 0 + + +def _show_across(a, pkg, hits): + """One compact line per platform that has *pkg*; --arch drills into detail.""" + rows = [] + for arch, verrev in hits: + try: + m = meta_or_legacy(a.cvmfs, arch, pkg, verrev) + except (OSError, ValueError): + # Present but unreadable/corrupt — keep it visible, don't drop it. + rows.append((arch, verrev, "", "", "unreadable .meta.json")) + continue + p = m.get("package", {}) or {} + rows.append((arch, _verrev(p), + m.get("build_id", "") or "", (p.get("hash", "") or "")[:8], + m.get("provenance", "") or "")) + if a.json: + print(json.dumps([{"arch": r[0], "verrev": r[1], "build_id": r[2], + "hash": r[3], "provenance": r[4]} for r in rows], indent=2)) + return 0 + print("%s — %d platform%s" % (pkg, len(rows), "" if len(rows) == 1 else "s")) + for arch, verrev, bid, h, prov in rows: + print(" %-30s %-10s %-38s %-8s %s" % (arch, verrev, bid, h, prov)) + print(" (add --arch for full build/provenance + --deps)") + return 0 + + +def _cmd_show(a): + pkg, _, ver = a.package.partition("/") + ver = ver or None + if a.arch: + return _show_one(a, a.arch, pkg, ver) + # No --arch: don't silently pick the first platform — show the package on + # every platform that has it, so provenance across the tree is visible. + hits = [] + for arch in list_platforms(a.cvmfs): + verrev = resolve_verrev(a.cvmfs, arch, pkg, ver) + if verrev: + hits.append((arch, verrev)) + if not hits: + print("%s%s not found on any platform under %s" + % (pkg, "/" + ver if ver else "", a.cvmfs)); return 1 + # JSON keeps one stable shape for no-arch — always the per-platform list, + # regardless of hit count. Text shows full detail when there's a single hit. + if not a.json and len(hits) == 1: + return _show_one(a, hits[0][0], pkg, ver, hits[0][1]) + return _show_across(a, pkg, hits) + + +def _cmd_summary(a): + arch = a.arch or (list_platforms(a.cvmfs) or [None])[0] + if not arch: + print("no arch (pass --arch)"); return 1 + pkgs, build_ids = summarize(a.cvmfs, arch) + if a.json: + print(json.dumps({"arch": arch, "packages": len(pkgs), + "build_ids": {b: len(v) for b, v in build_ids.items()}}, + indent=2)); return 0 + print("Platform %s: %d packages, %d build_id(s)" + % (arch, len(pkgs), len(build_ids))) + for bid, members in sorted(build_ids.items()): + print(" %s (%d packages)" % (bid, len(members))) + return 0 + + +def main(argv=None): + # Shared options live on a parent parser so they work AFTER the subcommand + # too (`bits cvmfs platforms --cvmfs ROOT`), not only before it. + common = argparse.ArgumentParser(add_help=False) + common.add_argument("--cvmfs", required=True, + help="root holding /Packages///.meta.json") + common.add_argument("--json", action="store_true", help="machine-readable output") + + ap = argparse.ArgumentParser(prog="bits cvmfs", + description="Inspect a deployed CVMFS bits tree.") + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("platforms", parents=[common], + help="list platforms + host compatibility") + sp = sub.add_parser("show", parents=[common], + help="print a package's build/provenance") + sp.add_argument("package", help="PKG or PKG/VERSION") + sp.add_argument("--arch", help="platform (default: all platforms that have it)") + sp.add_argument("--deps", action="store_true", help="include dependency tree") + sp.add_argument("--provenance", action="store_true", help="provenance fields only") + ss = sub.add_parser("summary", parents=[common], + help="per-platform package + build_id summary") + ss.add_argument("--arch", help="platform (default: first found)") + a = ap.parse_args(argv) + return {"platforms": _cmd_platforms, "show": _cmd_show, + "summary": _cmd_summary}[a.cmd](a) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bits_helpers/cvmfs_layout.py b/bits_helpers/cvmfs_layout.py index 4110bf56..2b7b5a95 100644 --- a/bits_helpers/cvmfs_layout.py +++ b/bits_helpers/cvmfs_layout.py @@ -42,6 +42,70 @@ def _render(template, subst): return _VAR_RE.sub(lambda m: str(subst.get(m.group(1), m.group(0))), template) +def split_reuse_policy(reuse_from): + """Split an optional trailing ``::`` off a ``--reuse-from`` value. + + Sugar so ``--reuse-from cvmfs::relaxed`` (or ``::relaxed``) can set the + reuse policy alongside the source. Returns ``(source, policy)`` where policy + is ``"strict"``/``"relaxed"`` or ``None`` when no valid suffix is present. + Absolute paths never contain ``::``, so splitting on the last ``::`` is + unambiguous. Pure; the caller reconciles it with an explicit --reuse-policy. + """ + if not reuse_from: + return reuse_from, None + head, sep, tail = reuse_from.rpartition("::") + if sep and tail.strip().lower() in ("strict", "relaxed"): + return head, tail.strip().lower() + return reuse_from, None + + +def resolve_reuse_from(reuse_from, layout): + """Resolve ``--reuse-from`` to an absolute modules-tree path, or None. + + ``None``/``""`` → None (no module reuse). The literal ``"cvmfs"`` → + ``layout["module_path"]`` (raises ValueError if there is no layout / + module_path). Any other value must be an absolute path (raises otherwise). + Pure and side-effect free so the caller decides how to report the error. + """ + if not reuse_from: + return None + if reuse_from == "cvmfs": + module_path = layout.get("module_path") if layout else None + if not module_path: + raise ValueError( + "--reuse-from cvmfs needs the modules location in the defaults " + "system: section — either a layout (module_dir/cvmfs_dir) or a " + "cvmfs_modules_template; none is configured.") + return module_path + if not os.path.isabs(reuse_from): + raise ValueError( + "--reuse-from must be an absolute path or the literal 'cvmfs' " + "(got %r)." % reuse_from) + return reuse_from + + +def reuse_module_path_from_templates(defaults_meta, architecture, injected_prefix=None): + """Derive the modulefiles BASE dir from the group's ``cvmfs_modules_template``. + + Lets ``--reuse-from cvmfs`` work off the single publish template a group + already declares (which also drives publishing), instead of duplicating it as + ``cvmfs_dir`` + ``module_dir``. Expands ``{prefix}`` and ``{platform}`` and + strips the trailing per-package leaf (``…/{pkg}``), yielding the base under + which per-package modulefiles live. ``architecture`` MUST be the DEPLOYED arch + (the raw ``-a`` value / ``abi_tag``), not the build-qualified family, so the + path matches where the packages actually live. Returns None when no modules + template (or no prefix) is configured. Pure. + """ + templates = resolve_cvmfs_templates(defaults_meta, injected_prefix) + if not templates or not templates.get("modules"): + return None + # Drop the trailing "/{token}" run (the per-package leaf, e.g. /{pkg} or + # /{pkg}/{tag}) to get the fixed base the modulefiles live under. + base = re.sub(r"(?:/\{[^}]*\})+$", "", templates["modules"]) + return (base.replace("{prefix}", templates["prefix"]) + .replace("{platform}", architecture)) + + def resolve_cvmfs_layout(defaults_meta, architecture): """Return the resolved CVMFS layout dict, or None when not configured. diff --git a/bits_helpers/cvmfs_reuse.py b/bits_helpers/cvmfs_reuse.py deleted file mode 100644 index d439366f..00000000 --- a/bits_helpers/cvmfs_reuse.py +++ /dev/null @@ -1,143 +0,0 @@ -# SPDX-FileCopyrightText: 2015-2026 CERN -# SPDX-License-Identifier: GPL-3.0-or-later - -""" -Relaxed-reuse matcher (ADR-0001, Stage 1b). - -Find a package already deployed in a blessed CVMFS release that can be grafted by -(name, architecture, build_id) — without the exact content-hash match that -``strict`` reuse requires. This is the read-only lookup the dependency resolver -calls when ``--reuse-policy relaxed`` is in effect; it never mutates the store. - -The deployed layout mirrors what ``CVMFSRemoteSync`` consumes: -``//Packages///.meta.json``, where -``.meta.json`` carries the ``build_id`` written by ``create_provenance_info`` -(Stage 0). A deployment that predates Stage 0 (no ``build_id``) simply never -matches, so relaxed reuse degrades to a normal build — safe by default. -""" - -import json -import os -from glob import glob - - -def _read_meta(path): - try: - with open(path) as fh: - return json.load(fh) - except (OSError, ValueError): - return None - - -def available_build_ids(package, architecture, store_root): - """Return ``{build_id: (version_dir, mtime)}`` for *package* on the store. - - Scans ``//Packages//*/.meta.json`` and - keeps, per build_id, the version directory with the newest ``.meta.json`` - mtime (the recency proxy used for "latest" — there is no build-time field in - the metadata). Defensive: a missing store or unreadable meta yields ``{}``. - """ - out = {} - if not (package and architecture and store_root): - return out - pkg_dir = os.path.join(store_root, architecture, "Packages", package) - if not os.path.isdir(pkg_dir): - return out - for ver_dir in glob(os.path.join(pkg_dir, "*")): - if not os.path.isdir(ver_dir): - continue - meta = _read_meta(os.path.join(ver_dir, ".meta.json")) - if not isinstance(meta, dict): - continue - bid = meta.get("build_id") - if not bid: - continue - try: - mtime = os.path.getmtime(os.path.join(ver_dir, ".meta.json")) - except OSError: - mtime = 0.0 - if bid not in out or mtime > out[bid][1]: - out[bid] = (ver_dir, mtime) - return out - - -def select_build_id(packages, architecture, store_root, strategy="latest"): - """Pick a build_id for relaxed reuse across the requested *packages*. - - ``latest`` : newest build_id available for the anchor (the last - requested package — the build target). A coherent release - shares one build_id across its whole closure, so the - target's newest build_id already covers its dependencies. - ``latest-common`` : newest build_id present for EVERY requested package. - - Only the top-level requested packages are known here (the resolved - dependency closure is not available until after this selection feeds the - resolver), so "common" is across the requested targets. Returns - ``(build_id | None, coverage)`` where coverage maps build_id -> set(packages). - """ - per_pkg = {p: available_build_ids(p, architecture, store_root) for p in packages} - coverage, newest = {}, {} - for pkg, ids in per_pkg.items(): - for bid, (_vd, mtime) in ids.items(): - coverage.setdefault(bid, set()).add(pkg) - newest[bid] = max(newest.get(bid, 0.0), mtime) - if not coverage: - return None, coverage - if strategy == "latest-common": - want = len(set(packages)) - common = [b for b, pkgs in coverage.items() if len(pkgs) == want] - if not common: - return None, coverage - return max(common, key=lambda b: newest[b]), coverage - # 'latest': anchor on the build target (last requested package) - anchor_ids = per_pkg.get(packages[-1], {}) if packages else {} - if not anchor_ids: - return None, coverage - return max(anchor_ids, key=lambda b: anchor_ids[b][1]), coverage - - -def graftable_match(package, architecture, build_id, store_root): - """Return a match descriptor for *package* under *build_id*, or None. - - Scans ``//Packages///.meta.json`` - and returns the first version whose recorded ``build_id`` matches and whose - ``architecture`` (when recorded) agrees: - - {"package", "version", "path", "hash", "build_id"} - - Defensive throughout: a missing store, an unreadable or legacy ``.meta.json``, - or any mismatch yields None (→ the package is built normally). - """ - if not (package and architecture and build_id and store_root): - return None - pkg_dir = os.path.join(store_root, architecture, "Packages", package) - if not os.path.isdir(pkg_dir): - return None - # Deterministic order so repeated resolutions agree; one version per package - # is expected within a coherent release, but be explicit anyway. - for ver_dir in sorted(glob(os.path.join(pkg_dir, "*"))): - if not os.path.isdir(ver_dir): - continue - meta = _read_meta(os.path.join(ver_dir, ".meta.json")) - if not isinstance(meta, dict): - continue - if meta.get("build_id") != build_id: - continue - meta_arch = meta.get("architecture") - if meta_arch and meta_arch != architecture: - continue - pkg_info = meta.get("package") if isinstance(meta.get("package"), dict) else {} - # Take version/revision from the deployed .meta.json (authoritative), NOT - # from the directory basename — that is "-" and cannot - # be split unambiguously (versions contain dashes). The consumer's reuse - # decision matches the deployed tarball by "--…", so the - # spec's version MUST equal the deployed version for the graft to fire. - return { - "package": package, - "version": pkg_info.get("version") or os.path.basename(ver_dir.rstrip("/")), - "revision": pkg_info.get("revision"), - "path": ver_dir, - "hash": pkg_info.get("hash"), - "build_id": build_id, - } - return None diff --git a/bits_helpers/sync.py b/bits_helpers/sync.py index 7d8898f6..ccf2db47 100644 --- a/bits_helpers/sync.py +++ b/bits_helpers/sync.py @@ -259,11 +259,8 @@ def remote_from_url(read_url, write_url, architecture, work_dir, insecure=False, # different backend, so pair the read-only reader with a writer built from # write_url (see DualRemoteSync). Without this, the write store was silently # dropped below and nothing was ever uploaded. - if write_url and (read_url.startswith("cvmfs://") or read_url.startswith("http")): - if read_url.startswith("http"): - reader = HttpRemoteSync(read_url, architecture, work_dir, insecure) - else: - reader = CVMFSRemoteSync(read_url, None, architecture, work_dir) + if write_url and read_url.startswith("http"): + reader = HttpRemoteSync(read_url, architecture, work_dir, insecure) return DualRemoteSync(reader, _writer_from_url(write_url, architecture, work_dir)) if read_url.startswith("http"): @@ -273,7 +270,11 @@ def remote_from_url(read_url, write_url, architecture, work_dir, insecure=False, if read_url.startswith("b3://"): return Boto3RemoteSync(read_url, write_url, architecture, work_dir) if read_url.startswith("cvmfs://"): - return CVMFSRemoteSync(read_url, None, architecture, work_dir) + dieOnError(True, + "--remote-store cvmfs:// is no longer supported. --remote-store " + "is the tarball store (s3://, b3://, rsync:, http). To reuse " + "components already deployed on CVMFS, use " + "--reuse-from |cvmfs instead.") if read_url: return RsyncRemoteSync(read_url, write_url, architecture, work_dir) return NoRemoteSync() @@ -339,10 +340,10 @@ def _first_supporting(self, method, default, *args, **kwargs): """Call *method* on the first backend that implements it, reader before writer. Store metadata (ADR-0005 rev-index markers, content-object listings) lives in - the STORE, and a read-only reader (HttpRemoteSync, CVMFSRemoteSync) may not + the STORE, and a read-only reader (e.g. HttpRemoteSync) may not implement the lookup at all. In the common ``--remote-store https://…::rw`` - setup both sides are the very same bucket (http read URL + b3 writer); in the - cvmfs-read/s3-write setup the metadata only ever exists on the writer. + setup both sides are the very same bucket (http read URL + b3 writer); in an + http-read/s3-write setup the metadata only ever exists on the writer. Delegating to the reader alone silently returned the empty default, so the revision counter never saw the markers: it could not reuse the recorded @@ -820,113 +821,6 @@ def upload_source(self, local_path, url_checksum, filename) -> None: dieOnError(err, "Unable to upload source archive to store.") -class CVMFSRemoteSync: - """ Sync packages build directory from CVMFS or similar - FS based deployment. The tarball will be created on the fly with a single - symlink to the remote store in it, so that unpacking really - means unpacking the symlink to the wanted package. - """ - - def __init__(self, remoteStore, writeStore, architecture, workdir) -> None: - self.remoteStore = re.sub("^cvmfs://", "", remoteStore) - # We do not support uploading directly to CVMFS, for obvious - # reasons. - assert(writeStore is None) - self.writeStore = None - self.architecture = architecture - self.workdir = workdir - - def fetch_tarball(self, spec) -> None: - arch = effective_arch(spec, self.architecture) - info("Downloading tarball for %s@%s-%s, if available", spec["package"], spec["version"], spec["revision"]) - # If we already have a tarball with any equivalent hash, don't check S3. - for pkg_hash in spec["remote_hashes"] + spec["local_hashes"]: - store_path = resolve_store_path(arch, pkg_hash) - pattern = os.path.join(self.workdir, store_path, "%s-*.tar.gz" % spec["package"]) - # Use os.path.isfile() to skip dangling symlinks that glob would otherwise return. - if any(os.path.isfile(t) for t in glob.glob(pattern)): - info("Reusing existing tarball for %s@%s", spec["package"], pkg_hash) - return - info("Could not find prebuilt tarball for %s@%s-%s, will be rebuilt", - spec["package"], spec["version"], spec["revision"]) - - def fetch_symlinks(self, spec) -> None: - # When using CVMFS, we create the symlinks grass by reading the . - info("Fetching available build hashes for %s, from %s", spec["package"], self.remoteStore) - arch = effective_arch(spec, self.architecture) - links_path = resolve_links_path(arch, spec["package"]) - os.makedirs(os.path.join(self.workdir, links_path), exist_ok=True) - - cvmfs_architecture = re.sub(r"slc(\d+)_x86-64", r"el\1-x86_64", self.architecture) - err = execute(r"""\ - set -x - # Exit without error in case we do not have any package published - test -d "{remote_store}/{cvmfs_architecture}/Packages/{package}" || exit 0 - mkdir -p "{workDir}/{links_path}" - for install_path in $(find "{remote_store}/{cvmfs_architecture}/Packages/{package}" -mindepth 1 -maxdepth 1 -type d); do - full_version="${{install_path##*/}}" - tarball={package}-$full_version.{architecture}.tar.gz - pkg_hash=$(cat "${{install_path}}/.build-hash" || jq -r '.package.hash' <${{install_path}}/.meta.json) - if [ "X$pkg_hash" = X ]; then - continue - fi - # POSIX 2-char prefix: this script runs under /bin/sh (dash), where a - # bash substring expansion would give "Bad substitution". - pref=$(printf %s "$pkg_hash" | cut -c1-2) - ln -sf ../../{architecture}/store/$pref/$pkg_hash/$tarball "{workDir}/{links_path}/$tarball" - # Create the dummy tarball, if it does not exists - test -f "{workDir}/{architecture}/store/$pref/$pkg_hash/$tarball" && continue - # Build the tree the reuse-unpack expects: ///... (the - # $full_version dir is the version-revision segment). Matches PKGPATH for - # the family-less deployed layout (Packages// has no family). - pkgroot="{workDir}/INSTALLROOT/$pkg_hash/{architecture}/{package}/$full_version" - mkdir -p "$pkgroot" - find "{remote_store}/{cvmfs_architecture}/Packages/{package}/$full_version" -mindepth 1 -maxdepth 1 ! -name etc -exec ln -sf {{}} "$pkgroot/" \; - cp -fr "{remote_store}/{cvmfs_architecture}/Packages/{package}/$full_version/etc" "$pkgroot/etc" - mkdir -p "{workDir}/TARS/{architecture}/store/$pref/$pkg_hash" - tar -C "{workDir}/INSTALLROOT/$pkg_hash" -czf "{workDir}/TARS/{architecture}/store/$pref/$pkg_hash/$tarball" . - rm -rf "{workDir}/INSTALLROOT/$pkg_hash" - done - """.format( - workDir=self.workdir, - architecture=arch, - cvmfs_architecture=cvmfs_architecture, - package=spec["package"], - remote_store=self.remoteStore, - links_path=links_path, - )) - print(f"fetch_symlink: maybe something wrong? {err}") - - def upload_symlinks_and_tarball(self, spec) -> None: - dieOnError(True, "CVMFS backend does not support uploading directly") - - def upload_shell_command(self, spec): - """Return None: CVMFS backend is read-only.""" - return None - - def fetch_source(self, url_checksum, filename, dest_dir) -> bool: - """Try to fetch a source archive from the CVMFS filesystem mount. - - The CVMFS remote store is a read-only filesystem path; we attempt a - plain file copy from the mirrored SOURCES/cache subtree. - """ - remote_path = os.path.join(self.remoteStore, - _source_remote_path(url_checksum, filename)) - dest = os.path.join(dest_dir, filename) - if not os.path.exists(remote_path): - return False - os.makedirs(dest_dir, exist_ok=True) - import shutil - try: - shutil.copy2(remote_path, dest) - return True - except OSError: - return False - - def upload_source(self, local_path, url_checksum, filename) -> None: - pass # CVMFS backend does not support uploading directly - - class S3RemoteSync: """Sync package build directory from and to S3 using s3cmd. diff --git a/bits_helpers/utilities.py b/bits_helpers/utilities.py index 9d1082f1..40e5d689 100644 --- a/bits_helpers/utilities.py +++ b/bits_helpers/utilities.py @@ -1591,7 +1591,7 @@ def recipeSourceLabel(pkgdir, provider_dirs=None): def getPackageList(packages, specs, configDir, preferSystem, noSystem, architecture, disable, defaults, performPreferCheck, performRequirementCheck, performValidateDefaults, overrides, taps, log, force_rebuild=(), - provider_dirs=None, defaults_meta=None, performCvmfsMatch=None): + provider_dirs=None, defaults_meta=None): """Resolve the full set of packages required by *packages*. *provider_dirs* is an optional ``dict`` returned by @@ -1953,36 +1953,6 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, # the legacy install layout //-. spec["pkg_family"] = resolve_pkg_family(defaults_meta or {}, spec["package"]) - # Relaxed CVMFS graft (ADR-0001). performCvmfsMatch is wired by build.py only - # under --reuse-policy relaxed (None otherwise → no effect, strict unchanged). - # It returns a descriptor for a package already deployed in the blessed - # release, or None. On a match we keep the spec in `specs` so consumers still - # source its deployed init.sh, but PRUNE its subtree — the grafted package's - # own dependencies are provided from /cvmfs via that init.sh cascade — and - # tag it (from_cvmfs/cvmfs_*) so the build step symlinks it instead of - # compiling (Stage 1c). The deployed hash is recorded for the hashing step. - if performCvmfsMatch is not None: - _m = performCvmfsMatch(spec) - if _m: - spec["from_cvmfs"] = True - spec["cvmfs_path"] = _m.get("path") - spec["cvmfs_build_id"] = _m.get("build_id") - if _m.get("hash"): - spec["cvmfs_hash"] = _m["hash"] - # Adopt the DEPLOYED version/revision (base versions win, ADR-0001) so the - # install path and the consumer's reuse-decision regex both match the - # deployed tarball. version/revision come from the deployed .meta.json. - if _m.get("version"): - spec["version"] = _m["version"] - spec["tag"] = _m["version"] - if _m.get("revision") is not None: - spec["revision"] = _m["revision"] - spec["requires"] = [] - spec["build_requires"] = [] - spec["runtime_requires"] = [] - specs[spec["package"]] = spec - continue - specs[spec["package"]] = spec packages += spec["requires"] diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index baf64b19..fb652b36 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -107,10 +107,9 @@ bits resolves these to `install_path` / `module_path` and uses them to default: - **docker build:** `--cvmfs-prefix` ← `/`, so packages compile at their final CVMFS prefix and relocation on publish is a no-op (explicit `--cvmfs-prefix` still wins); -- **reuse:** with `--reuse-cvmfs`, `--remote-store` ← `cvmfs://`, so - already-deployed components are reused via the `CVMFSRemoteSync` store (which - matches a deployed package's recorded `.build-hash`/`.meta.json` against the - hash bits computes — reuse happens only on a hash match). +- **reuse:** `--reuse-from cvmfs` resolves the deployed modules tree from the + same layout, so already-deployed components are set up from their published + modulefiles (`--remote-store` stays the tarball store; it is never `cvmfs://`). Builds that don't set any of these fields are unaffected. @@ -143,9 +142,9 @@ wins — and each has its own root of trust: 1. **Local store on the build node** (`$WORK_DIR/TARS`, already-unpacked `INSTALLROOT`) — artifacts this node built or fetched earlier. Ultimately trusted (produced here) and cheapest, so it is consulted first. -2. **CVMFS**, if mounted (`--reuse-cvmfs` / `cvmfs://`) — the published read-only - tree. Trusted by CVMFS itself: the repository is signed at Stratum-0 and the - client verifies it against the repo key in `/etc/cvmfs/keys`. No extra +2. **CVMFS**, when reusing a deployed release (`--reuse-from`) — the published + read-only tree. Trusted by CVMFS itself: the repository is signed at Stratum-0 + and the client verifies it against the repo key in `/etc/cvmfs/keys`. No extra bits-level attestation is needed for these artifacts. 3. **Remote archive (S3/HTTP), verified against a signed manifest** — content- addressed tarballs. Integrity comes from the content hash + `tarball_sha256`; @@ -862,10 +861,9 @@ bits build [options] PACKAGE [PACKAGE ...] |--------|-------------| | `--defaults PROFILE` | Defaults profile(s); use `::` to combine (e.g. `release::myproject`). Default: `release`. | | `--flavour NAME[=VALUE]` | Set a build-wide flavour variable (repeatable, comma-separated). `NAME`→`true`, `NAME=VALUE`→`VALUE`, `!NAME`→`false`. Gates `(?NAME)` conditional requires/sources/patches and is exported into the build environment; overrides a defaults `variables:` entry of the same name. See [Flavours](#flavours). | -| `--reuse-cvmfs` | Reuse already-deployed components from the defaults `cvmfs_dir:` area: sets `--remote-store cvmfs://` when no store is given. See [CVMFS layout](#cvmfs-layout). | -| `--reuse-policy {strict,relaxed}` | How CVMFS reuse is matched. `strict` (default): reuse only on an exact content-hash match; the result is publishable. `relaxed`: also graft deployed packages of a blessed release matched by (name, architecture, `build_id`), for fast local dev on top of e.g. an LCG release — only the top of the stack is built. Relaxed artifacts are *loose-provenance* and are refused by the publish path. Falls back to the defaults `reuse_policy:` value. See [Relaxed CVMFS reuse](#relaxed-cvmfs-reuse). | -| `--reuse-base BUILD_ID` | With `--reuse-policy relaxed`, the `build_id` of the deployed release to graft from (the value `bits` records under `build_id` in each deployed package's `.meta.json`). Falls back to the defaults `reuse_base:` value. | -| `--build-local PKG[,PKG…]` | Packages to always build locally even under `--reuse-policy relaxed` (e.g. one you need patched), instead of grafting them from the base. | +| `--reuse-from PATH\|cvmfs` | Reuse deployed components via their published modulefiles at this absolute modules-tree path (distinct from `--remote-store`, which is the tarball store). The literal `cvmfs` resolves the location from the defaults `system:` layout (`module_dir`/`cvmfs_dir`) or the `cvmfs_modules_template`. A trailing `::relaxed`/`::strict` also sets the reuse policy (e.g. `cvmfs::relaxed`). See [Reusing deployed components](#relaxed-cvmfs-reuse). | +| `--reuse-policy {strict,relaxed}` | How a reused (`--reuse-from`) component is matched. `strict` (default): reuse only on an exact content-hash match; the result is publishable. `relaxed`: reuse any version present in the one-release overlay, for fast local dev on top of e.g. an LCG release — only the top of the stack is built. Relaxed builds are *loose-provenance* and are refused by the publish path. Falls back to the defaults `reuse_policy:` value. | +| `--build-local PKG[,PKG…]` | Packages to always build locally even when they could be reused (e.g. one you need patched), instead of taking them from `--reuse-from`. | | `-a ARCH`, `--architecture ARCH` | Target architecture. Default: auto-detected, or the `architecture:` template from defaults (see [§9](#9-architecture-overview)). An explicit value here overrides the template. | | `--force-unknown-architecture` | Proceed even if architecture is unrecognised. | | `-j N`, `--jobs N` | Parallel compilation jobs per package. Default: CPU count. | @@ -2696,18 +2694,22 @@ When either `--remote-store` or `--write-store` is given, bits automatically set | `http://` or `https://` | HTTP/HTTPS | ✓ | — | None (public) or TLS; use `--insecure` to skip cert check | | `s3://BUCKET/PATH` | Amazon S3 via `s3cmd` | ✓ | ✓ | `~/.s3cfg` config file | | `b3://BUCKET/PATH` | S3-compatible via `boto3` | ✓ | ✓ | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` env vars | -| `cvmfs://REPO/PATH` | CernVM File System | ✓ | — | None (read-only filesystem) | | `rsync://HOST/PATH` or `/local/path` | rsync | ✓ | ✓ | SSH keys (`~/.ssh/`) or filesystem permissions | +> `cvmfs://` is **not** a `--remote-store` backend. `--remote-store` is the +> tarball store; to reuse components already deployed on CVMFS use +> [`--reuse-from`](#relaxed-cvmfs-reuse). A `cvmfs://` `--remote-store` is +> rejected with an error pointing at `--reuse-from`. + #### Mixing a read-only remote with a separate write store -A read-only `--remote-store` (`cvmfs://` or `http(s)://`) can be paired with a writable `--write-store` of a different backend, e.g. recall pre-built packages from a CVMFS release and upload newly-built ones to S3: +A read-only `--remote-store` (`http(s)://`) can be paired with a writable `--write-store` of a different backend, e.g. recall pre-built packages from an HTTP mirror and upload newly-built ones to S3: ```bash -bits build ... --remote-store cvmfs:///cvmfs/.../bits/ --write-store b3://mybucket +bits build ... --remote-store https://mirror.example/bits/ --write-store b3://mybucket ``` -Reads (recall) go to the remote store; uploads go to the write store. **Only freshly-built packages are uploaded** — packages recalled from the read-only store keep their original provenance and are not re-published (a CVMFS-recalled package has only a synthetic tarball of symlinks into `/cvmfs`, so uploading it would publish a stub). This is *strict* reuse only; `--reuse-policy relaxed` (loose provenance) remains barred from any write store. +Reads (recall) go to the remote store; uploads go to the write store. **Only freshly-built packages are uploaded** — packages recalled from the read-only store keep their original provenance and are not re-published. #### HTTP / HTTPS @@ -2807,14 +2809,6 @@ runners; if the key must never touch shared infrastructure, register one dedicated protected runner (it only dials out — no standing server). A ready template lives in `bits-console/.gitlab/sign-manifest.yml`. -#### CernVM File System (`cvmfs://`) - -Read-only. Instead of unpacking a remote tarball, bits creates a small local tarball containing symlinks that point into the already-mounted CVMFS repository. The build environment is constructed from the CVMFS paths without copying data locally: - -```bash -bits build --remote-store cvmfs://cvmfs.example.cern.ch/sw ROOT -``` - #### rsync / local filesystem Supports both remote hosts (via SSH) and local paths. Useful for shared NFS or a build server accessible over SSH: @@ -2949,7 +2943,6 @@ SOURCES/cache/a1/a1b2c3d4.../libfoo-1.2.tar.gz | `RsyncRemoteSync` | ✓ | ✓ | Uses `rsync -vW`; skipped if `--write-store` is absent. | | `S3RemoteSync` | ✓ | ✓ | Uses `s3cmd get/put`; skipped if `--write-store` is absent. | | `Boto3RemoteSync` | ✓ | ✓ | Native boto3 API; skips upload if the key already exists. | -| `CVMFSRemoteSync` | ✓ | — | Read-only filesystem mount; upload not supported. | #### Enabling source archive caching @@ -3050,43 +3043,46 @@ Steps to investigate: ``` The next build run will re-record the current digest and warn instead of aborting. -### Relaxed CVMFS reuse - -Strict reuse (the default) only reuses a deployed package when bits recomputes -the **exact same content hash** — which guarantees reproducibility and keeps the -result publishable, but means that to build one package on top of a blessed -release (e.g. an LCG release on `/cvmfs`) you must reproduce every hashed input -of the whole stack. **Relaxed reuse** trades that for speed in local -development: it grafts deployed packages matched by **(name, architecture, -`build_id`)** instead of by hash, so only the top of the stack is built and -everything below it is symlinked from `/cvmfs`. - -`build_id` is the per-release coherence token bits records in each package's -`.meta.json` (and which is identical for every package built together). Matching -on it — together with the *combined* architecture string, which already encodes -OS, compiler and build type — means the grafted set is ABI-consistent by -construction, without re-verifying the dependency closure. - -```bash -bits build --reuse-policy relaxed \ - --reuse-base LCG_109- \ - --remote-store cvmfs:///cvmfs/sft.cern.ch/lcg/releases \ - --defaults dev4 key4hep -``` - -Grafted dependencies are logged as **Unpacking** (no recompilation); only the -requested top package is **Compiling**. Use `--build-local PKG[,PKG…]` to force -specific packages to build locally anyway (e.g. one you need patched). - -**Provenance.** Any package built on top of a relaxed graft is *loose* — its -closure includes binaries adopted by name/`build_id` rather than verified hash, -so its own hash no longer certifies reproducible inputs. bits records -`provenance: loose` in such a package's `.meta.json`, and the **publish path -refuses loose artifacts** (`--reuse-policy relaxed` with `--write-store` or -`--pipeline` is rejected). Relaxed reuse is therefore strictly a dev/iteration -accelerator; production builds use `strict`. The matcher requires a `cvmfs://` -`--remote-store` and a `--reuse-base`; without either, relaxed reuse warns and -falls back to a normal build. + +### Reusing deployed components (`--reuse-from`) + +To build on top of a release already deployed on CVMFS, point `--reuse-from` at +its published **modules tree** (or the literal `cvmfs`, which resolves the +location from the defaults `system:` layout / `cvmfs_modules_template`). Each +reused component is set up from its deployed modulefile / `init.sh` — sourced in +place from `/cvmfs`, not copied — so only the top of the stack is built and +everything below it is consumed from the deployment. `--reuse-from` is distinct +from `--remote-store`, which remains the tarball store. + +```bash +bits build --reuse-from cvmfs::relaxed \ + --docker --docker-image \ + --architecture x86_64-el9-gcc14-opt \ + --defaults lcg::release::gcc14::opt \ + --build-local xrootd xrootd +``` + +Reused components are logged as **Reuse: … (not built)**; only the requested top +package is **Compiling**. Use `--build-local PKG[,PKG…]` to force specific +packages to build locally anyway (e.g. one you need patched). + +**Policy.** `--reuse-policy strict` (default) reuses a component only on an exact +content-hash match, so the build stays reproducible and publishable. +`--reuse-policy relaxed` reuses any version present in the one-release overlay +(matched via its `build_id`) — faster for local iteration, but **loose +provenance**: the result is not reproducible from hash alone, so the **publish +path refuses it** (`--reuse-policy relaxed` with `--write-store` or `--pipeline` +is rejected). The `::relaxed`/`::strict` suffix on `--reuse-from` sets the +policy inline; an explicit `--reuse-policy` must agree with it. + +> The builder image must match the reused release's ABI (OS + compiler): reused +> binaries carry the toolchain they were built with, so run them in an image that +> provides a compatible runtime. + +> **Note.** Earlier versions reused deployed packages through a `cvmfs://` +> `--remote-store` and a `--reuse-base ` graft; that path has been +> removed in favour of `--reuse-from`. A `cvmfs://` `--remote-store` now errors +> and points here. --- diff --git a/docs/adr/0001-cvmfs-relaxed-reuse.md b/docs/adr/0001-cvmfs-relaxed-reuse.md deleted file mode 100644 index 8d0c9349..00000000 --- a/docs/adr/0001-cvmfs-relaxed-reuse.md +++ /dev/null @@ -1,355 +0,0 @@ -# ADR-0001: Relaxed CVMFS package reuse via `build_id`, and importing foreign deployments - -**Status:** Proposed -**Date:** 2026-06-10 -**Deciders:** Predrag Buncic; bits maintainers; CVMFS/SFT release managers - -## Context - -bits reuses a published CVMFS package instead of recompiling it **only when the -full content hash matches** the hash bits computes for the current recipe -closure (recipe body — comments excluded — plus the `defaults` env, the -architecture string, and every transitive dependency's hash). This exact-hash -rule is correct and is what makes a bits artifact reproducible and safe to -publish: the hash certifies the *entire* input closure, so two packages that -share a hash are byte-for-byte interchangeable. - -Two real needs are not served by exact-hash-only reuse: - -1. **Fast local development on top of a blessed stack.** A developer wants to - build and test the *top* of a stack (e.g. key4hep, or one generator) on top - of an already-published, blessed release (an LCG release on - `/cvmfs/sft.cern.ch/...`) *without recompiling the world*. The deployed base - was built by a different pipeline/host, so its hashes will not match what - this developer's bits would compute — exact-hash reuse therefore rebuilds - everything, defeating the purpose. - -2. **Adopting pre-existing, non-bits CVMFS deployments.** Established releases - (LCG) are already deployed on CVMFS but lack bits-native metadata - (`.meta.json`, `build_id`) and may even lack a usable modulefile layout. We - want bits to *consume* such a deployment as a reuse source without rebuilding - or republishing it, and without write access to the (read-only, not-ours) - package tree. - -What bits already has (verified, current code): - -- `cvmfs://` remote store (`CVMFSRemoteSync`, `sync.py`): reads a deployed - `Packages////.build-hash` and grafts the deployed files into - the store so the normal reuse path consumes them. Read-only (no `writeStore`). -- Hash-gated reuse decision (`build.py` ~2090–2360) keyed on - `remote_hashes`/`local_hashes`; builds write `.build-hash`/`.meta.json`. -- The `islink && isdir` short-circuit that would let `sw///` - point straight at `/cvmfs` (true zero-copy) exists but is not wired for reuse. -- `prefer_system` machinery (a CLI/defaults-driven "this dep is provided - externally, prune it" hook in `getPackageList`). -- Qualified architecture strings that already encode the ABI-critical axes, - e.g. `ubuntu2510_x86-64-gcc15-dbg` (OS/glibc, compiler, build type). - -Forces: we must **not** weaken the integrity of the publish path (the -content-addressed store and the cvmfs-prepub transparency log depend on -hash purity), while making the dev path fast and able to adopt foreign releases. -**Hard constraint:** the simple case — a plain `bits build` / aliBuild with a -local alidist and no CVMFS — must remain bit-for-bit unaffected (identical -hashes, no forced rebuild, no schema break). Every decision below is therefore -opt-in and default-off, all new metadata is additive and defensively read, and -nothing in the hashed path (recipes, `MODULE_OPTIONS`, `bits-recipe-tools`) -changes. See the implementation plan's "Backward compatibility" section for the -invariants and the CI regression guard. - -## Decision - -Introduce an **opt-in, policy-gated relaxed reuse mode** plus a **`build_id` -coherence token** and an **importer** for foreign deployments. Concretely: - -**D1 — `--reuse-policy strict|relaxed` (CLI flag and `reuse_policy:` defaults -variable; default `strict`).** It is *not* a recipe field, because the same -recipes serve standard builds; the relaxation is a property of *this invocation*, -not of the package. `strict` is today's behaviour (exact-hash reuse only; -artifacts are hash-pure and publish-eligible). `relaxed` enables D2–D4. - -**D2 — Relaxed match key = (name, qualified-arch, `build_id`), not the hash.** -Because lcg.bits recipes do not pin dependency versions, "version" reduces to -"whatever the blessed set deploys": in `relaxed` mode **the base's versions -win**, overriding the recipe/`defaults` version intent. A package the top layer -genuinely needs built differently (newer version, a patch the base lacks) is -forced out of the graft with `--build-local [,…]` and built locally -above the frontier. The qualified arch supplies ABI safety (compiler + build -type + OS). `build_id` supplies *closure* safety: see D3. - -**D3 — `build_id`: a per-release coherence token stamped on every package built -together.** Same `build_id` ⇒ the packages were built as one coherent set ⇒ any -subset is mutually ABI-consistent **by construction**, so the resolver does not -have to walk and verify the dependency closure. `build_id` is **content-derived -and deterministic** (a human-readable release label, e.g. `LCG_109`, plus a -hash of the manifest = the sorted set of `{name, version, path, deps}`), so -independent or repeated imports of the same release agree and the id is -verifiable. Stored in `.meta.json` (`package.build_id`) and surfaced in the -modulefile as a `module-whatis "build_id: …"` line (not a `setenv`, to avoid -leaking it into the loaded environment). A later `base_build_id` lineage field -extends this to "built *against* release X in a separate run" (see Open -Questions). - -The `build_id` **manifest doubles as the strict-reproducibility spec.** Besides -the package set, it records the inputs that determine bits' content hashes — -the recipe-repo commits (`lcg.bits`/`stacks.bits`), the `bits-recipe-tools` -version, the `defaults`, and an explicit **`abi_tag`** (compiler, libstdc++ ABI -/ `_GLIBCXX_USE_CXX11_ABI`, `CXXSTD`, glibc/OS). So one object serves both -modes: the *coherence token* for `relaxed`, and the exact "what to check out to -get matching hashes" recipe for `strict`. This is what makes the publish-grade -path ("reuse a CVMFS package whose hash matches") dependable on a fresh host -rather than best-effort, and it lets cross-`build_id` (`base_build_id`) -compatibility be **checked** against `abi_tag` instead of assumed. - -**D4 — Frontier-cut at resolution time + zero-copy graft.** Reusing the existing -`prefer_system` hook in `getPackageList`: for each `requires`, if a deployed/ -overlay package matches (name via the alias map of D7, qualified-arch, and the -selected `build_id`), **prune that node and its entire subtree** from the build -graph, wire its modulefile environment into the consumers, and zero-copy -symlink `sw///` and `MODULES//` at `/cvmfs` (wiring -the dormant `islink && isdir` short-circuit). Everything *above* the frontier is -built locally against the grafted deps. - -**D5 — Provenance qualification, and it is contagious upward.** A `relaxed` -graft taints provenance. Crucially, taint **propagates up the closure**: any -package built above a relaxed graft is itself loose, because its inputs include -unverified binaries. Provenance is therefore "pure" only if the *entire* closure -is pure (hash-verified or locally built from pure inputs). Each artifact records -its provenance (`pure` | `loose`, plus the `build_id`(s) it borrowed) in -`.meta.json`. **The publish pipeline accepts only `pure` artifacts** and checks -the whole closure, not just the directly-grafted nodes. - -**D6 — Out-of-tree module + metadata overlay.** bits learns to read package -metadata from a *module-adjacent* location, not only from inside the package -tree. Native published packages keep `.meta.json` **with the package** (the -artifact stays self-describing and signable). Foreign/imported deployments get a -separate, bits-owned overlay (generated modulefiles + module-side metadata) that -*points into* the read-only foreign tree via `BASEDIR`/symlinks. The policy -selects the source: `strict` reads the package-side `.meta.json` (the binary's -own hash); `relaxed` reads the module-side `build_id`. - -**D7 — Importer (`bits import`).** Convert a foreign -deployment into a bits-consumable overlay: - -1. **Harvest** the deployed modulefiles by *evaluating* them - (`MODULEPATH= modulecmd sh display /`) rather than parsing raw - Tcl, capturing the resolved operations into one JSON **corpus**. Fallback - when no modulefiles exist (manifest-only deployments): synthesise the env - from the manifest's path info plus the matched recipe template. -2. **Classify** each entry's ops into three buckets: env ops that fit the - generic `BitsModule` template (`bin→PATH`, `lib/lib64→LD_LIBRARY_PATH`+ - `CMAKE_PREFIX_PATH`, `python→PYTHONPATH`, `pkgconfig→PKG_CONFIG_PATH`); - **`module load`/`prereq`/`depends-on` → a structured `deps:[…]` list** - (so names are remappable and so they form the release graph edges); and any - leftover `setenv`/etc. carried **verbatim**. ~90% are pure generic+deps; the - rest keep verbatim extras. -3. **Factor the base prefix** out of every path so generation is pure - substitution over (prefix, version, `build_id`). -4. **Closure-check** the `deps` edge set — refuse to stamp one `build_id` on a - set with dangling edges — then assign the deterministic `build_id` of D3. -5. **Generate** the bits overlay modulefiles (retargeted prefix + `build_id` + - remapped deps) and the module-side metadata of D6, with **path validation** - (the generated paths must exist in the CVMFS tree) and per-package overrides. - The modulefile is made **build-sufficient** — it adds the build hooks - (`CMAKE_PREFIX_PATH`/`PKG_CONFIG_PATH`/`CPATH`/`_ROOT`, guarded on the - tree) so a grafted dep's build env comes from *loading the modulefile* (via - `bits printenv`/module), the same mechanism as bits-native deps. No separate - `init.sh` is synthesised. - -The corpus is simultaneously the import source, the bits-native manifest for the -release, the template library, and the name corpus. - -**D8 — Name-alias map is the only fuzzy human input.** lcg.bits vocabulary and -the foreign deployment's vocabulary (e.g. `ROOT` vs `root`, `-local2` vs -hash-revisions) are reconciled once via an alias table, used **only** at -reuse-resolution time to match a recipe's `requires` to an overlay module. Env, -deps, paths, and the closure check all come from the corpus — not the recipe. - -**D9 — The published module-overlay directory is its own CVMFS nested -catalog.** When the generated modulefiles are published, the overlay root -carries a `.cvmfscatalog` marker so that subtree becomes an independent nested -catalog. This makes module enumeration use bits' **existing fast catalog path** -(`cvmfs_catalog.py` / `bitsModules`: read `user.catalog_counters`, fetch the -catalog, query SQLite — one HTTP round-trip instead of a per-file FUSE walk), -which is exactly how `bits q` already lists modules quickly on CVMFS. Both the -runtime listing (`bits q`/`avail`) and the reuse resolver scanning the overlay -for available `build_id` packages then enumerate in O(1 catalog fetch) rather -than walking the tree. The catalog boundary also lets the whole module set be -fetched/cached as a unit and snapshotted independently of the (much larger) -package payload. The importer emits the marker; native module publishing should -do the same for its `MODULES//` root. - -**D10 — Module sets are per-`build_id`; package payload is deduplicated; -multiple modulefiles may reference one package under owner attestation.** Every -package in a coherent release is published with its own modulefile, **even when -its content is unchanged** and CVMFS CAS deduplicates the underlying payload — -the modulefile is a tiny per-release entry, so a full per-release module set is -essentially free. The module overlay is therefore namespaced by `build_id` -(layout `MODULES///…`, one nested catalog each per D9), which -also removes the collision when several releases share a `MODULEPATH` (two -`Boost/1.90` from different releases are distinct module *files* pointing at the -same deduplicated payload). A single CVMFS package may be referenced by several -`(version, build_id)` modulefiles — i.e. an already-published binary can be -adopted into a *new* coherent set without rebuild. Because that breaks the -"built-together ⇒ coherent" guarantee of D3 (the binary was built in a -*different* run), it is valid **only as an explicit act of the repository owner, -who attests the package works within the new set**; the attestation (who, when, -which `base_build_id`/sets) is recorded in the new `build_id` manifest and is the -provenance for that owner-asserted reuse. bits trusts the attestation; it does -not re-derive coherence for cross-set references. - -## Options Considered - -### Option A: Exact-hash reuse only (status quo) -| Dimension | Assessment | -|-----------|------------| -| Complexity | Low (already built) | -| Reproducibility | Perfect | -| Dev iteration on blessed stacks | Poor — rebuilds everything | -| Adopt foreign (LCG) releases | Not possible | - -**Pros:** Strong integrity; nothing to build. **Cons:** Fails both target use cases. - -### Option B: Loose match on (name, version) only, no `build_id` -| Dimension | Assessment | -|-----------|------------| -| Complexity | Low | -| ABI safety | **Unsafe** — can mix incompatible closures (diamond ABI) | -| Reproducibility | None | - -**Pros:** Trivial. **Cons:** Silent ABI corruption when a consumer links a -grafted dep alongside a differently-built sub-dep. Rejected. - -### Option C: Relaxed match on (name, qualified-arch, `build_id`) + frontier-cut (CHOSEN) -| Dimension | Assessment | -|-----------|------------| -| Complexity | Medium | -| ABI safety | Safe *by construction* (same `build_id` ⇒ built together) | -| Dev iteration | Fast (build only the top) | -| Adopt foreign releases | Yes, via importer overlay | -| Reproducibility | Loose (explicitly gated out of publish) | - -**Pros:** Safe, fast, adopts foreign sets, clean strict/relaxed split. -**Cons:** Loose artifacts not reproducible (by design); needs `build_id`, -importer, alias map. - -### Option D: Regenerate modulefiles from lcg.bits recipes (rejected for env) -**Pros:** Recipe knows bespoke env. **Cons:** Depends on a matching recipe; -assumes the foreign layout matches the recipe's `MODULE_OPTIONS`; re-derives env -that already exists and is validated in the deployed modulefile. Rejected in -favour of harvesting the deployed modulefile (D7); the recipe is reduced to the -name-alias role (D8). - -### Option E: Relocate `.meta.json` globally to the module side (rejected) -**Pros:** One place; convenient for the importer. **Cons:** Breaks the -self-contained, signable native artifact (binary separated from its provenance). -Rejected in favour of the role-split overlay (D6). - -### Option F: Mutate the foreign package tree to inject bits metadata (rejected) -**Cons:** The foreign tree is read-only and not ours. Rejected; the overlay (D6) -adds compatibility without touching it. - -## Trade-off Analysis - -The core trade is **reproducibility for speed/adoptability**, made safe by two -disciplines that cost almost nothing because bits already has the hooks: - -- **ABI safety is bought by the qualified arch + `build_id`, not by closure - analysis.** This is the central bet: rather than verify that a grafted - subtree is internally consistent, we *trust a token that certifies it was - built together*. That collapses the hard part (graph consistency) into a - string compare, but it means `build_id` must be assigned only to genuinely - coherent sets (hence the closure-check in D7.4 and the determinism in D3). -- **Integrity is preserved by making loose-ness contagious and publish-blocked - (D5).** The strict path is untouched; the only way a loose binary reaches the - store is a pipeline bug, which the closure-wide provenance check guards. - -The residual risk is *misplaced trust*: if a `build_id` is stamped on an -incoherent set, or the qualified arch fails to capture an ABI axis, relaxed -reuse can produce broken binaries. Both are dev-only blast radius (never -published) and both are mitigated (closure-check; arch discipline). - -## Consequences - -**Easier:** developer iteration on blessed stacks (build only the top, instant -zero-copy base); adopting existing LCG releases as reuse sources without rebuild -or republish; reasoning about reuse safety (one token, not a graph walk); -finally wiring the true zero-copy symlink path. - -**Harder / new burden:** maintaining the name-alias map; keeping the importer's -env fidelity correct (harvest + validation + per-package overrides); ensuring -`build_id` is stamped only on closed/coherent sets; two metadata locations to -keep from drifting; the publish pipeline must enforce the closure-wide -provenance check. - -**To revisit:** `base_build_id` lineage for cross-build "work together"; -optionally tying `build_id` to the cvmfs-prepub signing/transparency log so a -blessed set's integrity is verifiable before adoption; GC/lifetime of relaxed -builds (they break if the blessed release is removed). - -## Open Questions / Gaps surfaced on review - -1. **Relaxed reuse presupposes "build against modules" — and the env must be - *build*-sufficient, not just runtime.** To compile the top layer against - grafted `/cvmfs` deps, the build environment must come from the deps' - **modulefiles**, not from bits' usual `INSTALLROOT` layout. The catch: a - harvested modulefile is runtime-oriented (`PATH`, `LD_LIBRARY_PATH`), whereas - compilation needs `CMAKE_PREFIX_PATH`, include/lib dirs, `pkg-config`, - `*_ROOT`. This is exactly the build-vs-runtime env drift that is already a - recurring bug class. So (a) the "build env == runtime modulefile env" - capability is a **prerequisite**, not an optional follow-on, for D4; and - (b) the importer must **validate that each grafted module is build-sufficient** - (or augment it) — this is the first place a relaxed build (e.g. key4hep on - LCG) will fail, at compile/link time, not runtime. -2. **ABI tag beyond the arch string.** Within one `build_id` ABI is automatic. - Across `build_id`s (the `base_build_id` lineage), compatibility must compare - ABI-relevant config (compiler, libstdc++ ABI / `_GLIBCXX_USE_CXX11_ABI`, - `CXXSTD`, glibc/OS). Record an explicit `abi_tag` with each `build_id` so - cross-build compatibility is *checked*, not assumed. -3. **Importer input modes.** The original motivating case is a manifest with - **no** modulefiles; `module show` harvest is impossible there. The importer - needs the manifest-only fallback (env from manifest paths + recipe template), - same corpus schema, different env source. -4. **Per-package "force local" escape.** A developer may need a package built - the way *their* recipe specifies (a patch/feature the blessed build lacks) - while grafting everything else. Support `relaxed` with an exclude list - (`--build-local pkgA,pkgB`), CLI/defaults, not a recipe field. -5. **Frontier/release selection.** When several `build_id`s are deployed, which - base to graft from must be explicit (`--reuse-base LCG_109`), with a sane - default (newest satisfying the most deps). -6. **Discoverability.** `bits q`/`avail` could filter by `build_id` so a user - can see "what the blessed set offers." -7. **Trust.** Relaxed reuse trusts the deployment; optional hardening is to - verify the `build_id` manifest signature (cvmfs-prepub transparency log) - before adoption. - -## Rollout / Sequencing - -The design couples two independently large efforts; decouple them to de-risk. - -- **Stage 0 — `build_id` only.** Add `build_id` (+ `abi_tag` + reproducibility - inputs, D3) to `.meta.json` and a `module-whatis` line. Small, immediately - useful for provenance/discoverability, and the foundation everything keys on. -- **Stage 1 — relaxed reuse of *bits-native* releases.** `--reuse-policy`, - frontier-cut, zero-copy graft, provenance propagation, **build-against-modules** - — validated against a release bits itself published (no importer needed; the - metadata already exists). This proves the reuse machinery and surfaces the - build-sufficient-env problem on a controlled surface. -- **Stage 2 — import foreign deployments.** The importer (D7), name-alias map - (D8), per-`build_id` overlay + nested catalog (D9/D10), manifest-only - fallback. This is where LCG adoption lands, on top of proven machinery. -- **Stage 3 — hardening.** `base_build_id` lineage + `abi_tag` compatibility - checks; signed-manifest verification; `bits q --build-id` filter. - -## Action Items - -1. [ ] Accept/iterate this ADR with release managers (esp. D5 publish-guard and D10 attestation). -2. [ ] Add `build_id` (+ `abi_tag`) to `.meta.json` and a `module-whatis` line via `MakeModule`; make it deterministic (D3). -3. [ ] Add `--reuse-policy strict|relaxed` (+ `reuse_policy:` defaults) and provenance fields (`pure|loose`, borrowed `build_id`s) in `.meta.json`. -4. [ ] Implement the resolution-time frontier-cut on the `prefer_system` hook; wire the zero-copy `islink && isdir` symlink path. -5. [ ] Implement closure-wide provenance propagation + the publish-pipeline `pure`-only guard. -6. [ ] Add out-of-tree overlay metadata reading (D6). -7. [ ] Build the importer (D7): `module show` harvest → corpus (prefix-factored, classified, deps as edges) → closure-check → deterministic `build_id` → generate overlay + validation + overrides; plus the manifest-only fallback. -8. [ ] Seed the lcg.bits ↔ foreign name-alias map (D8); report unmatched packages. -8a. [ ] Emit a `.cvmfscatalog` at the published module-overlay root (D9) so enumeration uses the fast catalog path; do the same for native `MODULES//`. -9. [ ] **Prerequisite track:** "build against modules" (build env == runtime modulefile env) — required by D4. -10. [ ] Defer: `base_build_id` lineage; signed-manifest verification; `bits q --build-id` filter. diff --git a/tests/test_args.py b/tests/test_args.py index bf07f144..de6b94c0 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -296,14 +296,12 @@ def test_defaults_are_inert(self): # resolves reusePolicy to "strict"); nothing changes. a = self._parse("build --force-unknown-architecture zlib") self.assertIsNone(a["reusePolicy"]) - self.assertIsNone(a["reuseBase"]) self.assertEqual(a["buildLocal"], []) def test_relaxed_flags_parse(self): a = self._parse("build --force-unknown-architecture --reuse-policy relaxed " - "--reuse-base LCG_109 --build-local p1,p2 zlib") + "--build-local p1,p2 zlib") self.assertEqual(a["reusePolicy"], "relaxed") - self.assertEqual(a["reuseBase"], "LCG_109") self.assertEqual(a["buildLocal"], ["p1", "p2"]) def test_initdotsh_flag_tristate(self): diff --git a/tests/test_async_build.py b/tests/test_async_build.py index 122f18b3..16c22e3c 100644 --- a/tests/test_async_build.py +++ b/tests/test_async_build.py @@ -60,21 +60,6 @@ def test_returns_none(self): self.assertIsNone(sync.upload_shell_command(GOOD_SPEC)) -class CVMFSRemoteSyncUploadCmdTest(unittest.TestCase): - """CVMFSRemoteSync is read-only — upload_shell_command returns None.""" - - def test_returns_none(self): - from bits_helpers.sync import CVMFSRemoteSync - # CVMFSRemoteSync asserts writeStore is None (no write support). - sync = CVMFSRemoteSync( - remoteStore="cvmfs://repo", - writeStore=None, - architecture=ARCH, - workdir=WORKDIR, - ) - self.assertIsNone(sync.upload_shell_command(GOOD_SPEC)) - - class RsyncRemoteSyncUploadCmdTest(unittest.TestCase): """RsyncRemoteSync returns None without write store, shell cmd with one.""" diff --git a/tests/test_bits_use.py b/tests/test_bits_use.py new file mode 100644 index 00000000..0270e339 --- /dev/null +++ b/tests/test_bits_use.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Tests for bits_helpers/bits_use — the .bitscmd saved-arg-profile.""" + +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from bits_helpers import bits_use as U + + +class BitsUseTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.p = os.path.join(self.tmp, ".bitscmd") + self._cwd = os.getcwd() + + def tearDown(self): + import shutil + os.chdir(self._cwd) + shutil.rmtree(self.tmp, ignore_errors=True) + + # ── sections round-trip ────────────────────────────────────────────────── + def test_write_read_sections(self): + U.write_section("common", ["--architecture", "x86_64-el9-gcc14-opt"], self.p) + U.write_section("build", ["--docker", "--sandbox", "off"], self.p) + sec = U.read_all(self.p) + self.assertEqual(sec["common"], ["--architecture", "x86_64-el9-gcc14-opt"]) + self.assertEqual(sec["build"], ["--docker", "--sandbox", "off"]) + + def test_global_aliases_common(self): + U.write_section("global", ["-a", "X"], self.p) + self.assertEqual(U.read_all(self.p).get("common"), ["-a", "X"]) + + def test_headerless_preamble_is_common(self): + with open(self.p, "w") as fh: + fh.write("--architecture X\n[build]\n--docker\n") + sec = U.read_all(self.p) + self.assertEqual(sec["common"], ["--architecture", "X"]) + self.assertEqual(sec["build"], ["--docker"]) + + def test_quoting_roundtrips(self): + U.write_section("build", ["--env", "A=b c", "--x", "a::b"], self.p) + self.assertEqual(U.read_all(self.p)["build"], ["--env", "A=b c", "--x", "a::b"]) + + def test_clear_one_and_all(self): + U.write_section("common", ["-a", "X"], self.p) + U.write_section("build", ["--docker"], self.p) + self.assertTrue(U.clear_section("build", self.p)) + self.assertNotIn("build", U.read_all(self.p)) + self.assertIn("common", U.read_all(self.p)) + self.assertTrue(U.clear_section(None, self.p)) + self.assertFalse(os.path.exists(self.p)) + + # ── merge / rewrite ────────────────────────────────────────────────────── + def _profile(self): + U.write_section("common", ["--architecture", "A", "--defaults", "D"], self.p) + U.write_section("build", ["--docker", "--reuse-from", "cvmfs::relaxed"], self.p) + + def test_merged_build_gets_common_and_section(self): + self._profile() + self.assertEqual( + U.merged_argv("build", ["xrootd"], self.p), + ["--architecture", "A", "--defaults", "D", + "--docker", "--reuse-from", "cvmfs::relaxed", "xrootd"]) + + def test_merged_q_gets_common_only(self): + self._profile() + self.assertEqual(U.merged_argv("q", ["ROOT"], self.p), + ["--architecture", "A", "--defaults", "D", "ROOT"]) + + def test_rewrite_injects_after_action(self): + self._profile() + self.assertEqual( + U.rewrite_argv(["build", "xrootd"], self.p), + ["build", "--architecture", "A", "--defaults", "D", + "--docker", "--reuse-from", "cvmfs::relaxed", "xrootd"]) + + def test_rewrite_skips_top_flags(self): + self._profile() + out = U.rewrite_argv(["-d", "build", "xrootd"], self.p) + self.assertEqual(out[:2], ["-d", "build"]) + self.assertIn("--docker", out) + + def test_rewrite_meta_commands_opt_out(self): + # Meta commands (use/cvmfs/store/version) take a different option set and + # must NOT receive the profile, or their argparse would reject it. + self._profile() + for meta in (["use", "build", "--docker"], ["cvmfs", "platforms"], + ["store", "ls"], ["version"], + ["verify", "--from-manifest", "m.json"]): # accepts neither flag + self.assertEqual(U.rewrite_argv(list(meta), self.p), list(meta)) + + def test_rewrite_no_profile_is_noop(self): + self.assertEqual(U.rewrite_argv(["build", "x"], self.p), ["build", "x"]) + + def test_rewrite_user_arg_comes_after_injected(self): + # A user's explicit --architecture must land AFTER the injected one so + # argparse last-wins lets it override. + self._profile() + out = U.rewrite_argv(["build", "--architecture", "B", "x"], self.p) + self.assertLess(out.index("A"), out.index("B")) + + def test_rewrite_option_before_action_is_noop(self): + self._profile() + self.assertEqual(U.rewrite_argv(["--weird", "x"], self.p), + ["--weird", "x"]) + + # ── --rewrite0 wire contract (wrapper reads with `while read -d ''`) ────── + def test_rewrite0_nul_terminated_last_token_survives(self): + import io, contextlib + self._profile() + os.chdir(self.tmp) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + U.main(["--rewrite0", "--", "build", "xrootd"]) + s = buf.getvalue() + self.assertTrue(s.endswith("\0")) # trailing NUL present + # `while read -d ''` keeps only fully-terminated fields; splitting on NUL + # and dropping the trailing empty must reproduce every token, incl. last. + toks = s.split("\0")[:-1] + self.assertEqual(toks[0], "build") + self.assertEqual(toks[-1], "xrootd") + + def test_malformed_profile_ignored_not_crash(self): + with open(self.p, "w") as fh: + fh.write('[build]\n--foo "unbalanced\n') # bad quoting + self.assertEqual(U.read_all(self.p), {}) # fail safe, no raise + self.assertEqual(U.rewrite_argv(["build", "x"], self.p), ["build", "x"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_build.py b/tests/test_build.py index 84f0521c..38e2010d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -652,6 +652,78 @@ def test_initdotsh_from_modules_shared_dependency(self) -> None: self.assertIn('export LIBSHARED_INCLUDE_DIR="${LIBSHARED_ROOT}/include"', shared) self.assertNotIn('$BITS_ARCH_PREFIX"/libshared', shared) + def test_initdotsh_reuse_modules(self) -> None: + """A dependency satisfied from a reused CVMFS release is set up via its + overlay modulefile (module use/load); a locally-built dependency keeps its + init.sh sourcing — so a plain/legacy package can consume a module-reused + dep. Dormant (byte-identical) when no reuse_modulepath is passed.""" + base = {"revision": "1", "hash": "h", "commit_hash": "c"} + specs = { + "App": dict(base, package="App", version="1.0", + requires=["CMake", "Boost"]), + "CMake": dict(base, package="CMake", version="3.30.6", requires=[]), + "Boost": dict(base, package="Boost", version="1.90.0", requires=[]), + } + # Baseline: no reuse base -> both deps sourced from the local tree. + baseline = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False) + self.assertNotIn("/cvmfs", baseline) + self.assertIn('"$WORK_DIR/$BITS_ARCH_PREFIX"/CMake/3.30.6-1/etc/profile.d/init.sh', + baseline) + + # Mark CMake reused-from-CVMFS; Boost stays locally built. + specs["CMake"]["reuse_module_id"] = "CMake/3.30.6-1" + out = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False, + reuse_cvmfs_base="/cvmfs/x/Packages") + # CMake sourced from its DEPLOYED init.sh on CVMFS, under a WORK_DIR + # override. BITS_ARCH_PREFIX="." (non-null) survives the deployed init.sh's + # `:=` default, which "" would not. + self.assertIn('WORK_DIR="/cvmfs/x/Packages"; BITS_ARCH_PREFIX="."', out) + self.assertIn('. "/cvmfs/x/Packages/CMake/3.30.6-1/etc/profile.d/init.sh"', out) + # ...and NOT from the local tree; Boost (built) still is. + self.assertNotIn('"$WORK_DIR/$BITS_ARCH_PREFIX"/CMake/3.30.6-1/etc/profile.d/init.sh', + out) + self.assertIn('"$WORK_DIR/$BITS_ARCH_PREFIX"/Boost/1.90.0-1/etc/profile.d/init.sh', + out) + + # A reused dep's pkg-config .pc is re-staged with a corrected prefix (a + # deployed .pc can bake a wrong prefix=). The staging loop covers the + # REUSED dep (CMake) via its _ROOT, not the locally-built Boost. + self.assertIn('reuse-pkgconfig', out) + self.assertIn('"${CMAKE_ROOT:-}"', out) + self.assertNotIn('"${BOOST_ROOT:-}"', out) + self.assertIn('export PKG_CONFIG_PATH="$_bits_rpc', out) + + # Dormant safety: marker set but no reuse base -> identical to baseline. + dormant = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False) + self.assertEqual(dormant, baseline) + self.assertNotIn('reuse-pkgconfig', baseline) + + def test_initdotsh_reuse_orders_prereq_before_reused(self) -> None: + """A local prerequisite of a reused dep must be sourced BEFORE it: the + reused dep's deployed init.sh transitively sources that prereq (guarded on + its _REVISION) and would look on CVMFS, where a local-only build is absent. + Emitting in topological order sets the prereq's _REVISION first so the + guard skips the CVMFS re-source.""" + base = {"revision": "1", "hash": "h", "commit_hash": "c"} + specs = { + "App": dict(base, package="App", version="1.0", + requires=["CMake", "Tools"]), + # CMake (reused) declares Tools as a dependency, so its deployed + # init.sh sources Tools; Tools itself is built locally. + "CMake": dict(base, package="CMake", version="3.30.6", + requires=["Tools"]), + "Tools": dict(base, package="Tools", version="0.0.32", requires=[]), + } + specs["CMake"]["reuse_module_id"] = "CMake/3.30.6-1" + out = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False, + reuse_cvmfs_base="/cvmfs/x/Packages") + tools_line = '"$WORK_DIR/$BITS_ARCH_PREFIX"/Tools/0.0.32-1/etc/profile.d/init.sh' + cmake_line = '. "/cvmfs/x/Packages/CMake/3.30.6-1/etc/profile.d/init.sh"' + self.assertIn(tools_line, out) + self.assertIn(cmake_line, out) + self.assertLess(out.index(tools_line), out.index(cmake_line), + "local prerequisite Tools must be sourced before reused CMake") + if __name__ == '__main__': unittest.main() diff --git a/tests/test_cvmfs_fetch_symlinks.py b/tests/test_cvmfs_fetch_symlinks.py deleted file mode 100644 index e43c6341..00000000 --- a/tests/test_cvmfs_fetch_symlinks.py +++ /dev/null @@ -1,57 +0,0 @@ -# SPDX-FileCopyrightText: 2015-2026 CERN -# SPDX-License-Identifier: GPL-3.0-or-later - -""" -Integration test for CVMFSRemoteSync.fetch_symlinks tarball synthesis. - -The reuse-unpack in build_template.sh does `mv $TMP/$PKGHASH/$PKGPATH ...` -where PKGPATH = //, so the synthesized dummy tarball MUST -carry that version-revision directory level. Runs the real shell (needs -jq/tar/find), so it is skipped where those are absent. -""" - -import json -import os -import shutil -import tarfile -import tempfile -import unittest - -from bits_helpers.sync import CVMFSRemoteSync - -_TOOLS = all(shutil.which(t) for t in ("jq", "tar", "find")) -ARCH = "x86_64-el9-gcc15-opt" # the slc->el rewrite does not touch this name -HASH = "26681160ad2eec00361f7df05dd5a94f6cbdf9e6" - - -@unittest.skipUnless(_TOOLS, "needs jq/tar/find") -class FetchSymlinksLayoutTest(unittest.TestCase): - - def setUp(self): - self.root = tempfile.mkdtemp() - self.work = tempfile.mkdtemp() - pkg = os.path.join(self.root, ARCH, "Packages", "CMake", "3.30.6-1") - os.makedirs(os.path.join(pkg, "bin")) - os.makedirs(os.path.join(pkg, "etc")) - open(os.path.join(pkg, "bin", "cmake"), "w").close() - with open(os.path.join(pkg, ".meta.json"), "w") as fh: - json.dump({"package": {"hash": HASH, "version": "3.30.6", - "revision": "1"}}, fh) - - def test_tarball_has_version_revision_level(self): - sync = CVMFSRemoteSync("cvmfs://" + self.root, None, ARCH, self.work) - sync.fetch_symlinks({"package": "CMake", "version": "3.30.6", - "revision": "1", "remote_hashes": [HASH], - "local_hashes": []}) - tb = os.path.join(self.work, "TARS", ARCH, "store", HASH[:2], HASH, - "CMake-3.30.6-1.%s.tar.gz" % ARCH) - self.assertTrue(os.path.isfile(tb), "synthesized tarball missing") - names = set(tarfile.open(tb).getnames()) - # PKGPATH tail that build_template.sh's `mv` will look for. - verrev = "./%s/CMake/3.30.6-1" % ARCH - self.assertIn(verrev, names) - self.assertIn(verrev + "/bin", names) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_cvmfs_import.py b/tests/test_cvmfs_import.py index 5062924a..f53d243c 100644 --- a/tests/test_cvmfs_import.py +++ b/tests/test_cvmfs_import.py @@ -13,8 +13,197 @@ closure_check, compute_corpus_build_id, generate_modulefile, build_module_meta, AliasMap, corpus_from_manifest, _infer_base_prefix, write_overlay, - import_release, + import_release, rewrite_module_anchor, + strip_base_dep, harvest_trusted, import_trusted_release, overlay_reuse_module, ) +import json as _json +import os as _os +import tempfile as _tempfile +import unittest as _unittest + + +# A deployed bits modulefile anchored on $::env(BASEDIR) (as pasted from CVMFS). +_DEPLOYED_BOOST = """\ +#%Module1.0 +set version 1.90.0-1 +if ![ is-loaded 'BASE/1.0' ] { module load BASE/1.0 } +set PKG_ROOT $::env(BASEDIR)/Boost/$version +if {[file isdirectory $PKG_ROOT/lib]} { prepend-path LD_LIBRARY_PATH $PKG_ROOT/lib } +setenv BOOST_ROOT $PKG_ROOT +""" + + +class RewriteModuleAnchorTest(_unittest.TestCase): + + def test_basedir_replaced_with_absolute_base(self): + out = rewrite_module_anchor(_DEPLOYED_BOOST, "/cvmfs/r/x86_64-el9-gcc15-opt/Packages/") + # BASEDIR reference gone; PKG_ROOT now absolute; trailing slash trimmed. + self.assertNotIn("BASEDIR", out) + self.assertIn("set PKG_ROOT /cvmfs/r/x86_64-el9-gcc15-opt/Packages/Boost/$version", out) + # Everything else (guard, $version, BASE load, deps) preserved. + self.assertIn("module load BASE/1.0", out) + self.assertIn("file isdirectory $PKG_ROOT/lib", out) + + def test_both_env_forms(self): + self.assertEqual(rewrite_module_anchor("$env(BASEDIR)/a", "/b"), "/b/a") + self.assertEqual(rewrite_module_anchor("$::env(BASEDIR)/a", "/b"), "/b/a") + + def test_no_basedir_is_noop(self): + self.assertEqual(rewrite_module_anchor("prepend-path PATH /x/bin\n", "/b"), + "prepend-path PATH /x/bin\n") + + +def _mf(pkg, verrev, deps): + """A deployed bits modulefile as really published: a multi-line proc block, + a MULTI-LINE `if { ... }` BASE guard, single-line dep guards, BASEDIR anchor.""" + lines = ["#%%Module1.0", + "proc ModulesHelp { } {", ' puts stderr "help"', "}", + "set version %s" % verrev, "# Dependencies", + "if ![ is-loaded 'BASE/1.0' ] {", " module load BASE/1.0", "}"] + for d in deps: + lines.append('if ![ is-loaded "%s" ] { module load %s }' % (d, d)) + lines += ["set PKG_ROOT $::env(BASEDIR)/%s/$version" % pkg, + "prepend-path PATH $PKG_ROOT/bin", + "setenv %s_ROOT $PKG_ROOT" % pkg.upper()] + return "\n".join(lines) + "\n" + + +def _braces_balanced(text): + return text.count("{") == text.count("}") + + +class StripBaseDepTest(_unittest.TestCase): + + def test_strips_only_base(self): + text = _mf("Boost", "1.90.0-1", ["CMake/3.30.6-1", "Python/3.13.11-1"]) + self.assertTrue(_braces_balanced(text)) + out = strip_base_dep(text) + # The whole multi-line BASE block is gone — no orphan '}' left behind + # (regression: the deployment's guard spans `if {` / `module load` / `}`). + self.assertNotIn("module load BASE", out) + self.assertNotIn("is-loaded 'BASE", out) + self.assertTrue(_braces_balanced(out), "unbalanced braces after strip:\n" + out) + # Real deps and the proc block survive; BASEDIR ref stays (re-anchor removes it). + self.assertIn("module load CMake/3.30.6-1", out) + self.assertIn("module load Python/3.13.11-1", out) + self.assertIn("proc ModulesHelp", out) + self.assertIn("BASEDIR", out) + + def test_strips_single_line_base_guard(self): + # The one-line form must still be handled (balanced { } on one line). + text = "if ![ is-loaded 'BASE/1.0' ] { module load BASE/1.0 }\nset x 1\n" + out = strip_base_dep(text) + self.assertNotIn("BASE", out) + self.assertIn("set x 1", out) + self.assertTrue(_braces_balanced(out)) + + def test_base_prefix_package_not_stripped(self): + # A package whose id merely starts with "BASE" must be kept. + text = ('if ![ is-loaded "BASECAMP/1.0" ] { module load BASECAMP/1.0 }\n' + "if ![ is-loaded 'BASE/1.0' ] { module load BASE/1.0 }\n") + out = strip_base_dep(text) + self.assertIn("module load BASECAMP/1.0", out) + self.assertNotIn("module load BASE/1.0", out) + + +class HarvestTrustedTest(_unittest.TestCase): + + def _deploy(self, root): + arch = "x86_64-el9-gcc15-opt" + mroot = _os.path.join(root, arch, "Modules", "modulefiles") + proot = _os.path.join(root, arch, "Packages") + pkgs = {"Boost": ("1.90.0-1", ["CMake/3.30.6-1", "Python/3.13.11-1"], "hB"), + "CMake": ("3.30.6-1", [], "hC"), + "Python": ("3.13.11-1", [], "hP")} + for pkg, (verrev, deps, h) in pkgs.items(): + md = _os.path.join(mroot, pkg); _os.makedirs(md) + with open(_os.path.join(md, verrev), "w") as fh: + fh.write(_mf(pkg, verrev, deps)) + pd = _os.path.join(proot, pkg, verrev); _os.makedirs(pd) + with open(_os.path.join(pd, ".meta.json"), "w") as fh: + _json.dump({"build_id": "rel-XYZ", + "package": {"hash": h, "version": verrev.split("-")[0], + "revision": "1"}}, fh) + return mroot, proot, arch + + def test_reanchor_hash_and_build_id(self): + with _tempfile.TemporaryDirectory() as root: + mroot, proot, arch = self._deploy(root) + corpus, hashes, build_id = harvest_trusted(mroot, proot) + self.assertEqual(build_id, "rel-XYZ") + self.assertEqual(hashes["Boost/1.90.0-1"], "hB") + boost = corpus["Boost/1.90.0-1"]["rendered"] + self.assertNotIn("BASEDIR", boost) # re-anchored + self.assertNotIn("BASE/1.0", boost) # BASE dep stripped + self.assertIn("set PKG_ROOT %s/Boost/$version" % proot, boost) + self.assertEqual(corpus["Boost/1.90.0-1"]["deps"], + ["CMake/3.30.6-1", "Python/3.13.11-1"]) + + def test_import_trusted_writes_overlay(self): + with _tempfile.TemporaryDirectory() as root, \ + _tempfile.TemporaryDirectory() as out: + mroot, proot, arch = self._deploy(root) + res = import_trusted_release(mroot, proot, arch, out) + self.assertEqual(res["build_id"], "rel-XYZ") + self.assertEqual(res["dangling"], []) # closure complete + self.assertEqual(res["overlay_path"], + _os.path.join(out, "rel-XYZ", arch)) + mf = _os.path.join(out, "rel-XYZ", arch, "Boost", "1.90.0-1") + with open(mf) as fh: + self.assertIn("%s/Boost/$version" % proot, fh.read()) + with open(_os.path.join(out, "rel-XYZ", arch, "Boost", + ".1.90.0-1.meta.json")) as fh: + self.assertEqual(_json.load(fh)["hash"], "hB") + + +class OverlayReuseModuleTest(_unittest.TestCase): + + def _overlay(self, root): + d = _os.path.join(root, "Boost"); _os.makedirs(d) + open(_os.path.join(d, "1.90.0-1"), "w").close() + with open(_os.path.join(d, ".1.90.0-1.meta.json"), "w") as fh: + _json.dump({"hash": "hB"}, fh) + return root + + def test_strict_hash_match(self): + with _tempfile.TemporaryDirectory() as ov: + self._overlay(ov) + self.assertEqual(overlay_reuse_module(ov, "Boost", want_hash="hB"), + "Boost/1.90.0-1") + self.assertIsNone(overlay_reuse_module(ov, "Boost", want_hash="other")) + + def test_relaxed_any_version(self): + with _tempfile.TemporaryDirectory() as ov: + self._overlay(ov) + self.assertEqual(overlay_reuse_module(ov, "Boost", want_hash=None), + "Boost/1.90.0-1") + + def test_missing_package_or_overlay(self): + with _tempfile.TemporaryDirectory() as ov: + self._overlay(ov) + self.assertIsNone(overlay_reuse_module(ov, "Nope", want_hash="hB")) + self.assertIsNone(overlay_reuse_module(None, "Boost", want_hash="hB")) + + +class WriteOverlayRenderedTest(_unittest.TestCase): + + def test_rendered_modulefile_written_verbatim(self): + rendered = rewrite_module_anchor(_DEPLOYED_BOOST, "/cvmfs/r/Packages") + corpus = {"Boost/1.90.0-1": {"version": "1.90.0", "revision": "1", + "deps": [], "rendered": rendered}} + with _tempfile.TemporaryDirectory() as out: + write_overlay(corpus, "bid-1", "x86_64-el9-gcc15-opt", out, + package_hashes={"Boost/1.90.0-1": "abc123"}) + mf = _os.path.join(out, "bid-1", "x86_64-el9-gcc15-opt", "Boost", "1.90.0-1") + with open(mf) as fh: + text = fh.read() + # The deployment's own (re-anchored) modulefile, not a regenerated one. + self.assertEqual(text, rendered) + meta_path = _os.path.join(_os.path.dirname(mf), ".1.90.0-1.meta.json") + with open(meta_path) as fh: + meta = _json.load(fh) + self.assertEqual(meta["hash"], "abc123") + self.assertEqual(meta["build_id"], "bid-1") PREFIX = "/cvmfs/x/ROOT/6.38.00" diff --git a/tests/test_cvmfs_import_cmd.py b/tests/test_cvmfs_import_cmd.py index cac4af7b..f7c783f7 100644 --- a/tests/test_cvmfs_import_cmd.py +++ b/tests/test_cvmfs_import_cmd.py @@ -79,6 +79,34 @@ def test_open_release_forced(self): self.assertTrue(doImport( _args(out, importManifest=mf, importForce=True), None)) + def test_trusted_harvest_writes_overlay(self): + with tempfile.TemporaryDirectory() as dep, tempfile.TemporaryDirectory() as out: + arch = "x86_64-el9-gcc13" + mroot = os.path.join(dep, "Modules", "modulefiles") + proot = os.path.join(dep, "Packages") + for pkg, ver, deps, h in [("ROOT", "6.38.00-1", [], "hR")]: + os.makedirs(os.path.join(mroot, pkg)) + with open(os.path.join(mroot, pkg, ver), "w") as fh: + fh.write("#%%Module1.0\nset PKG_ROOT $::env(BASEDIR)/%s/%s\n" + "prepend-path PATH $PKG_ROOT/bin\n" % (pkg, ver)) + os.makedirs(os.path.join(proot, pkg, ver)) + with open(os.path.join(proot, pkg, ver, ".meta.json"), "w") as fh: + json.dump({"build_id": "rel-1", + "package": {"hash": h, "version": "6.38.00", + "revision": "1"}}, fh) + ok = doImport(_args(out, architecture=arch, importTrusted=True, + importModulepath=mroot, importInstallBase=proot), None) + self.assertTrue(ok) + mf = os.path.join(out, "rel-1", arch, "ROOT", "6.38.00-1") + with open(mf) as fh: + text = fh.read() + self.assertIn("%s/ROOT/6.38.00-1" % proot, text) # re-anchored absolute + self.assertNotIn("BASEDIR", text) + + def test_trusted_needs_modulepath_and_install_base(self): + with tempfile.TemporaryDirectory() as out: + self.assertFalse(doImport(_args(out, importTrusted=True), None)) + def test_no_source_errors(self): with tempfile.TemporaryDirectory() as out: self.assertFalse(doImport(_args(out), None)) diff --git a/tests/test_cvmfs_inspect.py b/tests/test_cvmfs_inspect.py new file mode 100644 index 00000000..5ff1e35b --- /dev/null +++ b/tests/test_cvmfs_inspect.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Tests for bits_helpers/cvmfs_inspect — read-only CVMFS tree inspection.""" + +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from bits_helpers import cvmfs_inspect as I + + +def _meta(arch, name, ver, rev, build_id="bid-1", provenance="pure", deps=None): + return { + "architecture": arch, "abi_tag": arch, "build_id": build_id, + "defaults": ["lcg", "release"], "reuse_policy": "strict", + "provenance": provenance, "bits_version": "0.1", "dist": {"commit": "c" * 40}, + "package": {"name": name, "version": ver, "revision": rev, "hash": "h" + name}, + "dependencies": deps or {"direct": {"build": [], "runtime": []}, + "recursive": {"build": [], "runtime": []}}, + } + + +class CvmfsInspectTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self._pkg("x86_64-el9-gcc14-opt", "Davix", "0.8.10-1", _meta( + "x86_64-el9-gcc14-opt", "Davix", "0.8.10", "1", + deps={"direct": {"build": [{"name": "bits-recipe-tools", "version": "0.0.32", "revision": "1"}], + "runtime": [{"name": "Boost", "version": "1.90.0", "revision": "1"}]}, + "recursive": {"build": [], "runtime": []}})) + self._pkg("x86_64-el9-gcc14-opt", "Boost", "1.90.0-1", + _meta("x86_64-el9-gcc14-opt", "Boost", "1.90.0", "1")) + self._pkg("aarch64-el9-gcc14-opt", "Davix", "0.8.10-1", + _meta("aarch64-el9-gcc14-opt", "Davix", "0.8.10", "1", build_id="bid-arm")) + + def tearDown(self): + import shutil + shutil.rmtree(self.root, ignore_errors=True) + + def _pkg(self, arch, name, verrev, meta): + d = os.path.join(self.root, arch, "Packages", name, verrev) + os.makedirs(d) + with open(os.path.join(d, ".meta.json"), "w") as fh: + json.dump(meta, fh) + + def test_list_platforms(self): + self.assertEqual(I.list_platforms(self.root), + ["aarch64-el9-gcc14-opt", "x86_64-el9-gcc14-opt"]) + + def test_list_packages_and_resolve(self): + pk = I.list_packages(self.root, "x86_64-el9-gcc14-opt") + self.assertEqual(sorted(pk), ["Boost", "Davix"]) + self.assertEqual(I.resolve_verrev(self.root, "x86_64-el9-gcc14-opt", "Davix"), + "0.8.10-1") + self.assertEqual( + I.resolve_verrev(self.root, "x86_64-el9-gcc14-opt", "Davix", "0.8.10"), + "0.8.10-1") + self.assertIsNone(I.resolve_verrev(self.root, "x86_64-el9-gcc14-opt", "Nope")) + + def test_classify_three_states(self): + host = {"os": "el9", "_machine": "x86_64", "machine": "x86_64"} + self.assertEqual(I.classify_platform("x86_64-el9-gcc14-opt", host)[0], "native") + # right machine, wrong OS -> needs a container + self.assertEqual(I.classify_platform("x86_64-el8-gcc14-opt", host)[0], "container") + # wrong machine -> incompatible + self.assertEqual(I.classify_platform("aarch64-el9-gcc14-opt", host)[0], "incompatible") + + def test_format_meta_deps(self): + m = I.read_meta(self.root, "x86_64-el9-gcc14-opt", "Davix", "0.8.10-1") + out = I.format_meta(m, deps=True) + self.assertIn("Davix 0.8.10-1", out) + self.assertIn("build_id:", out) + self.assertIn("provenance: pure", out) + self.assertIn("bits-recipe-tools 0.0.32-1", out) # dep tree present + # provenance_only drops the dep tree + self.assertNotIn("bits-recipe-tools", I.format_meta(m, provenance_only=True)) + + def test_summarize_groups_by_build_id(self): + pkgs, bids = I.summarize(self.root, "x86_64-el9-gcc14-opt") + self.assertEqual(sorted(pkgs), ["Boost", "Davix"]) + self.assertEqual(sorted(bids), ["bid-1"]) + self.assertEqual(len(bids["bid-1"]), 2) + + # ── `show` platform selection ──────────────────────────────────────────── + class _Args: + def __init__(self, root, package, arch=None, deps=False, + provenance=False, json=False): + self.cvmfs, self.package, self.arch = root, package, arch + self.deps, self.provenance, self.json = deps, provenance, json + + def test_show_no_arch_lists_every_platform(self): + # Davix exists on both platforms -> compact multi-platform view (JSON). + import io, contextlib + a = self._Args(self.root, "Davix", json=True) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + got = json.loads(buf.getvalue()) + self.assertEqual(sorted(r["arch"] for r in got), + ["aarch64-el9-gcc14-opt", "x86_64-el9-gcc14-opt"]) + self.assertTrue(all(r["verrev"] == "0.8.10-1" for r in got)) + + def test_show_no_arch_single_hit_is_full_detail(self): + # Boost is on one platform only -> falls through to full detail. + import io, contextlib + a = self._Args(self.root, "Boost") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + out = buf.getvalue() + self.assertIn("Boost 1.90.0-1", out) + self.assertIn("build_id:", out) # detail block, not a table row + + def test_show_with_arch_is_full_detail(self): + import io, contextlib + a = self._Args(self.root, "Davix", arch="aarch64-el9-gcc14-opt") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + self.assertIn("bid-arm", buf.getvalue()) + + def test_show_not_found_anywhere(self): + import io, contextlib + a = self._Args(self.root, "Nope") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 1) + self.assertIn("not found on any platform", buf.getvalue()) + + # ── legacy (pre-bits) trees without .meta.json ─────────────────────────── + def _legacy_pkg(self, arch, name, verrev): + # a package dir with NO .meta.json, like /cvmfs/alice.cern.ch/... + os.makedirs(os.path.join(self.root, arch, "Packages", name, verrev)) + + def test_meta_or_legacy_absent_is_synthesized(self): + self._legacy_pkg("el5-x86_64", "xrootd", "v3.3.3") + m = I.meta_or_legacy(self.root, "el5-x86_64", "xrootd", "v3.3.3") + self.assertTrue(m.get("_legacy")) + self.assertEqual(m["package"]["name"], "xrootd") + self.assertEqual(m["package"]["version"], "v3.3.3") + self.assertEqual(m["architecture"], "el5-x86_64") + + def test_meta_or_legacy_corrupt_still_raises(self): + d = os.path.join(self.root, "el5-x86_64", "Packages", "bad", "v1") + os.makedirs(d) + with open(os.path.join(d, ".meta.json"), "w") as fh: + fh.write("{ not json") + with self.assertRaises(ValueError): + I.meta_or_legacy(self.root, "el5-x86_64", "bad", "v1") + + def test_show_no_arch_json_is_list_even_for_single_hit(self): + # Stable JSON shape: no-arch --json always returns a per-platform list, + # so a caller sees the same shape whether 1 or N platforms match. + import io, contextlib + a = self._Args(self.root, "Boost", json=True) # Boost: single platform + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + got = json.loads(buf.getvalue()) + self.assertIsInstance(got, list) + self.assertEqual(len(got), 1) + self.assertEqual(got[0]["arch"], "x86_64-el9-gcc14-opt") + + def test_show_across_keeps_unreadable_platform_visible(self): + # A corrupt .meta.json on one platform must not silently vanish. + d = os.path.join(self.root, "aarch64-el9-gcc14-opt", "Packages", + "Boost", "1.90.0-1") + os.makedirs(d) + with open(os.path.join(d, ".meta.json"), "w") as fh: + fh.write("{ broken") + import io, contextlib + a = self._Args(self.root, "Boost", json=True) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + got = {r["arch"]: r for r in json.loads(buf.getvalue())} + self.assertEqual(len(got), 2) # both platforms present + self.assertEqual(got["aarch64-el9-gcc14-opt"]["provenance"], + "unreadable .meta.json") + + def test_meta_or_legacy_dangling_symlink_raises(self): + d = os.path.join(self.root, "el5-x86_64", "Packages", "xrootd", "v3.3.3") + os.makedirs(d) + os.symlink("/no/such/target", os.path.join(d, ".meta.json")) + with self.assertRaises(OSError): # not treated as legacy + I.meta_or_legacy(self.root, "el5-x86_64", "xrootd", "v3.3.3") + + def test_show_legacy_detail_notes_missing_meta(self): + import io, contextlib + self._legacy_pkg("el5-x86_64", "xrootd", "v3.3.3") + a = self._Args(self.root, "xrootd", arch="el5-x86_64") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + out = buf.getvalue() + self.assertIn("xrootd v3.3.3", out) + self.assertNotIn("v3.3.3-", out) # no trailing dash when no revision + self.assertIn("legacy", out) + self.assertIn("no .meta.json", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cvmfs_layout.py b/tests/test_cvmfs_layout.py index 0ca684f2..fd69d51c 100644 --- a/tests/test_cvmfs_layout.py +++ b/tests/test_cvmfs_layout.py @@ -12,7 +12,8 @@ from bits_helpers.cvmfs_layout import resolve_cvmfs_layout as R from bits_helpers.cvmfs_layout import resolve_cvmfs_templates as RT from bits_helpers.cvmfs_layout import ( - resolve_release, path_release, bake_release, _declared_release) + resolve_release, path_release, bake_release, _declared_release, + resolve_reuse_from, split_reuse_policy, reuse_module_path_from_templates) ARCH = "ubuntu2510_x86-64-gcc15-dbg" @@ -245,5 +246,87 @@ def test_end_to_end_explicit_release(self): self.assertEqual(rel, "dev3") +class ResolveReuseFromTest(unittest.TestCase): + + def test_none_and_empty(self): + self.assertIsNone(resolve_reuse_from(None, None)) + self.assertIsNone(resolve_reuse_from("", {"module_path": "/x"})) + + def test_absolute_path_passthrough(self): + self.assertEqual(resolve_reuse_from("/cvmfs/x/modules", None), + "/cvmfs/x/modules") + + def test_relative_path_rejected(self): + with self.assertRaises(ValueError): + resolve_reuse_from("modules", None) + + def test_cvmfs_resolves_from_layout(self): + layout = R({"cvmfs_dir": "/cvmfs/r"}, ARCH) # module_path defaults to /modules + self.assertEqual(resolve_reuse_from("cvmfs", layout), + os.path.join("/cvmfs/r", ARCH, "modules")) + + def test_cvmfs_without_layout_fails(self): + with self.assertRaises(ValueError): + resolve_reuse_from("cvmfs", None) + with self.assertRaises(ValueError): + resolve_reuse_from("cvmfs", {}) # layout present but no module_path + + +class SplitReusePolicyTest(unittest.TestCase): + + def test_no_suffix(self): + self.assertEqual(split_reuse_policy("cvmfs"), ("cvmfs", None)) + self.assertEqual(split_reuse_policy("/cvmfs/x/modulefiles"), + ("/cvmfs/x/modulefiles", None)) + + def test_none_and_empty(self): + self.assertEqual(split_reuse_policy(None), (None, None)) + self.assertEqual(split_reuse_policy(""), ("", None)) + + def test_policy_suffix(self): + self.assertEqual(split_reuse_policy("cvmfs::relaxed"), ("cvmfs", "relaxed")) + self.assertEqual(split_reuse_policy("cvmfs::strict"), ("cvmfs", "strict")) + self.assertEqual(split_reuse_policy("/cvmfs/x/modulefiles::relaxed"), + ("/cvmfs/x/modulefiles", "relaxed")) + + def test_case_insensitive(self): + self.assertEqual(split_reuse_policy("cvmfs::RELAXED"), ("cvmfs", "relaxed")) + + def test_unknown_suffix_is_not_a_policy(self): + # A non-policy trailing token is left as part of the source, untouched. + self.assertEqual(split_reuse_policy("cvmfs::loose"), ("cvmfs::loose", None)) + + +class ReuseModulePathFromTemplatesTest(unittest.TestCase): + + def test_derives_base_from_declared_template(self): + meta = {"system": { + "prefix": "/cvmfs/sft.cern.ch/lcg/bits", + "cvmfs_modules_template": "{prefix}/{platform}/Modules/modulefiles/{pkg}", + }} + # Uses the DEPLOYED (raw) arch and strips the trailing /{pkg}. + self.assertEqual( + reuse_module_path_from_templates(meta, "x86_64-el9-gcc14-opt"), + "/cvmfs/sft.cern.ch/lcg/bits/x86_64-el9-gcc14-opt/Modules/modulefiles") + + def test_default_template_when_only_prefix(self): + # No explicit modules template -> resolve_cvmfs_templates supplies the + # conventional default, which we still reduce to the base. + meta = {"system": {"prefix": "/cvmfs/r"}} + self.assertEqual(reuse_module_path_from_templates(meta, "el9"), + "/cvmfs/r/el9/Modules/modulefiles") + + def test_injected_prefix_wins(self): + meta = {"system": {"cvmfs_modules_template": + "{prefix}/{platform}/Modules/modulefiles/{pkg}"}} + self.assertEqual( + reuse_module_path_from_templates(meta, "el9", injected_prefix="/cvmfs/inj"), + "/cvmfs/inj/el9/Modules/modulefiles") + + def test_none_when_no_prefix_or_template(self): + self.assertIsNone(reuse_module_path_from_templates({}, "el9")) + self.assertIsNone(reuse_module_path_from_templates(None, "el9")) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_cvmfs_reuse.py b/tests/test_cvmfs_reuse.py index 2ff475e9..9fb51a44 100644 --- a/tests/test_cvmfs_reuse.py +++ b/tests/test_cvmfs_reuse.py @@ -1,142 +1,7 @@ # SPDX-FileCopyrightText: 2015-2026 CERN # SPDX-License-Identifier: GPL-3.0-or-later -""" -Tests for bits_helpers/cvmfs_reuse.graftable_match() (ADR-0001 Stage 1b). - -The matcher is read-only and not yet wired into the resolver, so these tests -fully cover its behaviour against a faked deployed-store tree. -""" - -import json -import os -import tempfile -import unittest - -from bits_helpers.cvmfs_reuse import graftable_match, select_build_id - - -ARCH = "ubuntu2510_x86-64-gcc15-dbg" - - -def _deploy(root, pkg, version, revision="1", *, build_id=None, architecture=ARCH, - pkg_hash="h", write_meta=True): - """Create //Packages//-/ + .meta.json. - - The directory is named version-revision (as bits deploys); version/revision - are recorded authoritatively in .meta.json's package field. - """ - d = os.path.join(root, architecture, "Packages", pkg, "%s-%s" % (version, revision)) - os.makedirs(d, exist_ok=True) - if write_meta: - meta = {"architecture": architecture, - "package": {"hash": pkg_hash, "version": version, "revision": revision}} - if build_id is not None: - meta["build_id"] = build_id - with open(os.path.join(d, ".meta.json"), "w") as fh: - json.dump(meta, fh) - return d - - -class TestGraftableMatch(unittest.TestCase): - - def setUp(self): - self.root = tempfile.mkdtemp() - - def test_match_on_name_arch_build_id(self): - d = _deploy(self.root, "ROOT", "6.38.00", "1", build_id="LCG_109-abc") - m = graftable_match("ROOT", ARCH, "LCG_109-abc", self.root) - self.assertIsNotNone(m) - # version/revision come from .meta.json, not the dir basename - self.assertEqual(m["version"], "6.38.00") - self.assertEqual(m["revision"], "1") - self.assertEqual(m["path"], d) - self.assertEqual(m["hash"], "h") - self.assertEqual(m["build_id"], "LCG_109-abc") - - def test_no_match_wrong_build_id(self): - _deploy(self.root, "ROOT", "6.38.00-1", build_id="LCG_108-zzz") - self.assertIsNone(graftable_match("ROOT", ARCH, "LCG_109-abc", self.root)) - - def test_no_match_wrong_arch(self): - _deploy(self.root, "ROOT", "6.38.00-1", build_id="LCG_109-abc") - self.assertIsNone( - graftable_match("ROOT", "osx_arm64_gcc15", "LCG_109-abc", self.root)) - - def test_no_match_missing_store(self): - self.assertIsNone(graftable_match("ROOT", ARCH, "LCG_109-abc", - os.path.join(self.root, "nope"))) - - def test_legacy_deploy_without_build_id_never_matches(self): - _deploy(self.root, "ROOT", "6.38.00-1", build_id=None) # pre-Stage-0 - self.assertIsNone(graftable_match("ROOT", ARCH, "LCG_109-abc", self.root)) - - def test_no_meta_json_never_matches(self): - _deploy(self.root, "ROOT", "6.38.00-1", write_meta=False) - self.assertIsNone(graftable_match("ROOT", ARCH, "LCG_109-abc", self.root)) - - def test_picks_the_matching_version_among_several(self): - _deploy(self.root, "Boost", "1.88.0", "1", build_id="OTHER") - _deploy(self.root, "Boost", "1.90.0", "1", build_id="LCG_109-abc") - m = graftable_match("Boost", ARCH, "LCG_109-abc", self.root) - self.assertIsNotNone(m) - self.assertEqual(m["version"], "1.90.0") - self.assertEqual(m["revision"], "1") - - def test_empty_inputs_are_safe(self): - self.assertIsNone(graftable_match("", ARCH, "x", self.root)) - self.assertIsNone(graftable_match("ROOT", "", "x", self.root)) - self.assertIsNone(graftable_match("ROOT", ARCH, "", self.root)) - self.assertIsNone(graftable_match("ROOT", ARCH, "x", "")) - - -class TestSelectBuildId(unittest.TestCase): - - def setUp(self): - self.root = tempfile.mkdtemp() - - def _mtime(self, d, t): - os.utime(os.path.join(d, ".meta.json"), (t, t)) - - def test_latest_anchors_on_build_target(self): - # ROOT is the target (last requested). Newer build_id must win for it, - # regardless of what CMake carries. - old = _deploy(self.root, "ROOT", "6.0", "1", build_id="rel-old") - new = _deploy(self.root, "ROOT", "6.1", "1", build_id="rel-new") - self._mtime(old, 1000); self._mtime(new, 2000) - _deploy(self.root, "CMake", "3.30", "1", build_id="rel-old") - bid, cov = select_build_id(["CMake", "ROOT"], ARCH, self.root, "latest") - self.assertEqual(bid, "rel-new") - self.assertIn("ROOT", cov["rel-new"]) - - def test_latest_common_requires_all_packages(self): - # rel-new is newest but only ROOT has it; rel-old is shared by both. - r_old = _deploy(self.root, "ROOT", "6.0", "1", build_id="rel-old") - r_new = _deploy(self.root, "ROOT", "6.1", "1", build_id="rel-new") - c_old = _deploy(self.root, "CMake", "3.30", "1", build_id="rel-old") - self._mtime(r_old, 1000); self._mtime(r_new, 2000); self._mtime(c_old, 1000) - bid, _ = select_build_id(["CMake", "ROOT"], ARCH, self.root, "latest-common") - self.assertEqual(bid, "rel-old") - - def test_latest_common_tolerates_duplicate_packages(self): - # A duplicated target must not make the shared-by-all test unsatisfiable. - _deploy(self.root, "ROOT", "6.1", "1", build_id="rel-x") - bid, _ = select_build_id(["ROOT", "ROOT"], ARCH, self.root, "latest-common") - self.assertEqual(bid, "rel-x") - - def test_latest_common_none_when_no_shared_id(self): - _deploy(self.root, "ROOT", "6.1", "1", build_id="rel-a") - _deploy(self.root, "CMake", "3.30", "1", build_id="rel-b") - bid, _ = select_build_id(["CMake", "ROOT"], ARCH, self.root, "latest-common") - self.assertIsNone(bid) - - def test_none_when_store_empty_or_no_build_id(self): - self.assertEqual((None, {}), select_build_id(["ROOT"], ARCH, self.root, "latest")) - _deploy(self.root, "ROOT", "6.1", "1", build_id=None) # legacy: no build_id - bid, cov = select_build_id(["ROOT"], ARCH, self.root, "latest") - self.assertIsNone(bid) - self.assertEqual(cov, {}) - - -if __name__ == "__main__": - unittest.main() +# Tests for the legacy ADR-0001 graftable_match/select_build_id, removed in the +# Step 5 cleanup along with cvmfs_reuse.py. Deployed reuse is now driven by +# --reuse-from; its tests live in test_cvmfs_import.py / test_cvmfs_layout.py. +# This file is scheduled for `git rm`. diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 8fa4e964..448ddb39 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -131,19 +131,22 @@ def _record_specs(self, specs): finally: os.environ.pop("BITS_DIST_HASH", None) - def test_provenance_loose_when_closure_has_graft(self): + def test_provenance_pure_when_closure_is_clean(self): + # No untracked_requires anywhere in the closure -> pure. (The legacy + # cvmfs:// graft that also produced "loose" was removed in Step 5.) specs = { "a": _spec("a", full_runtime_requires=["dep"], full_build_requires=[]), - "dep": _spec("dep", from_cvmfs=True), + "dep": _spec("dep"), # locally built } - self.assertEqual(self._record_specs(specs)["provenance"], "loose") + self.assertEqual(self._record_specs(specs)["provenance"], "pure") - def test_provenance_pure_when_no_graft_in_closure(self): + def test_provenance_loose_when_closure_has_untracked(self): + # untracked_requires decouples a dependency from the hash -> loose. specs = { - "a": _spec("a", full_runtime_requires=["dep"], full_build_requires=[]), - "dep": _spec("dep"), # locally built, not grafted + "a": _spec("a", untracked_requires=["dep"]), + "dep": _spec("dep"), } - self.assertEqual(self._record_specs(specs)["provenance"], "pure") + self.assertEqual(self._record_specs(specs)["provenance"], "loose") class TestBuildIdFromManifest(unittest.TestCase): diff --git a/tests/test_reuse_resolver.py b/tests/test_reuse_resolver.py deleted file mode 100644 index 007a994e..00000000 --- a/tests/test_reuse_resolver.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2015-2026 CERN -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Tests for the relaxed-reuse frontier-cut in getPackageList (ADR-0001 Stage 1b). - -A `performCvmfsMatch` callback that returns a deployed-package descriptor causes -that package to be grafted: kept in `specs` (so consumers still depend on it and -source its deployed init.sh) but with its dependency subtree pruned and tagged -`from_cvmfs`. With the default callback (None) the resolver is unchanged. -""" -import os -import sys -import unittest -from unittest.mock import patch - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - -from bits_helpers.utilities import getPackageList - - -RECIPES = { - "myapp": "package: myapp\nversion: v1\nrequires:\n - mydep\n---\n", - "mydep": "package: mydep\nversion: v1\nrequires:\n - mysubdep\n---\n", - "mysubdep": "package: mysubdep\nversion: v1\n---\n", - "defaults-release": "package: defaults-release\nversion: v1\n---\n", -} - - -def _resolve(perform_cvmfs_match): - specs = {} - - def fake_resolveFilename(taps, pkg, configDir, genPkgs): - return (pkg + ".sh", "/pkgdir") - - def fake_getRecipeReader(filename, *a, **k): - pkg = filename.replace(".sh", "") - content = RECIPES.get(pkg, "package: {p}\nversion: v1\n---\n".format(p=pkg)) - return lambda: content - - with patch("bits_helpers.utilities.resolveFilename", side_effect=fake_resolveFilename), \ - patch("bits_helpers.utilities.getRecipeReader", side_effect=fake_getRecipeReader), \ - patch("bits_helpers.utilities.getGeneratedPackages", return_value={"/pkgdir": {}}), \ - patch("bits_helpers.utilities.load_for_spec", return_value=None), \ - patch("bits_helpers.utilities.merge_into_spec", return_value=None): - getPackageList( - packages=["myapp"], specs=specs, configDir="/fake", - preferSystem=False, noSystem="*", architecture="slc9_x86-64", - disable=[], defaults=["release"], - performPreferCheck=lambda pkg, cmd: (1, ""), - performRequirementCheck=lambda pkg, cmd: (0, ""), - performValidateDefaults=lambda spec: (True, "", None), - overrides={"defaults-release": {}}, taps={}, - log=lambda *a, **k: None, defaults_meta=None, - performCvmfsMatch=perform_cvmfs_match, - ) - return specs - - -class TestRelaxedFrontierCut(unittest.TestCase): - - def test_no_callback_is_unchanged(self): - specs = _resolve(None) - for p in ("myapp", "mydep", "mysubdep"): - self.assertIn(p, specs) - self.assertNotIn("from_cvmfs", specs[p]) - - def test_graft_prunes_subtree_but_keeps_consumer_dep(self): - # Graft 'mydep' from a blessed release: it stays in specs, its subtree - # ('mysubdep') is pruned, and the consumer ('myapp') still depends on it. - def match(spec): - if spec["package"] == "mydep": - return {"path": "/cvmfs/rel/mydep/v1", "build_id": "LCG_109-x", - "hash": "deadbeef", "version": "v1"} - return None - - specs = _resolve(match) - self.assertIn("mydep", specs) - self.assertTrue(specs["mydep"]["from_cvmfs"]) - self.assertEqual(specs["mydep"]["cvmfs_path"], "/cvmfs/rel/mydep/v1") - self.assertEqual(specs["mydep"]["cvmfs_hash"], "deadbeef") - self.assertEqual(specs["mydep"]["requires"], []) - # subtree pruned - self.assertNotIn("mysubdep", specs) - # consumer still present and still depends on the grafted package - self.assertIn("myapp", specs) - self.assertNotIn("from_cvmfs", specs["myapp"]) - self.assertIn("mydep", specs["myapp"]["requires"]) - - def test_no_match_builds_everything(self): - specs = _resolve(lambda spec: None) - for p in ("myapp", "mydep", "mysubdep"): - self.assertIn(p, specs) - self.assertNotIn("from_cvmfs", specs[p]) - - -class TestStoreHashesGraft(unittest.TestCase): - """A from_cvmfs spec adopts the deployed hash so the existing reuse path - symlinks it instead of building (ADR-0001 Stage 1c).""" - - def test_from_cvmfs_adopts_deployed_hash(self): - from bits_helpers.build import storeHashes - specs = { - "ROOT": { - "package": "ROOT", "version": "v1", "revision": "1", - "from_cvmfs": True, "cvmfs_hash": "deadbeef", - "requires": [], "pkg_family": "", - }, - } - storeHashes("ROOT", specs, considerRelocation=False) - s = specs["ROOT"] - self.assertEqual(s["hash"], "deadbeef") - self.assertEqual(s["remote_revision_hash"], "deadbeef") - self.assertEqual(s["local_revision_hash"], "deadbeef") - self.assertEqual(s["remote_hashes"], ["deadbeef"]) - self.assertEqual(s["local_hashes"], ["deadbeef"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_source_cache.py b/tests/test_source_cache.py index a741afe8..1bc09e88 100644 --- a/tests/test_source_cache.py +++ b/tests/test_source_cache.py @@ -7,7 +7,7 @@ * ``_source_remote_path()`` — canonical remote path helper * ``NoRemoteSync``, ``HttpRemoteSync``, ``RsyncRemoteSync``, - ``S3RemoteSync``, ``Boto3RemoteSync``, ``CVMFSRemoteSync`` — + ``S3RemoteSync``, ``Boto3RemoteSync`` — ``fetch_source()`` / ``upload_source()`` methods * ``download()`` — ``sync_helper`` integration (local-cache hit, remote-store hit, upstream download with subsequent archive upload) @@ -369,53 +369,6 @@ def test_upload_skipped_with_no_write_store(self): syncer.s3.upload_file.assert_not_called() -# --------------------------------------------------------------------------- -# CVMFSRemoteSync -# --------------------------------------------------------------------------- - -class CVMFSRemoteSyncSourceTest(unittest.TestCase): - """CVMFSRemoteSync.fetch_source reads from the filesystem mount.""" - - def _make_syncer(self, remote_path): - return sync.CVMFSRemoteSync( - remoteStore="cvmfs://{}".format(remote_path), - writeStore=None, - architecture="slc9_x86-64", - workdir="/sw", - ) - - def test_fetch_success(self): - with tempfile.TemporaryDirectory() as tmp: - # Lay out the file at the expected remote filesystem path. - remote_file = os.path.join( - tmp, _source_remote_path(TEST_URL_HASH, TEST_FILENAME), - ) - _write_fake_file(remote_file) - - syncer = self._make_syncer(tmp) - dest_dir = os.path.join(tmp, "dest") - result = syncer.fetch_source(TEST_URL_HASH, TEST_FILENAME, dest_dir) - - self.assertTrue(result) - dest_file = os.path.join(dest_dir, TEST_FILENAME) - self.assertTrue(os.path.isfile(dest_file)) - with open(dest_file, "rb") as fh: - self.assertEqual(fh.read(), _FAKE_CONTENT) - - def test_fetch_miss(self): - with tempfile.TemporaryDirectory() as tmp: - syncer = self._make_syncer(tmp) # remote dir empty - result = syncer.fetch_source(TEST_URL_HASH, TEST_FILENAME, - os.path.join(tmp, "dest")) - self.assertFalse(result) - - def test_upload_is_noop(self): - with tempfile.TemporaryDirectory() as tmp: - syncer = self._make_syncer(tmp) - # Must not raise even though CVMFS is read-only. - syncer.upload_source("/tmp/libfoo.tar.gz", TEST_URL_HASH, TEST_FILENAME) - - # --------------------------------------------------------------------------- # download() — sync_helper integration # --------------------------------------------------------------------------- diff --git a/tests/test_sync.py b/tests/test_sync.py index e006c67c..206ef3bd 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -479,32 +479,34 @@ def test_tarball_upload_spec_with_architecture_key(self) -> None: @patch("bits_helpers.sync.Boto3RemoteSync._s3_init", new=MagicMock()) class DualRemoteSyncTestCase(unittest.TestCase): - """Cross-backend stores: recall from CVMFS, upload freshly-built to S3.""" + """Cross-backend stores: recall from an HTTP mirror, upload freshly-built to S3/B3.""" - READ = "cvmfs:///cvmfs/sft-nightlies-test.cern.ch/lcg/bits/" + READ = "http://mirror.example/tars/" WRITE = "b3://bucketofpieces" # ── remote_from_url dispatch ───────────────────────────────────────────── - def test_cvmfs_read_plus_write_returns_dual(self): + def test_http_read_plus_write_returns_dual(self): helper = sync.remote_from_url(self.READ, self.WRITE, ARCHITECTURE, "/work") self.assertIsInstance(helper, sync.DualRemoteSync) - self.assertIsInstance(helper.reader, sync.CVMFSRemoteSync) + self.assertIsInstance(helper.reader, sync.HttpRemoteSync) self.assertIsInstance(helper.writer, sync.Boto3RemoteSync) # Writer targets the write store for both its read-back and its uploads. self.assertEqual(helper.writer.writeStore, "bucketofpieces") self.assertEqual(helper.writer.remoteStore, "bucketofpieces") - def test_cvmfs_read_without_write_is_unchanged(self): - # No write store -> the old read-only CVMFS helper, not a Dual. - helper = sync.remote_from_url(self.READ, "", ARCHITECTURE, "/work") - self.assertIsInstance(helper, sync.CVMFSRemoteSync) - def test_same_backend_is_unchanged(self): helper = sync.remote_from_url("b3://bucket", "b3://bucket", ARCHITECTURE, "/work") self.assertIsInstance(helper, sync.Boto3RemoteSync) + @patch("bits_helpers.sync.error", new=MagicMock()) + def test_cvmfs_read_store_is_rejected(self): + # cvmfs:// --remote-store is retired; deployed reuse is via --reuse-from. + with self.assertRaises(SystemExit): + sync.remote_from_url("cvmfs:///cvmfs/x/", "", ARCHITECTURE, "/work") + @patch("bits_helpers.sync.error", new=MagicMock()) def test_cvmfs_write_target_is_rejected(self): + # cvmfs is read-only: reject it as a --write-store (via the dual path). with self.assertRaises(SystemExit): sync.remote_from_url(self.READ, "cvmfs:///somewhere", ARCHITECTURE, "/work")