From ac9e765a16bff7450360a9914fb719142ecf3313 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 16:47:02 +0200 Subject: [PATCH 01/30] reuse: add --reuse-from (parse + resolve), not yet consumed Introduce --reuse-from |cvmfs, the module-based reuse source, kept separate from --remote-store (the tarball store). 'cvmfs' resolves module_path from the defaults system: layout, failing if unset. Pure resolver resolve_reuse_from; nothing consumes the value yet. --- bits_helpers/args.py | 6 ++++++ bits_helpers/build.py | 10 ++++++++++ bits_helpers/cvmfs_layout.py | 24 ++++++++++++++++++++++++ tests/test_cvmfs_layout.py | 29 ++++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/bits_helpers/args.py b/bits_helpers/args.py index c303fa59..48a49bcb 100644 --- a/bits_helpers/args.py +++ b/bits_helpers/args.py @@ -734,6 +734,12 @@ def doParseArgs(): "build_id for the build target) and 'latest-common' (newest build_id " "shared by all requested packages); the same happens when left empty " "under relaxed. The chosen build_id is announced.")) + build_remote.add_argument("--reuse-from", dest="reuseFrom", metavar="PATH|cvmfs", default=None, + help=("Reuse deployed components via their published modulefiles at this " + "absolute modules-tree path (distinct from --remote-store, which is the " + "tarball store). The literal 'cvmfs' resolves the exact location from the " + "defaults `system:` layout (module_dir under cvmfs_dir); fails if that is " + "not configured.")) build_remote.add_argument("--build-local", dest="buildLocal", metavar="PKG[,PKG...]", default="", help=("Comma-separated packages to always build locally even under " "--reuse-policy relaxed (e.g. a package you need patched), rather than " diff --git a/bits_helpers/build.py b/bits_helpers/build.py index feec667b..d7eeb0c6 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -2398,6 +2398,16 @@ def defaultsReader(): args.remoteStore = "cvmfs://" + _cvmfs["cvmfs_dir"] info("Reusing deployed components: --remote-store %s", args.remoteStore) + # Resolve --reuse-from into an absolute modules-tree path ('cvmfs' -> the + # defaults system: layout module_path). Nothing consumes it yet (later step). + from bits_helpers.cvmfs_layout import resolve_reuse_from + try: + args.reuseFrom = resolve_reuse_from(getattr(args, "reuseFrom", None), _cvmfs) + except ValueError as exc: + dieOnError(True, str(exc)) + if args.reuseFrom: + info("Reuse-from modules: %s", args.reuseFrom) + # Build-host policy knobs live under a single `system:` entry in defaults. # These control *how* the build runs (network, CPU) — not *what* it produces — # so, unlike `env:`, they are NOT folded into any package hash and changing diff --git a/bits_helpers/cvmfs_layout.py b/bits_helpers/cvmfs_layout.py index 4110bf56..03b24dc3 100644 --- a/bits_helpers/cvmfs_layout.py +++ b/bits_helpers/cvmfs_layout.py @@ -42,6 +42,30 @@ def _render(template, subst): return _VAR_RE.sub(lambda m: str(subst.get(m.group(1), m.group(0))), template) +def resolve_reuse_from(reuse_from, layout): + """Resolve ``--reuse-from`` to an absolute modules-tree path, or None. + + ``None``/``""`` → None (no module reuse). The literal ``"cvmfs"`` → + ``layout["module_path"]`` (raises ValueError if there is no layout / + module_path). Any other value must be an absolute path (raises otherwise). + Pure and side-effect free so the caller decides how to report the error. + """ + if not reuse_from: + return None + if reuse_from == "cvmfs": + module_path = layout.get("module_path") if layout else None + if not module_path: + raise ValueError( + "--reuse-from cvmfs needs a modules layout (module_dir/cvmfs_dir) " + "in the defaults system: section; none is configured.") + return module_path + if not os.path.isabs(reuse_from): + raise ValueError( + "--reuse-from must be an absolute path or the literal 'cvmfs' " + "(got %r)." % reuse_from) + return reuse_from + + def resolve_cvmfs_layout(defaults_meta, architecture): """Return the resolved CVMFS layout dict, or None when not configured. diff --git a/tests/test_cvmfs_layout.py b/tests/test_cvmfs_layout.py index 0ca684f2..566b4981 100644 --- a/tests/test_cvmfs_layout.py +++ b/tests/test_cvmfs_layout.py @@ -12,7 +12,8 @@ from bits_helpers.cvmfs_layout import resolve_cvmfs_layout as R from bits_helpers.cvmfs_layout import resolve_cvmfs_templates as RT from bits_helpers.cvmfs_layout import ( - resolve_release, path_release, bake_release, _declared_release) + resolve_release, path_release, bake_release, _declared_release, + resolve_reuse_from) ARCH = "ubuntu2510_x86-64-gcc15-dbg" @@ -245,5 +246,31 @@ def test_end_to_end_explicit_release(self): self.assertEqual(rel, "dev3") +class ResolveReuseFromTest(unittest.TestCase): + + def test_none_and_empty(self): + self.assertIsNone(resolve_reuse_from(None, None)) + self.assertIsNone(resolve_reuse_from("", {"module_path": "/x"})) + + def test_absolute_path_passthrough(self): + self.assertEqual(resolve_reuse_from("/cvmfs/x/modules", None), + "/cvmfs/x/modules") + + def test_relative_path_rejected(self): + with self.assertRaises(ValueError): + resolve_reuse_from("modules", None) + + def test_cvmfs_resolves_from_layout(self): + layout = R({"cvmfs_dir": "/cvmfs/r"}, ARCH) # module_path defaults to /modules + self.assertEqual(resolve_reuse_from("cvmfs", layout), + os.path.join("/cvmfs/r", ARCH, "modules")) + + def test_cvmfs_without_layout_fails(self): + with self.assertRaises(ValueError): + resolve_reuse_from("cvmfs", None) + with self.assertRaises(ValueError): + resolve_reuse_from("cvmfs", {}) # layout present but no module_path + + if __name__ == "__main__": unittest.main() From e0406a52e26022054f416c97238985bc13a38661 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 16:54:36 +0200 Subject: [PATCH 02/30] import: re-anchor helper + rendered-modulefile overlay Add rewrite_module_anchor: replace a deployed modulefile's shared $::env(BASEDIR) with an absolute install base so a reused copy is self-anchoring (the first BASE on MODULEPATH no longer pins every package's location). write_overlay now writes a pre-rendered modulefile when an entry carries one; foreign entries still regenerate from ops. Mechanism for trusted harvest; no producer wired yet. --- bits_helpers/cvmfs_import.py | 21 ++++++++++++- tests/test_cvmfs_import.py | 58 +++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/bits_helpers/cvmfs_import.py b/bits_helpers/cvmfs_import.py index efc7512b..3566d561 100644 --- a/bits_helpers/cvmfs_import.py +++ b/bits_helpers/cvmfs_import.py @@ -230,6 +230,21 @@ def _shell_id(name): return "".join(c if (c.isalnum() or c == "_") else "_" for c in name) +def rewrite_module_anchor(text, install_base): + """Re-anchor a bits-built modulefile so it no longer needs the shared BASEDIR. + + A deployed bits modulefile derives its install root from ``$::env(BASEDIR)`` + (set by the BASE module), so the first BASE on MODULEPATH pins the location + for every package. When such a modulefile is reused next to a local build + that would misplace it. Replace the ``BASEDIR`` env reference with the + absolute *install_base* so the copy is self-anchoring and order-independent; + everything else (guards, deps, $version) is preserved verbatim. + """ + base = (install_base or "").rstrip("/") + return (text.replace("$::env(BASEDIR)", base) + .replace("$env(BASEDIR)", base)) + + def build_module_meta(module_id, entry, build_id, package_hash="", abi_tag=""): """Module-side ``.meta.json`` payload for a corpus *entry* (D6 overlay). @@ -471,8 +486,12 @@ def write_overlay(corpus, build_id, arch, out_root, alias=None, continue dest = os.path.join(arch_root, name) os.makedirs(dest, exist_ok=True) + # A trusted-harvest entry carries the deployment's own modulefile, + # already re-anchored (entry["rendered"]); a foreign one is regenerated + # from its parsed ops. One writer, two sources. with open(os.path.join(dest, vfile), "w") as fh: - fh.write(generate_modulefile(bits_id, remapped, build_id)) + fh.write(entry.get("rendered") + or generate_modulefile(bits_id, remapped, build_id)) meta = build_module_meta(bits_id, entry, build_id, package_hash=package_hashes.get(module_id, ""), abi_tag=abi_tag) diff --git a/tests/test_cvmfs_import.py b/tests/test_cvmfs_import.py index 5062924a..a7774cbd 100644 --- a/tests/test_cvmfs_import.py +++ b/tests/test_cvmfs_import.py @@ -13,8 +13,64 @@ closure_check, compute_corpus_build_id, generate_modulefile, build_module_meta, AliasMap, corpus_from_manifest, _infer_base_prefix, write_overlay, - import_release, + import_release, rewrite_module_anchor, ) +import json as _json +import os as _os +import tempfile as _tempfile +import unittest as _unittest + + +# A deployed bits modulefile anchored on $::env(BASEDIR) (as pasted from CVMFS). +_DEPLOYED_BOOST = """\ +#%Module1.0 +set version 1.90.0-1 +if ![ is-loaded 'BASE/1.0' ] { module load BASE/1.0 } +set PKG_ROOT $::env(BASEDIR)/Boost/$version +if {[file isdirectory $PKG_ROOT/lib]} { prepend-path LD_LIBRARY_PATH $PKG_ROOT/lib } +setenv BOOST_ROOT $PKG_ROOT +""" + + +class RewriteModuleAnchorTest(_unittest.TestCase): + + def test_basedir_replaced_with_absolute_base(self): + out = rewrite_module_anchor(_DEPLOYED_BOOST, "/cvmfs/r/x86_64-el9-gcc15-opt/Packages/") + # BASEDIR reference gone; PKG_ROOT now absolute; trailing slash trimmed. + self.assertNotIn("BASEDIR", out) + self.assertIn("set PKG_ROOT /cvmfs/r/x86_64-el9-gcc15-opt/Packages/Boost/$version", out) + # Everything else (guard, $version, BASE load, deps) preserved. + self.assertIn("module load BASE/1.0", out) + self.assertIn("file isdirectory $PKG_ROOT/lib", out) + + def test_both_env_forms(self): + self.assertEqual(rewrite_module_anchor("$env(BASEDIR)/a", "/b"), "/b/a") + self.assertEqual(rewrite_module_anchor("$::env(BASEDIR)/a", "/b"), "/b/a") + + def test_no_basedir_is_noop(self): + self.assertEqual(rewrite_module_anchor("prepend-path PATH /x/bin\n", "/b"), + "prepend-path PATH /x/bin\n") + + +class WriteOverlayRenderedTest(_unittest.TestCase): + + def test_rendered_modulefile_written_verbatim(self): + rendered = rewrite_module_anchor(_DEPLOYED_BOOST, "/cvmfs/r/Packages") + corpus = {"Boost/1.90.0-1": {"version": "1.90.0", "revision": "1", + "deps": [], "rendered": rendered}} + with _tempfile.TemporaryDirectory() as out: + write_overlay(corpus, "bid-1", "x86_64-el9-gcc15-opt", out, + package_hashes={"Boost/1.90.0-1": "abc123"}) + mf = _os.path.join(out, "bid-1", "x86_64-el9-gcc15-opt", "Boost", "1.90.0-1") + with open(mf) as fh: + text = fh.read() + # The deployment's own (re-anchored) modulefile, not a regenerated one. + self.assertEqual(text, rendered) + meta_path = _os.path.join(_os.path.dirname(mf), ".1.90.0-1.meta.json") + with open(meta_path) as fh: + meta = _json.load(fh) + self.assertEqual(meta["hash"], "abc123") + self.assertEqual(meta["build_id"], "bid-1") PREFIX = "/cvmfs/x/ROOT/6.38.00" From 48dac95c16f5cef5e5de43b18faa2736288604be Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 17:10:37 +0200 Subject: [PATCH 03/30] import: trusted harvest of a bits-built deployment bits import --trusted reads a deployed release's own modulefiles (--modulepath) and re-anchors each to --install-base, strips the now unneeded BASE dep, and captures package hash + build_id from the install tree's .meta.json. Deterministic (no modulecmd); feeds the same overlay writer. Produces the self-anchoring modules a strict reuse consumes. --- bits_helpers/args.py | 9 +++ bits_helpers/cvmfs_import.py | 109 +++++++++++++++++++++++++++++++ bits_helpers/cvmfs_import_cmd.py | 20 ++++++ tests/test_cvmfs_import.py | 83 +++++++++++++++++++++++ tests/test_cvmfs_import_cmd.py | 28 ++++++++ 5 files changed, 249 insertions(+) diff --git a/bits_helpers/args.py b/bits_helpers/args.py index 48a49bcb..8f512051 100644 --- a/bits_helpers/args.py +++ b/bits_helpers/args.py @@ -397,6 +397,15 @@ def doParseArgs(): metavar="FILE", default=None, help="JSON manifest to import instead of harvesting (fallback when " "no modulefiles exist).") + import_parser.add_argument("--trusted", dest="importTrusted", action="store_true", + help="Trusted mode: harvest a bits-built deployment directly, reading " + "its own modulefiles (--modulepath) and re-anchoring them to " + "--install-base, capturing package hashes from the install tree. " + "Deterministic (no modulecmd); publishable strict reuse.") + import_parser.add_argument("--install-base", dest="importInstallBase", + metavar="DIR", default=None, + help="With --trusted, the absolute Packages root the modulefiles' " + "BASEDIR resolves to (and where each package's .meta.json lives).") import_parser.add_argument("--aliases", dest="importAliases", metavar="FILE", default=None, help="JSON name-alias map (foreign -> bits names).") diff --git a/bits_helpers/cvmfs_import.py b/bits_helpers/cvmfs_import.py index 3566d561..19ef5eb4 100644 --- a/bits_helpers/cvmfs_import.py +++ b/bits_helpers/cvmfs_import.py @@ -245,6 +245,115 @@ def rewrite_module_anchor(text, install_base): .replace("$env(BASEDIR)", base)) +def _read_meta(path): + """Load a JSON ``.meta.json`` defensively; None on any read/parse error.""" + import json + try: + with open(path) as fh: + return json.load(fh) + except (OSError, ValueError): + return None + + +def _is_base_id(token): + """True if *token* names the BASE module (BASE or BASE/), quotes/braces + tolerated — matched as a whole id so BASECAMP/BASELINE are NOT BASE.""" + t = token.strip("'\"{}") + return t == "BASE" or t.startswith("BASE/") + + +def strip_base_dep(text): + """Drop the ``BASE`` module dependency line(s) from a re-anchored modulefile. + + A deployed bits modulefile loads ``BASE`` only to obtain ``BASEDIR``; once + ``rewrite_module_anchor`` has inlined that as an absolute path, ``BASE`` is + unneeded (and may be absent from the reuse set). Real deps (``CMake``, + ``Python``…) are kept. A line is a BASE dependency when a ``load`` / ``prereq`` + / ``is-loaded`` directive on it targets BASE as a module id — matched by token + (like ``_module_load_deps``) so a package named ``BASE…`` is not mis-stripped. + """ + _directives = ("load", "prereq", "prereq-all", "depends-on", "is-loaded") + out = [] + for line in text.splitlines(): + toks = line.split() + if any(tok in _directives and i + 1 < len(toks) and _is_base_id(toks[i + 1]) + for i, tok in enumerate(toks)): + continue + out.append(line) + return "\n".join(out) + ("\n" if text.endswith("\n") else "") + + +def _module_load_deps(text): + """Ordered, de-duplicated ``module load `` targets in *text*, excluding BASE.""" + deps = [] + for line in text.splitlines(): + toks = line.split() + for i, tok in enumerate(toks): + if tok == "load" and i and toks[i - 1] == "module" and i + 1 < len(toks): + dep = toks[i + 1].strip("'\"{}") + if dep and dep != "BASE" and not dep.startswith("BASE/") and dep not in deps: + deps.append(dep) + return deps + + +def harvest_trusted(module_root, install_base): + """Harvest a bits-built deployment into a re-anchored corpus. + + *module_root* is the deployed modulefiles tree (``/`` files, e.g. + ``.../Modules/modulefiles``); *install_base* is the absolute ``Packages`` root + the modulefiles' ``BASEDIR`` should resolve to. For each modulefile: re-anchor + it to *install_base*, strip its ``BASE`` dep, and read the co-located package + ``.meta.json`` (``///.meta.json``) for the content + hash / build_id. Returns ``(corpus, package_hashes, build_id)``; build_id is + the deployment's recorded one (all packages share it), or "" if none carried it. + """ + import json + import os + corpus, hashes, build_id = {}, {}, "" + if not (module_root and os.path.isdir(module_root)): + return corpus, hashes, build_id + for pkg in sorted(os.listdir(module_root)): + pkg_dir = os.path.join(module_root, pkg) + if not os.path.isdir(pkg_dir): + continue + for verrev in sorted(os.listdir(pkg_dir)): + mfile = os.path.join(pkg_dir, verrev) + if not os.path.isfile(mfile): + continue + with open(mfile) as fh: + rendered = strip_base_dep(rewrite_module_anchor(fh.read(), install_base)) + module_id = "%s/%s" % (pkg, verrev) + meta = _read_meta(os.path.join(install_base, pkg, verrev, ".meta.json")) or {} + pkg_info = meta.get("package") if isinstance(meta.get("package"), dict) else {} + hashes[module_id] = pkg_info.get("hash", "") + build_id = build_id or meta.get("build_id", "") + corpus[module_id] = { + "version": pkg_info.get("version"), + "revision": pkg_info.get("revision"), + "deps": _module_load_deps(rendered), + "rendered": rendered, + "base_prefix": install_base, + } + return corpus, hashes, build_id + + +def import_trusted_release(module_root, install_base, arch, out_root, label="reuse", + force=False): + """Import a trusted bits deployment: harvest → build_id → write overlay. + + Uses the deployment's recorded build_id when present, else a corpus-derived + one. Returns ``{"build_id", "written", "dangling"}`` (same shape as + ``import_release``). + """ + corpus, hashes, dep_build_id = harvest_trusted(module_root, install_base) + dangling = closure_check(corpus) + if dangling and not force: + return {"build_id": None, "written": [], "dangling": dangling} + build_id = dep_build_id or compute_corpus_build_id(corpus, label) + written = write_overlay(corpus, build_id, arch, out_root, package_hashes=hashes) + return {"build_id": build_id, "written": written, "dangling": dangling} + + def build_module_meta(module_id, entry, build_id, package_hash="", abi_tag=""): """Module-side ``.meta.json`` payload for a corpus *entry* (D6 overlay). diff --git a/bits_helpers/cvmfs_import_cmd.py b/bits_helpers/cvmfs_import_cmd.py index d88c3a0f..b5664bff 100644 --- a/bits_helpers/cvmfs_import_cmd.py +++ b/bits_helpers/cvmfs_import_cmd.py @@ -56,6 +56,26 @@ def doImport(args, parser): manifest = getattr(args, "importManifest", None) modulepath = getattr(args, "importModulepath", None) + # Trusted mode: harvest a bits-built deployment directly (no modulecmd) and + # re-anchor its own modulefiles to the absolute Packages root. + if getattr(args, "importTrusted", False): + from bits_helpers.cvmfs_import import import_trusted_release + install_base = getattr(args, "importInstallBase", None) + if not (modulepath and install_base): + error("import --trusted needs --modulepath and " + "--install-base ") + return False + result = import_trusted_release(modulepath, install_base, arch, out_root, + label=label, force=force) + if result["build_id"] is None: + error("import: release is not closed; missing deps: %s", + ", ".join(result["dangling"])) + return False + info("import: build_id %s", result["build_id"]) + info("import: wrote %d module(s) under %s", len(result["written"]), + os.path.join(out_root, result["build_id"], arch)) + return True + if manifest: try: with open(manifest) as fh: diff --git a/tests/test_cvmfs_import.py b/tests/test_cvmfs_import.py index a7774cbd..224c0020 100644 --- a/tests/test_cvmfs_import.py +++ b/tests/test_cvmfs_import.py @@ -14,6 +14,7 @@ build_module_meta, AliasMap, corpus_from_manifest, _infer_base_prefix, write_overlay, import_release, rewrite_module_anchor, + strip_base_dep, harvest_trusted, import_trusted_release, ) import json as _json import os as _os @@ -52,6 +53,88 @@ def test_no_basedir_is_noop(self): "prepend-path PATH /x/bin\n") +def _mf(pkg, verrev, deps): + """A deployed bits modulefile (BASEDIR-anchored, loads BASE + deps).""" + lines = ["#%%Module1.0", "set version %s" % verrev, + "if ![ is-loaded 'BASE/1.0' ] { module load BASE/1.0 }"] + for d in deps: + lines.append('if ![ is-loaded "%s" ] { module load %s }' % (d, d)) + lines += ["set PKG_ROOT $::env(BASEDIR)/%s/$version" % pkg, + "prepend-path PATH $PKG_ROOT/bin", + "setenv %s_ROOT $PKG_ROOT" % pkg.upper()] + return "\n".join(lines) + "\n" + + +class StripBaseDepTest(_unittest.TestCase): + + def test_strips_only_base(self): + text = _mf("Boost", "1.90.0-1", ["CMake/3.30.6-1", "Python/3.13.11-1"]) + out = strip_base_dep(text) + # The BASE load/is-loaded line is gone; real deps and BASEDIR ref remain + # (re-anchoring, not this function, removes BASEDIR). + self.assertNotIn("module load BASE", out) + self.assertNotIn("is-loaded 'BASE", out) + self.assertIn("module load CMake/3.30.6-1", out) + self.assertIn("module load Python/3.13.11-1", out) + self.assertIn("BASEDIR", out) + + def test_base_prefix_package_not_stripped(self): + # A package whose id merely starts with "BASE" must be kept. + text = ('if ![ is-loaded "BASECAMP/1.0" ] { module load BASECAMP/1.0 }\n' + "if ![ is-loaded 'BASE/1.0' ] { module load BASE/1.0 }\n") + out = strip_base_dep(text) + self.assertIn("module load BASECAMP/1.0", out) + self.assertNotIn("module load BASE/1.0", out) + + +class HarvestTrustedTest(_unittest.TestCase): + + def _deploy(self, root): + arch = "x86_64-el9-gcc15-opt" + mroot = _os.path.join(root, arch, "Modules", "modulefiles") + proot = _os.path.join(root, arch, "Packages") + pkgs = {"Boost": ("1.90.0-1", ["CMake/3.30.6-1", "Python/3.13.11-1"], "hB"), + "CMake": ("3.30.6-1", [], "hC"), + "Python": ("3.13.11-1", [], "hP")} + for pkg, (verrev, deps, h) in pkgs.items(): + md = _os.path.join(mroot, pkg); _os.makedirs(md) + with open(_os.path.join(md, verrev), "w") as fh: + fh.write(_mf(pkg, verrev, deps)) + pd = _os.path.join(proot, pkg, verrev); _os.makedirs(pd) + with open(_os.path.join(pd, ".meta.json"), "w") as fh: + _json.dump({"build_id": "rel-XYZ", + "package": {"hash": h, "version": verrev.split("-")[0], + "revision": "1"}}, fh) + return mroot, proot, arch + + def test_reanchor_hash_and_build_id(self): + with _tempfile.TemporaryDirectory() as root: + mroot, proot, arch = self._deploy(root) + corpus, hashes, build_id = harvest_trusted(mroot, proot) + self.assertEqual(build_id, "rel-XYZ") + self.assertEqual(hashes["Boost/1.90.0-1"], "hB") + boost = corpus["Boost/1.90.0-1"]["rendered"] + self.assertNotIn("BASEDIR", boost) # re-anchored + self.assertNotIn("BASE/1.0", boost) # BASE dep stripped + self.assertIn("set PKG_ROOT %s/Boost/$version" % proot, boost) + self.assertEqual(corpus["Boost/1.90.0-1"]["deps"], + ["CMake/3.30.6-1", "Python/3.13.11-1"]) + + def test_import_trusted_writes_overlay(self): + with _tempfile.TemporaryDirectory() as root, \ + _tempfile.TemporaryDirectory() as out: + mroot, proot, arch = self._deploy(root) + res = import_trusted_release(mroot, proot, arch, out) + self.assertEqual(res["build_id"], "rel-XYZ") + self.assertEqual(res["dangling"], []) # closure complete + mf = _os.path.join(out, "rel-XYZ", arch, "Boost", "1.90.0-1") + with open(mf) as fh: + self.assertIn("%s/Boost/$version" % proot, fh.read()) + with open(_os.path.join(out, "rel-XYZ", arch, "Boost", + ".1.90.0-1.meta.json")) as fh: + self.assertEqual(_json.load(fh)["hash"], "hB") + + class WriteOverlayRenderedTest(_unittest.TestCase): def test_rendered_modulefile_written_verbatim(self): diff --git a/tests/test_cvmfs_import_cmd.py b/tests/test_cvmfs_import_cmd.py index cac4af7b..f7c783f7 100644 --- a/tests/test_cvmfs_import_cmd.py +++ b/tests/test_cvmfs_import_cmd.py @@ -79,6 +79,34 @@ def test_open_release_forced(self): self.assertTrue(doImport( _args(out, importManifest=mf, importForce=True), None)) + def test_trusted_harvest_writes_overlay(self): + with tempfile.TemporaryDirectory() as dep, tempfile.TemporaryDirectory() as out: + arch = "x86_64-el9-gcc13" + mroot = os.path.join(dep, "Modules", "modulefiles") + proot = os.path.join(dep, "Packages") + for pkg, ver, deps, h in [("ROOT", "6.38.00-1", [], "hR")]: + os.makedirs(os.path.join(mroot, pkg)) + with open(os.path.join(mroot, pkg, ver), "w") as fh: + fh.write("#%%Module1.0\nset PKG_ROOT $::env(BASEDIR)/%s/%s\n" + "prepend-path PATH $PKG_ROOT/bin\n" % (pkg, ver)) + os.makedirs(os.path.join(proot, pkg, ver)) + with open(os.path.join(proot, pkg, ver, ".meta.json"), "w") as fh: + json.dump({"build_id": "rel-1", + "package": {"hash": h, "version": "6.38.00", + "revision": "1"}}, fh) + ok = doImport(_args(out, architecture=arch, importTrusted=True, + importModulepath=mroot, importInstallBase=proot), None) + self.assertTrue(ok) + mf = os.path.join(out, "rel-1", arch, "ROOT", "6.38.00-1") + with open(mf) as fh: + text = fh.read() + self.assertIn("%s/ROOT/6.38.00-1" % proot, text) # re-anchored absolute + self.assertNotIn("BASEDIR", text) + + def test_trusted_needs_modulepath_and_install_base(self): + with tempfile.TemporaryDirectory() as out: + self.assertFalse(doImport(_args(out, importTrusted=True), None)) + def test_no_source_errors(self): with tempfile.TemporaryDirectory() as out: self.assertFalse(doImport(_args(out), None)) From c161a7b5969b6547f48c6350f8f687b66edd37c9 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 17:19:21 +0200 Subject: [PATCH 04/30] build: set up reused deps via modules in init.sh (dormant) generate_initdotsh gains a per-dependency branch: a dep marked reuse_module_id (satisfied from a reused CVMFS release) is set up via 'module use ' + 'module load ' instead of sourcing its init.sh, while locally-built deps are unchanged. Per-dependency, so a legacy package can consume a module-reused dep. Dormant until the marker and reuse_modulepath are wired (next step). --- bits_helpers/build.py | 17 +++++++++++++++-- tests/test_build.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index d7eeb0c6..a07cdd26 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -1105,7 +1105,8 @@ def _pkg_install_path(workDir, architecture, spec): def generate_initdotsh(package, specs, architecture, workDir="sw", post_build=False, - from_modules=False, cmake_prefix_env=False): + from_modules=False, cmake_prefix_env=False, + reuse_modulepath=None): """Return the contents of the given package's etc/profile/init.sh as a string. If post_build is true, also generate variables pointing to the package @@ -1173,7 +1174,19 @@ def _dep_init_path(dep): package=quote(dep_spec["package"]), ver_rev=quote(ver_rev(dep_spec)), ) - lines.extend(_dep_init_path(dep) for dep in spec.get("requires", ())) + # A dependency satisfied from a reused CVMFS release is set up via its overlay + # modulefile (module use/load); one built locally keeps the init.sh sourcing. + # The choice is per-DEPENDENCY, not per this package's mode, so a legacy-built + # package can still consume a module-reused dependency. Reused deps go first so + # their env is in place before any built dep's init.sh references it. + _reqs = list(spec.get("requires", ())) + _reused = [d for d in _reqs + if reuse_modulepath and specs[d].get("reuse_module_id")] + if _reused: + lines.append('module use "%s"' % reuse_modulepath) + lines.extend("module load %s" % specs[d]["reuse_module_id"] for d in _reused) + _reused_set = set(_reused) + lines.extend(_dep_init_path(dep) for dep in _reqs if dep not in _reused_set) if post_build: bigpackage = pkg_to_shell_id(package) diff --git a/tests/test_build.py b/tests/test_build.py index 84f0521c..c51c434d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -652,6 +652,36 @@ def test_initdotsh_from_modules_shared_dependency(self) -> None: self.assertIn('export LIBSHARED_INCLUDE_DIR="${LIBSHARED_ROOT}/include"', shared) self.assertNotIn('$BITS_ARCH_PREFIX"/libshared', shared) + def test_initdotsh_reuse_modules(self) -> None: + """A dependency satisfied from a reused CVMFS release is set up via its + overlay modulefile (module use/load); a locally-built dependency keeps its + init.sh sourcing — so a plain/legacy package can consume a module-reused + dep. Dormant (byte-identical) when no reuse_modulepath is passed.""" + base = {"revision": "1", "hash": "h", "commit_hash": "c"} + specs = { + "App": dict(base, package="App", version="1.0", + requires=["CMake", "Boost"]), + "CMake": dict(base, package="CMake", version="3.30.6", requires=[]), + "Boost": dict(base, package="Boost", version="1.90.0", requires=[]), + } + # Baseline: no reuse path -> both deps sourced via init.sh, no modules. + baseline = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False) + self.assertNotIn("module use", baseline) + self.assertIn("/CMake/3.30.6-1/etc/profile.d/init.sh", baseline) + + # Mark CMake reused-from-CVMFS; Boost stays locally built. + specs["CMake"]["reuse_module_id"] = "CMake/3.30.6-1" + out = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False, + reuse_modulepath="/ov/bid/slc7_x86-64") + self.assertIn('module use "/ov/bid/slc7_x86-64"', out) + self.assertIn("module load CMake/3.30.6-1", out) + self.assertNotIn("/CMake/3.30.6-1/etc/profile.d/init.sh", out) # not sourced + self.assertIn("/Boost/1.90.0-1/etc/profile.d/init.sh", out) # still built + + # Dormant safety: marker set but no modulepath -> identical to baseline. + dormant = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False) + self.assertEqual(dormant, baseline) + if __name__ == '__main__': unittest.main() From c31ba2443bd912dc2a99bc0dae41890a9a7b04e5 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 17:30:16 +0200 Subject: [PATCH 05/30] bits wrapper: route 'import' to bitsBuild The import subcommand is dispatched in bitsBuild but was missing from the wrapper's command allowlist, so 'bits import' failed with 'Unknown command'. Add it to the route list. --- bits | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bits b/bits index 9d51d158..adb49592 100755 --- a/bits +++ b/bits @@ -400,7 +400,7 @@ fi for arg in "$@" do case $arg in - architecture|brew|build|certify|clean|cleanup|compliance|cvmfs-path|deps|doctor|gc|init|publish|stats|status|store-stats|verify|version|-debug|-d) + architecture|brew|build|certify|clean|cleanup|compliance|cvmfs-path|deps|doctor|gc|import|init|publish|stats|status|store-stats|verify|version|-debug|-d) mkdir -p "$BITS_WORK_DIR" || echo "Cannot create directory: $BITS_WORK_DIR" "$BITSDIR/bitsBuild" "$@" exit $? From 91aca5182afe520c373a4f39c2938166b5deaef7 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 17:38:19 +0200 Subject: [PATCH 06/30] import: strip the multi-line BASE guard cleanly The deployed BASE dependency is a multi-line 'if ![ is-loaded BASE ] { module load BASE }' block; the old per-line strip dropped the first two lines but left the orphan '}', so the reused modulefile failed to load (invalid command name }). Make strip_base_dep brace-aware: consume through the matching '}'. Found by the testbed module load. --- bits_helpers/cvmfs_import.py | 35 ++++++++++++++++++++++++++--------- tests/test_cvmfs_import.py | 29 ++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/bits_helpers/cvmfs_import.py b/bits_helpers/cvmfs_import.py index 19ef5eb4..39c6d0a7 100644 --- a/bits_helpers/cvmfs_import.py +++ b/bits_helpers/cvmfs_import.py @@ -262,24 +262,41 @@ def _is_base_id(token): return t == "BASE" or t.startswith("BASE/") +def _line_loads_base(line): + """True if *line* has a ``load`` / ``prereq`` / ``is-loaded`` directive whose + next token is the BASE module id — matched by token so ``BASECAMP`` etc. are + not mis-matched.""" + _directives = ("load", "prereq", "prereq-all", "depends-on", "is-loaded") + toks = line.split() + return any(tok in _directives and i + 1 < len(toks) and _is_base_id(toks[i + 1]) + for i, tok in enumerate(toks)) + + def strip_base_dep(text): - """Drop the ``BASE`` module dependency line(s) from a re-anchored modulefile. + """Drop the ``BASE`` module dependency from a re-anchored modulefile. A deployed bits modulefile loads ``BASE`` only to obtain ``BASEDIR``; once ``rewrite_module_anchor`` has inlined that as an absolute path, ``BASE`` is unneeded (and may be absent from the reuse set). Real deps (``CMake``, - ``Python``…) are kept. A line is a BASE dependency when a ``load`` / ``prereq`` - / ``is-loaded`` directive on it targets BASE as a module id — matched by token - (like ``_module_load_deps``) so a package named ``BASE…`` is not mis-stripped. + ``Python``…) are kept. Handles both the one-line guard and the multi-line + ``if ![ is-loaded 'BASE/1.0' ] {`` / ``module load BASE/1.0`` / ``}`` block: + when the BASE line opens an unbalanced ``{``, consume through its matching + ``}`` so no orphan brace is left behind. """ - _directives = ("load", "prereq", "prereq-all", "depends-on", "is-loaded") out = [] - for line in text.splitlines(): - toks = line.split() - if any(tok in _directives and i + 1 < len(toks) and _is_base_id(toks[i + 1]) - for i, tok in enumerate(toks)): + lines = text.splitlines() + i, n = 0, len(lines) + while i < n: + line = lines[i] + if _line_loads_base(line): + depth = line.count("{") - line.count("}") + i += 1 + while depth > 0 and i < n: + depth += lines[i].count("{") - lines[i].count("}") + i += 1 continue out.append(line) + i += 1 return "\n".join(out) + ("\n" if text.endswith("\n") else "") diff --git a/tests/test_cvmfs_import.py b/tests/test_cvmfs_import.py index 224c0020..8d2df947 100644 --- a/tests/test_cvmfs_import.py +++ b/tests/test_cvmfs_import.py @@ -54,9 +54,12 @@ def test_no_basedir_is_noop(self): def _mf(pkg, verrev, deps): - """A deployed bits modulefile (BASEDIR-anchored, loads BASE + deps).""" - lines = ["#%%Module1.0", "set version %s" % verrev, - "if ![ is-loaded 'BASE/1.0' ] { module load BASE/1.0 }"] + """A deployed bits modulefile as really published: a multi-line proc block, + a MULTI-LINE `if { ... }` BASE guard, single-line dep guards, BASEDIR anchor.""" + lines = ["#%%Module1.0", + "proc ModulesHelp { } {", ' puts stderr "help"', "}", + "set version %s" % verrev, "# Dependencies", + "if ![ is-loaded 'BASE/1.0' ] {", " module load BASE/1.0", "}"] for d in deps: lines.append('if ![ is-loaded "%s" ] { module load %s }' % (d, d)) lines += ["set PKG_ROOT $::env(BASEDIR)/%s/$version" % pkg, @@ -65,19 +68,35 @@ def _mf(pkg, verrev, deps): return "\n".join(lines) + "\n" +def _braces_balanced(text): + return text.count("{") == text.count("}") + + class StripBaseDepTest(_unittest.TestCase): def test_strips_only_base(self): text = _mf("Boost", "1.90.0-1", ["CMake/3.30.6-1", "Python/3.13.11-1"]) + self.assertTrue(_braces_balanced(text)) out = strip_base_dep(text) - # The BASE load/is-loaded line is gone; real deps and BASEDIR ref remain - # (re-anchoring, not this function, removes BASEDIR). + # The whole multi-line BASE block is gone — no orphan '}' left behind + # (regression: the deployment's guard spans `if {` / `module load` / `}`). self.assertNotIn("module load BASE", out) self.assertNotIn("is-loaded 'BASE", out) + self.assertTrue(_braces_balanced(out), "unbalanced braces after strip:\n" + out) + # Real deps and the proc block survive; BASEDIR ref stays (re-anchor removes it). self.assertIn("module load CMake/3.30.6-1", out) self.assertIn("module load Python/3.13.11-1", out) + self.assertIn("proc ModulesHelp", out) self.assertIn("BASEDIR", out) + def test_strips_single_line_base_guard(self): + # The one-line form must still be handled (balanced { } on one line). + text = "if ![ is-loaded 'BASE/1.0' ] { module load BASE/1.0 }\nset x 1\n" + out = strip_base_dep(text) + self.assertNotIn("BASE", out) + self.assertIn("set x 1", out) + self.assertTrue(_braces_balanced(out)) + def test_base_prefix_package_not_stripped(self): # A package whose id merely starts with "BASE" must be kept. text = ('if ![ is-loaded "BASECAMP/1.0" ] { module load BASECAMP/1.0 }\n' From 1a2607bf6f3ded930e4d83989213b30c2f318894 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 17:46:12 +0200 Subject: [PATCH 07/30] build: import the reuse overlay from --reuse-from (4a, dormant) When --reuse-from is set, doBuild runs import_trusted_release to build the local re-anchored overlay (install base from the layout) and threads its path into generate_initdotsh. import_trusted_release now returns the overlay path. Dormant: nothing is marked reused yet, so builds are unchanged; a --reuse-from build just logs the imported overlay. --- bits_helpers/build.py | 28 ++++++++++++++++++++++++++-- bits_helpers/cvmfs_import.py | 8 ++++++-- tests/test_cvmfs_import.py | 2 ++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index a07cdd26..e1911f10 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -2418,8 +2418,30 @@ def defaultsReader(): args.reuseFrom = resolve_reuse_from(getattr(args, "reuseFrom", None), _cvmfs) except ValueError as exc: dieOnError(True, str(exc)) + # Build the local, re-anchored overlay from the deployment named by + # --reuse-from, so reused deps can be set up via modules. The install base (the + # deployment's Packages root) comes from the layout — available for + # --reuse-from cvmfs. Dormant until packages are marked reused (next step): the + # overlay is produced and its path threaded to generate_initdotsh, but nothing + # is grafted yet, so the build is unchanged. + args.reuseOverlay = None if args.reuseFrom: info("Reuse-from modules: %s", args.reuseFrom) + _install_base = _cvmfs.get("install_path") if _cvmfs else None + dieOnError(not _install_base, + "--reuse-from needs the deployment's Packages root; declare " + "install_dir / cvmfs_dir in the defaults system: layout.") + from bits_helpers.cvmfs_import import import_trusted_release + _res = import_trusted_release(args.reuseFrom, _install_base, args.architecture, + os.path.join(workDir, "MODULES")) + if _res.get("build_id"): + args.reuseOverlay = _res["overlay_path"] + info("Reuse overlay: %d module(s) -> %s", + len(_res["written"]), args.reuseOverlay) + else: + warning("Reuse overlay: release under %s is not closed (missing: %s); " + "no modules imported.", args.reuseFrom, + ", ".join(_res.get("dangling", []))) # Build-host policy knobs live under a single `system:` entry in defaults. # These control *how* the build runs (network, CPU) — not *what* it produces — @@ -3871,10 +3893,12 @@ def _build_row(pkg): "provenance": create_provenance_info(spec["package"], specs, args), "initdotsh_deps": generate_initdotsh(p, specs, args.architecture, workDir=init_workDir, post_build=False, from_modules=getattr(args, "initdotshFromModules", False), - cmake_prefix_env=_cmake_prefix_env), + cmake_prefix_env=_cmake_prefix_env, + reuse_modulepath=getattr(args, "reuseOverlay", None)), "initdotsh_full": generate_initdotsh(p, specs, args.architecture, workDir=init_workDir, post_build=True, from_modules=getattr(args, "initdotshFromModules", False), - cmake_prefix_env=_cmake_prefix_env), + cmake_prefix_env=_cmake_prefix_env, + reuse_modulepath=getattr(args, "reuseOverlay", None)), "develPrefix": develPrefix, "workDir": workDir, "configDir": abspath(args.configDir), diff --git a/bits_helpers/cvmfs_import.py b/bits_helpers/cvmfs_import.py index 39c6d0a7..7fca0353 100644 --- a/bits_helpers/cvmfs_import.py +++ b/bits_helpers/cvmfs_import.py @@ -362,13 +362,17 @@ def import_trusted_release(module_root, install_base, arch, out_root, label="reu one. Returns ``{"build_id", "written", "dangling"}`` (same shape as ``import_release``). """ + import os corpus, hashes, dep_build_id = harvest_trusted(module_root, install_base) dangling = closure_check(corpus) if dangling and not force: - return {"build_id": None, "written": [], "dangling": dangling} + return {"build_id": None, "written": [], "dangling": dangling, + "overlay_path": None} build_id = dep_build_id or compute_corpus_build_id(corpus, label) written = write_overlay(corpus, build_id, arch, out_root, package_hashes=hashes) - return {"build_id": build_id, "written": written, "dangling": dangling} + # Return the overlay path so callers need not reconstruct //. + return {"build_id": build_id, "written": written, "dangling": dangling, + "overlay_path": os.path.join(out_root, build_id, arch)} def build_module_meta(module_id, entry, build_id, package_hash="", abi_tag=""): diff --git a/tests/test_cvmfs_import.py b/tests/test_cvmfs_import.py index 8d2df947..b6a67b21 100644 --- a/tests/test_cvmfs_import.py +++ b/tests/test_cvmfs_import.py @@ -146,6 +146,8 @@ def test_import_trusted_writes_overlay(self): res = import_trusted_release(mroot, proot, arch, out) self.assertEqual(res["build_id"], "rel-XYZ") self.assertEqual(res["dangling"], []) # closure complete + self.assertEqual(res["overlay_path"], + _os.path.join(out, "rel-XYZ", arch)) mf = _os.path.join(out, "rel-XYZ", arch, "Boost", "1.90.0-1") with open(mf) as fh: self.assertIn("%s/Boost/$version" % proot, fh.read()) From a0b9263fb05a5a70fec407615ec53d35b83eefdc Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 18:03:32 +0200 Subject: [PATCH 08/30] build: mark overlay-satisfied packages for module reuse (4b-1) After each package's hash is finalized, if the reuse overlay satisfies it (strict: same remote hash, publishable; relaxed: any version in the one-release overlay) set reuse_module_id so consumers 'module load' it instead of sourcing its init.sh. defaults-*/--build-local never grafted. The package still builds here; skipping its build is the next step. --- bits_helpers/build.py | 18 ++++++++++++++++++ bits_helpers/cvmfs_import.py | 22 ++++++++++++++++++++++ tests/test_cvmfs_import.py | 31 ++++++++++++++++++++++++++++++- 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index e1911f10..d3c6153b 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -3341,6 +3341,24 @@ def _build_row(pkg): debug("Hashes for recipe %s are %s (remote); %s (local)", p, ", ".join(spec["remote_hashes"]), ", ".join(spec["local_hashes"])) + # 4b-1: if the reuse overlay satisfies this package, mark it so its consumers + # set it up via 'module load' (generate_initdotsh) instead of sourcing its + # init.sh. Strict = same remote hash (byte-identical, publishable); relaxed = + # any version in the (one-release) overlay. defaults-* and --build-local + # packages are never grafted. The package still builds here; skipping its + # build is a later step. + if getattr(args, "reuseOverlay", None) and not spec["package"].startswith("defaults-"): + _bl = set(x for x in (getattr(args, "buildLocal", "") or "").split(",") if x) + _relaxed = getattr(args, "reusePolicy", "strict") == "relaxed" + _want = None if _relaxed else spec.get("remote_revision_hash") + # In strict mode a missing hash must NOT fall through to match-any. + if spec["package"] not in _bl and (_relaxed or _want): + from bits_helpers.cvmfs_import import overlay_reuse_module + _mid = overlay_reuse_module(args.reuseOverlay, spec["package"], want_hash=_want) + if _mid: + spec["reuse_module_id"] = _mid + info("Reuse: %s from CVMFS overlay as module %s", p, _mid) + # Warn if a package declares architecture: shared but has arch-specific # deps — the shared label would be misleading in that case because its # hash (and therefore install path) will differ across platforms. diff --git a/bits_helpers/cvmfs_import.py b/bits_helpers/cvmfs_import.py index 7fca0353..58fb90f9 100644 --- a/bits_helpers/cvmfs_import.py +++ b/bits_helpers/cvmfs_import.py @@ -375,6 +375,28 @@ def import_trusted_release(module_root, install_base, arch, out_root, label="reu "overlay_path": os.path.join(out_root, build_id, arch)} +def overlay_reuse_module(overlay_path, package, want_hash=None): + """Return ``/`` if the overlay satisfies *package*, else None. + + The overlay is ``//`` (modulefile) alongside a + hidden ``..meta.json``. Strict (``want_hash`` given): match a module + whose recorded hash equals it — a byte-identical, publishable reuse. Relaxed + (``want_hash`` None): any module for the package (the overlay is one coherent + release). Defensive: a missing overlay/package yields None. + """ + import os + pkg_dir = os.path.join(overlay_path or "", package) + if not (overlay_path and os.path.isdir(pkg_dir)): + return None + for name in sorted(os.listdir(pkg_dir)): + if name.startswith("."): # skip the hidden ..meta.json + continue + meta = _read_meta(os.path.join(pkg_dir, ".%s.meta.json" % name)) or {} + if want_hash is None or meta.get("hash") == want_hash: + return "%s/%s" % (package, name) + return None + + def build_module_meta(module_id, entry, build_id, package_hash="", abi_tag=""): """Module-side ``.meta.json`` payload for a corpus *entry* (D6 overlay). diff --git a/tests/test_cvmfs_import.py b/tests/test_cvmfs_import.py index b6a67b21..f53d243c 100644 --- a/tests/test_cvmfs_import.py +++ b/tests/test_cvmfs_import.py @@ -14,7 +14,7 @@ build_module_meta, AliasMap, corpus_from_manifest, _infer_base_prefix, write_overlay, import_release, rewrite_module_anchor, - strip_base_dep, harvest_trusted, import_trusted_release, + strip_base_dep, harvest_trusted, import_trusted_release, overlay_reuse_module, ) import json as _json import os as _os @@ -156,6 +156,35 @@ def test_import_trusted_writes_overlay(self): self.assertEqual(_json.load(fh)["hash"], "hB") +class OverlayReuseModuleTest(_unittest.TestCase): + + def _overlay(self, root): + d = _os.path.join(root, "Boost"); _os.makedirs(d) + open(_os.path.join(d, "1.90.0-1"), "w").close() + with open(_os.path.join(d, ".1.90.0-1.meta.json"), "w") as fh: + _json.dump({"hash": "hB"}, fh) + return root + + def test_strict_hash_match(self): + with _tempfile.TemporaryDirectory() as ov: + self._overlay(ov) + self.assertEqual(overlay_reuse_module(ov, "Boost", want_hash="hB"), + "Boost/1.90.0-1") + self.assertIsNone(overlay_reuse_module(ov, "Boost", want_hash="other")) + + def test_relaxed_any_version(self): + with _tempfile.TemporaryDirectory() as ov: + self._overlay(ov) + self.assertEqual(overlay_reuse_module(ov, "Boost", want_hash=None), + "Boost/1.90.0-1") + + def test_missing_package_or_overlay(self): + with _tempfile.TemporaryDirectory() as ov: + self._overlay(ov) + self.assertIsNone(overlay_reuse_module(ov, "Nope", want_hash="hB")) + self.assertIsNone(overlay_reuse_module(None, "Boost", want_hash="hB")) + + class WriteOverlayRenderedTest(_unittest.TestCase): def test_rendered_modulefile_written_verbatim(self): From ff59e63074985266668a3dec0d25cabf6b54e567 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 18:18:45 +0200 Subject: [PATCH 09/30] reuse: derive Packages base for an explicit --reuse-from path With --reuse-from cvmfs the Packages root comes from the layout; with an explicit .../Modules/modulefiles path and no layout, derive it by the deployment convention (Modules/modulefiles<->Packages, the map BASE uses) so the explicit form works without configuring the layout. --- bits_helpers/build.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index d3c6153b..fd4fdcae 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -2428,9 +2428,15 @@ def defaultsReader(): if args.reuseFrom: info("Reuse-from modules: %s", args.reuseFrom) _install_base = _cvmfs.get("install_path") if _cvmfs else None + if not _install_base and "Modules/modulefiles" in args.reuseFrom: + # Explicit --reuse-from path, no layout: derive the Packages root by the + # deployment convention (the same Modules/modulefiles<->Packages map the + # BASE module uses). + _install_base = args.reuseFrom.replace("Modules/modulefiles", "Packages") dieOnError(not _install_base, "--reuse-from needs the deployment's Packages root; declare " - "install_dir / cvmfs_dir in the defaults system: layout.") + "install_dir / cvmfs_dir in the defaults system: layout, or point " + "--reuse-from at a .../Modules/modulefiles tree.") from bits_helpers.cvmfs_import import import_trusted_release _res = import_trusted_release(args.reuseFrom, _install_base, args.architecture, os.path.join(workDir, "MODULES")) From ca2a85c1816dd4be994bac669f8f913816899e40 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 18:34:48 +0200 Subject: [PATCH 10/30] build: skip build/unpack for overlay-reused packages (4b-2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A package satisfied by the reuse overlay is now set up from modules only: it adopts a consistent identity, records already_installed, and continues before fetch_symlinks — so the legacy CVMFS tarball synthesis + relocate path is skipped entirely (that was the relocate-me.sh failure). Devel packages excluded. Initialise mainBuildFamily before the loop so reusing the target package doesn't leave the final banner unset. --- bits_helpers/build.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index fd4fdcae..3d20617c 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -3329,6 +3329,13 @@ def _build_row(pkg): import atexit atexit.register(lambda ex=_prefetch_executor: ex.shutdown(wait=False, cancel_futures=True)) + # Default build_family for the final banner, matching the in-loop formula + # (build.py sets it per-package below). Needed so a reused/skipped MAIN package + # does not leave mainBuildFamily unset; the in-loop assignment overrides it + # whenever the main package is actually built. + _mdp = getattr(args, "develPrefix", develPackageBranch) + mainBuildFamily = ("{}-{}".format(_mdp, "_".join(args.defaults)) if _mdp + else "_".join(args.defaults)) while buildOrder: p = buildOrder.pop(0) spec = specs[p] @@ -3347,13 +3354,15 @@ def _build_row(pkg): debug("Hashes for recipe %s are %s (remote); %s (local)", p, ", ".join(spec["remote_hashes"]), ", ".join(spec["local_hashes"])) - # 4b-1: if the reuse overlay satisfies this package, mark it so its consumers - # set it up via 'module load' (generate_initdotsh) instead of sourcing its - # init.sh. Strict = same remote hash (byte-identical, publishable); relaxed = - # any version in the (one-release) overlay. defaults-* and --build-local - # packages are never grafted. The package still builds here; skipping its - # build is a later step. - if getattr(args, "reuseOverlay", None) and not spec["package"].startswith("defaults-"): + # 4b: if the reuse overlay satisfies this package, set it up from modules + # instead of building. Its consumers 'module load' it (generate_initdotsh); + # it is neither built nor materialized locally, so we skip the whole + # build/unpack path here (this is what avoids the legacy tarball synthesis + + # relocate-me.sh). Strict = same remote hash (byte-identical, publishable); + # relaxed = any version in the one-release overlay. defaults-*, --build-local + # and development packages are never grafted. + if (getattr(args, "reuseOverlay", None) and not spec["is_devel_pkg"] + and not spec["package"].startswith("defaults-")): _bl = set(x for x in (getattr(args, "buildLocal", "") or "").split(",") if x) _relaxed = getattr(args, "reusePolicy", "strict") == "relaxed" _want = None if _relaxed else spec.get("remote_revision_hash") @@ -3362,8 +3371,19 @@ def _build_row(pkg): from bits_helpers.cvmfs_import import overlay_reuse_module _mid = overlay_reuse_module(args.reuseOverlay, spec["package"], want_hash=_want) if _mid: + # Adopt a consistent identity for the manifest, then skip the build. spec["reuse_module_id"] = _mid - info("Reuse: %s from CVMFS overlay as module %s", p, _mid) + _verrev = _mid.split("/", 1)[1] + spec["revision"] = (_verrev[len(spec["version"]) + 1:] + if _verrev.startswith(spec["version"] + "-") else _verrev) + spec["hash"] = spec.get("remote_revision_hash") or spec.get("hash", "") + spec["cachedTarball"] = "" + spec.setdefault("deps_hash", "") + info("Reuse: %s from CVMFS overlay as module %s (not built)", p, _mid) + if getattr(args, "manifest", None) is not None: + args.manifest.add_package(spec, "already_installed", + effective_architecture=effective_arch(spec, args.architecture)) + continue # Warn if a package declares architecture: shared but has arch-specific # deps — the shared label would be misleading in that case because its From de3dbd6dd6f7f93cc02580c5325212ad77876e64 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 18:55:05 +0200 Subject: [PATCH 11/30] reuse: handle --build-local as the normalized list args.buildLocal is normalized to a list (args.py:1965), but the 4b reuse guard treated it as a comma string and called .split() -> AttributeError when --build-local was passed. Accept both list and string forms. --- bits_helpers/build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 3d20617c..346694fe 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -3363,7 +3363,10 @@ def _build_row(pkg): # and development packages are never grafted. if (getattr(args, "reuseOverlay", None) and not spec["is_devel_pkg"] and not spec["package"].startswith("defaults-")): - _bl = set(x for x in (getattr(args, "buildLocal", "") or "").split(",") if x) + _bl_raw = getattr(args, "buildLocal", None) or [] + if isinstance(_bl_raw, str): + _bl_raw = _bl_raw.split(",") + _bl = set(x for x in _bl_raw if x) _relaxed = getattr(args, "reusePolicy", "strict") == "relaxed" _want = None if _relaxed else spec.get("remote_revision_hash") # In strict mode a missing hash must NOT fall through to match-any. From 214e860381030684efeb8ed8c97f31234cb7c00c Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 19:02:58 +0200 Subject: [PATCH 12/30] reuse: emit the overlay modulepath as the container path generate_initdotsh was given the host overlay abspath, so 'module use' inside the build container pointed at a nonexistent /home/... path. Pass the overlay under init_workDir (the container workdir under --docker), so it resolves at $WORK_DIR/MODULES// in the container. --- bits_helpers/build.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 346694fe..23e0954f 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -2425,6 +2425,7 @@ def defaultsReader(): # overlay is produced and its path threaded to generate_initdotsh, but nothing # is grafted yet, so the build is unchanged. args.reuseOverlay = None + args.reuseBuildId = None if args.reuseFrom: info("Reuse-from modules: %s", args.reuseFrom) _install_base = _cvmfs.get("install_path") if _cvmfs else None @@ -2441,7 +2442,8 @@ def defaultsReader(): _res = import_trusted_release(args.reuseFrom, _install_base, args.architecture, os.path.join(workDir, "MODULES")) if _res.get("build_id"): - args.reuseOverlay = _res["overlay_path"] + args.reuseOverlay = _res["overlay_path"] # host path, for reading + args.reuseBuildId = _res["build_id"] # to build the container path info("Reuse overlay: %d module(s) -> %s", len(_res["written"]), args.reuseOverlay) else: @@ -3927,6 +3929,12 @@ def _build_row(pkg): ver_rev(spec)) init_workDir = container_workDir if args.docker else args.workDir + # Overlay modulepath as seen from where init.sh runs (the container workdir + # under --docker, the host workdir otherwise) — NOT the host abspath, which + # would not exist inside the build container. + _reuse_modulepath = (os.path.join(init_workDir, "MODULES", + args.reuseBuildId, args.architecture) + if getattr(args, "reuseBuildId", None) else None) makedirs(scriptDir, exist_ok=True) # Remember where the resource monitor will write this package's trace so we # can aggregate build stats once the run finishes (P3). @@ -3941,11 +3949,11 @@ def _build_row(pkg): "initdotsh_deps": generate_initdotsh(p, specs, args.architecture, workDir=init_workDir, post_build=False, from_modules=getattr(args, "initdotshFromModules", False), cmake_prefix_env=_cmake_prefix_env, - reuse_modulepath=getattr(args, "reuseOverlay", None)), + reuse_modulepath=_reuse_modulepath), "initdotsh_full": generate_initdotsh(p, specs, args.architecture, workDir=init_workDir, post_build=True, from_modules=getattr(args, "initdotshFromModules", False), cmake_prefix_env=_cmake_prefix_env, - reuse_modulepath=getattr(args, "reuseOverlay", None)), + reuse_modulepath=_reuse_modulepath), "develPrefix": develPrefix, "workDir": workDir, "configDir": abspath(args.configDir), From 551e344ef7523b0ce7325c7fae6d8830ad47673e Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 19:26:50 +0200 Subject: [PATCH 13/30] reuse: set up reused deps by sourcing their CVMFS init.sh (4c) Replace module use/load (which needs modulecmd in the image) with the standard mechanism: source the dep's deployed init.sh from CVMFS. Point its $WORK_DIR/$BITS_ARCH_PREFIX at the Packages base while sourcing; BITS_ARCH_PREFIX="." (non-null) survives the deployed init.sh's `:= ` default that "" would not. Mount /cvmfs into the build container when reuse is active. No image change, no modulecmd. --- bits_helpers/build.py | 56 +++++++++++++++++++++++++++++-------------- tests/test_build.py | 27 +++++++++++++-------- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 23e0954f..6ab87cbf 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -1106,7 +1106,7 @@ def _pkg_install_path(workDir, architecture, spec): def generate_initdotsh(package, specs, architecture, workDir="sw", post_build=False, from_modules=False, cmake_prefix_env=False, - reuse_modulepath=None): + reuse_cvmfs_base=None): """Return the contents of the given package's etc/profile/init.sh as a string. If post_build is true, also generate variables pointing to the package @@ -1174,17 +1174,33 @@ def _dep_init_path(dep): package=quote(dep_spec["package"]), ver_rev=quote(ver_rev(dep_spec)), ) - # A dependency satisfied from a reused CVMFS release is set up via its overlay - # modulefile (module use/load); one built locally keeps the init.sh sourcing. - # The choice is per-DEPENDENCY, not per this package's mode, so a legacy-built - # package can still consume a module-reused dependency. Reused deps go first so - # their env is in place before any built dep's init.sh references it. + # A dependency satisfied from a reused CVMFS release is set up by sourcing its + # DEPLOYED init.sh from CVMFS — the same mechanism as a local dep, just from + # the deployment. The deployed init.sh resolves paths via "$WORK_DIR/ + # $BITS_ARCH_PREFIX", so we point those at the CVMFS Packages base while + # sourcing (and restore after) so its own and its transitive deps' paths land + # on CVMFS. Per-DEPENDENCY, so a legacy-built package can consume a reused dep. + # Reused deps go first so their env is in place before a built dep references + # it. Needs /cvmfs mounted in the build container (no modulecmd required). _reqs = list(spec.get("requires", ())) _reused = [d for d in _reqs - if reuse_modulepath and specs[d].get("reuse_module_id")] + if reuse_cvmfs_base and specs[d].get("reuse_module_id")] if _reused: - lines.append('module use "%s"' % reuse_modulepath) - lines.extend("module load %s" % specs[d]["reuse_module_id"] for d in _reused) + # Point the deployed init.sh's "$WORK_DIR/$BITS_ARCH_PREFIX" at the CVMFS + # Packages base. BITS_ARCH_PREFIX MUST be non-null (the deployed init.sh's + # `: "${BITS_ARCH_PREFIX:=}"` would otherwise re-add the arch); "." is + # a harmless no-op segment (/./ == /). Save/restore so + # locally-built deps sourced afterwards keep the local WORK_DIR. + lines.append('_bits_swd="${WORK_DIR:-}"; _bits_sap="${BITS_ARCH_PREFIX:-}"') + lines.append('WORK_DIR="%s"; BITS_ARCH_PREFIX="."' % reuse_cvmfs_base) + for d in _reused: + dep_spec = specs[d] + verrev = dep_spec["reuse_module_id"].split("/", 1)[1] + lines.append( + '[ -n "${%s_REVISION}" ] || . "%s/%s/%s/etc/profile.d/init.sh"' + % (pkg_to_shell_id(d), reuse_cvmfs_base, dep_spec["package"], verrev)) + lines.append('WORK_DIR="${_bits_swd}"; BITS_ARCH_PREFIX="${_bits_sap}"; ' + 'unset _bits_swd _bits_sap') _reused_set = set(_reused) lines.extend(_dep_init_path(dep) for dep in _reqs if dep not in _reused_set) @@ -2426,6 +2442,7 @@ def defaultsReader(): # is grafted yet, so the build is unchanged. args.reuseOverlay = None args.reuseBuildId = None + args.reuseCvmfsBase = None if args.reuseFrom: info("Reuse-from modules: %s", args.reuseFrom) _install_base = _cvmfs.get("install_path") if _cvmfs else None @@ -2443,7 +2460,8 @@ def defaultsReader(): os.path.join(workDir, "MODULES")) if _res.get("build_id"): args.reuseOverlay = _res["overlay_path"] # host path, for reading - args.reuseBuildId = _res["build_id"] # to build the container path + args.reuseBuildId = _res["build_id"] + args.reuseCvmfsBase = _install_base # CVMFS Packages base for init.sh info("Reuse overlay: %d module(s) -> %s", len(_res["written"]), args.reuseOverlay) else: @@ -3929,12 +3947,10 @@ def _build_row(pkg): ver_rev(spec)) init_workDir = container_workDir if args.docker else args.workDir - # Overlay modulepath as seen from where init.sh runs (the container workdir - # under --docker, the host workdir otherwise) — NOT the host abspath, which - # would not exist inside the build container. - _reuse_modulepath = (os.path.join(init_workDir, "MODULES", - args.reuseBuildId, args.architecture) - if getattr(args, "reuseBuildId", None) else None) + # Reused deps are set up by sourcing their deployed init.sh from the CVMFS + # Packages base (an absolute /cvmfs path, identical on host and in the + # container once /cvmfs is mounted). + _reuse_cvmfs_base = getattr(args, "reuseCvmfsBase", None) makedirs(scriptDir, exist_ok=True) # Remember where the resource monitor will write this package's trace so we # can aggregate build stats once the run finishes (P3). @@ -3949,11 +3965,11 @@ def _build_row(pkg): "initdotsh_deps": generate_initdotsh(p, specs, args.architecture, workDir=init_workDir, post_build=False, from_modules=getattr(args, "initdotshFromModules", False), cmake_prefix_env=_cmake_prefix_env, - reuse_modulepath=_reuse_modulepath), + reuse_cvmfs_base=_reuse_cvmfs_base), "initdotsh_full": generate_initdotsh(p, specs, args.architecture, workDir=init_workDir, post_build=True, from_modules=getattr(args, "initdotshFromModules", False), cmake_prefix_env=_cmake_prefix_env, - reuse_modulepath=_reuse_modulepath), + reuse_cvmfs_base=_reuse_cvmfs_base), "develPrefix": develPrefix, "workDir": workDir, "configDir": abspath(args.configDir), @@ -4145,10 +4161,14 @@ def _build_row(pkg): "-v {workdir}:{container_workDir} {roSources}-v{configDir}:/pkgdist.bits:ro " "-v {scriptDir}/build.sh:/build.sh:ro " "-v {bits_dir}:/bits " + "{cvmfsMount}" "{mirrorVolume} {develVolumes} {additionalEnv} {additionalVolumes} " "-e HOME=/tmp -e SHELL=/bin/bash -e WORK_DIR_OVERRIDE={container_workDir} -e BITS_CONFIG_DIR_OVERRIDE=/pkgdist.bits {extraArgs} {image} bash -ex /build.sh" ).format( jobLabel=("--label bits-job=%s " % quote(_job_id)) if _job_id else "", + # Mount /cvmfs read-only when reusing deployed components, so a reused + # dep's init.sh (and its files under /cvmfs) resolve inside the container. + cvmfsMount=("-v /cvmfs:/cvmfs:ro " if getattr(args, "reuseCvmfsBase", None) else ""), platformArg="--platform %s " % quote(_docker_platform) if _docker_platform else "", roSources=_ro_sources, image=quote(args.dockerImage), diff --git a/tests/test_build.py b/tests/test_build.py index c51c434d..5d8e5e27 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -664,21 +664,28 @@ def test_initdotsh_reuse_modules(self) -> None: "CMake": dict(base, package="CMake", version="3.30.6", requires=[]), "Boost": dict(base, package="Boost", version="1.90.0", requires=[]), } - # Baseline: no reuse path -> both deps sourced via init.sh, no modules. + # Baseline: no reuse base -> both deps sourced from the local tree. baseline = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False) - self.assertNotIn("module use", baseline) - self.assertIn("/CMake/3.30.6-1/etc/profile.d/init.sh", baseline) + self.assertNotIn("/cvmfs", baseline) + self.assertIn('"$WORK_DIR/$BITS_ARCH_PREFIX"/CMake/3.30.6-1/etc/profile.d/init.sh', + baseline) # Mark CMake reused-from-CVMFS; Boost stays locally built. specs["CMake"]["reuse_module_id"] = "CMake/3.30.6-1" out = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False, - reuse_modulepath="/ov/bid/slc7_x86-64") - self.assertIn('module use "/ov/bid/slc7_x86-64"', out) - self.assertIn("module load CMake/3.30.6-1", out) - self.assertNotIn("/CMake/3.30.6-1/etc/profile.d/init.sh", out) # not sourced - self.assertIn("/Boost/1.90.0-1/etc/profile.d/init.sh", out) # still built - - # Dormant safety: marker set but no modulepath -> identical to baseline. + reuse_cvmfs_base="/cvmfs/x/Packages") + # CMake sourced from its DEPLOYED init.sh on CVMFS, under a WORK_DIR + # override. BITS_ARCH_PREFIX="." (non-null) survives the deployed init.sh's + # `:=` default, which "" would not. + self.assertIn('WORK_DIR="/cvmfs/x/Packages"; BITS_ARCH_PREFIX="."', out) + self.assertIn('. "/cvmfs/x/Packages/CMake/3.30.6-1/etc/profile.d/init.sh"', out) + # ...and NOT from the local tree; Boost (built) still is. + self.assertNotIn('"$WORK_DIR/$BITS_ARCH_PREFIX"/CMake/3.30.6-1/etc/profile.d/init.sh', + out) + self.assertIn('"$WORK_DIR/$BITS_ARCH_PREFIX"/Boost/1.90.0-1/etc/profile.d/init.sh', + out) + + # Dormant safety: marker set but no reuse base -> identical to baseline. dormant = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False) self.assertEqual(dormant, baseline) From fabdf3650b9af58ae16142e63c7c754630df05f4 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Mon, 24 Aug 2026 22:06:25 +0200 Subject: [PATCH 14/30] reuse: source reused-dep init.sh in topological order A reused dep's deployed init.sh transitively re-sources its own deps (guarded on _REVISION) from CVMFS. A local-built prerequisite must be set up first so that guard skips the CVMFS re-source, which misses a local-only build (e.g. bits-recipe-tools before a reused CMake). --- bits_helpers/build.py | 50 +++++++++++++++++++++++++++---------------- tests/test_build.py | 26 ++++++++++++++++++++++ 2 files changed, 58 insertions(+), 18 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 6ab87cbf..e816fc74 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -1180,29 +1180,43 @@ def _dep_init_path(dep): # $BITS_ARCH_PREFIX", so we point those at the CVMFS Packages base while # sourcing (and restore after) so its own and its transitive deps' paths land # on CVMFS. Per-DEPENDENCY, so a legacy-built package can consume a reused dep. - # Reused deps go first so their env is in place before a built dep references - # it. Needs /cvmfs mounted in the build container (no modulecmd required). + # Needs /cvmfs mounted in the build container (no modulecmd required). _reqs = list(spec.get("requires", ())) - _reused = [d for d in _reqs - if reuse_cvmfs_base and specs[d].get("reuse_module_id")] - if _reused: + _reused_set = {d for d in _reqs + if reuse_cvmfs_base and specs[d].get("reuse_module_id")} + + def _reused_dep_lines(d): # Point the deployed init.sh's "$WORK_DIR/$BITS_ARCH_PREFIX" at the CVMFS # Packages base. BITS_ARCH_PREFIX MUST be non-null (the deployed init.sh's # `: "${BITS_ARCH_PREFIX:=}"` would otherwise re-add the arch); "." is # a harmless no-op segment (/./ == /). Save/restore so - # locally-built deps sourced afterwards keep the local WORK_DIR. - lines.append('_bits_swd="${WORK_DIR:-}"; _bits_sap="${BITS_ARCH_PREFIX:-}"') - lines.append('WORK_DIR="%s"; BITS_ARCH_PREFIX="."' % reuse_cvmfs_base) - for d in _reused: - dep_spec = specs[d] - verrev = dep_spec["reuse_module_id"].split("/", 1)[1] - lines.append( - '[ -n "${%s_REVISION}" ] || . "%s/%s/%s/etc/profile.d/init.sh"' - % (pkg_to_shell_id(d), reuse_cvmfs_base, dep_spec["package"], verrev)) - lines.append('WORK_DIR="${_bits_swd}"; BITS_ARCH_PREFIX="${_bits_sap}"; ' - 'unset _bits_swd _bits_sap') - _reused_set = set(_reused) - lines.extend(_dep_init_path(dep) for dep in _reqs if dep not in _reused_set) + # locally-built deps keep the local WORK_DIR. + dep_spec = specs[d] + verrev = dep_spec["reuse_module_id"].split("/", 1)[1] + return [ + '_bits_swd="${WORK_DIR:-}"; _bits_sap="${BITS_ARCH_PREFIX:-}"', + 'WORK_DIR="%s"; BITS_ARCH_PREFIX="."' % reuse_cvmfs_base, + '[ -n "${%s_REVISION}" ] || . "%s/%s/%s/etc/profile.d/init.sh"' + % (pkg_to_shell_id(d), reuse_cvmfs_base, dep_spec["package"], verrev), + 'WORK_DIR="${_bits_swd}"; BITS_ARCH_PREFIX="${_bits_sap}"; ' + 'unset _bits_swd _bits_sap', + ] + + if _reused_set: + # Emit deps in topological order (prerequisites first) so a dep set up + # before a reused dep whose deployed init.sh transitively references it — + # e.g. a locally-built bits-recipe-tools before a reused CMake — sets its + # _REVISION first, and the deployed init.sh's guard skips the re-source + # (which would look on CVMFS where a local-only build does not exist). + _req_set = set(_reqs) + _order = [d for d in topological_sort(specs) if d in _req_set] + for d in _order: + if d in _reused_set: + lines.extend(_reused_dep_lines(d)) + else: + lines.append(_dep_init_path(d)) + else: + lines.extend(_dep_init_path(dep) for dep in _reqs) if post_build: bigpackage = pkg_to_shell_id(package) diff --git a/tests/test_build.py b/tests/test_build.py index 5d8e5e27..b51663f8 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -689,6 +689,32 @@ def test_initdotsh_reuse_modules(self) -> None: dormant = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False) self.assertEqual(dormant, baseline) + def test_initdotsh_reuse_orders_prereq_before_reused(self) -> None: + """A local prerequisite of a reused dep must be sourced BEFORE it: the + reused dep's deployed init.sh transitively sources that prereq (guarded on + its _REVISION) and would look on CVMFS, where a local-only build is absent. + Emitting in topological order sets the prereq's _REVISION first so the + guard skips the CVMFS re-source.""" + base = {"revision": "1", "hash": "h", "commit_hash": "c"} + specs = { + "App": dict(base, package="App", version="1.0", + requires=["CMake", "Tools"]), + # CMake (reused) declares Tools as a dependency, so its deployed + # init.sh sources Tools; Tools itself is built locally. + "CMake": dict(base, package="CMake", version="3.30.6", + requires=["Tools"]), + "Tools": dict(base, package="Tools", version="0.0.32", requires=[]), + } + specs["CMake"]["reuse_module_id"] = "CMake/3.30.6-1" + out = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False, + reuse_cvmfs_base="/cvmfs/x/Packages") + tools_line = '"$WORK_DIR/$BITS_ARCH_PREFIX"/Tools/0.0.32-1/etc/profile.d/init.sh' + cmake_line = '. "/cvmfs/x/Packages/CMake/3.30.6-1/etc/profile.d/init.sh"' + self.assertIn(tools_line, out) + self.assertIn(cmake_line, out) + self.assertLess(out.index(tools_line), out.index(cmake_line), + "local prerequisite Tools must be sourced before reused CMake") + if __name__ == '__main__': unittest.main() From f65d1484acc7922202077e9da85cbcdb09491693 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 00:30:59 +0200 Subject: [PATCH 15/30] reuse: silence legacy build_id-graft warnings under --reuse-from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The relaxed build_id graft path warns 'packages will be built / no packages will be grafted' when it has no cvmfs:// store or --reuse-base. Under --reuse-from overlay reuse those warnings are misleading — the overlay grafts independently. Gate them on args.reuseOverlay. Non-reuse-from output unchanged. --- bits_helpers/build.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index e816fc74..677b601e 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -2664,9 +2664,14 @@ def _truthy(v): _strategy = "latest-common" if str(args.reuseBase).strip().lower() == "latest-common" \ else "latest" _store = args.remoteStore or "" + # These warnings describe the legacy build_id graft path only. Silence them + # when --reuse-from overlay reuse is active: that path grafts on its own and + # the "packages will be built" message would contradict it. + _overlay_active = bool(getattr(args, "reuseOverlay", None)) if not _store.startswith("cvmfs://"): - warning("relaxed reuse: auto-select needs a cvmfs:// --remote-store; " - "no build_id selected, packages will be built.") + if not _overlay_active: + warning("relaxed reuse: auto-select needs a cvmfs:// --remote-store; " + "no build_id selected, packages will be built.") args.reuseBase = "" else: from bits_helpers.cvmfs_reuse import select_build_id @@ -2679,8 +2684,9 @@ def _truthy(v): len(_cov.get(_bid, ())), len(packages)) args.reuseBase = _bid else: - warning("relaxed reuse: no %s build_id found under %s/%s/Packages; " - "packages will be built.", _strategy, _root, args.architecture) + if not _overlay_active: + warning("relaxed reuse: no %s build_id found under %s/%s/Packages; " + "packages will be built.", _strategy, _root, args.architecture) args.reuseBase = "" # Publish guard: relaxed builds are loose-provenance (their closure includes # unverified deployed binaries) and must never reach a write store / publish @@ -2824,12 +2830,18 @@ def performPreferCheckWithTempDir(pkg, cmd): if getattr(args, "reusePolicy", "strict") == "relaxed": _base = getattr(args, "reuseBase", "") or "" _store = args.remoteStore or "" + # Silence these legacy-graft warnings when --reuse-from overlay reuse is + # active: the overlay grafts independently, so "no packages will be + # grafted" would be misleading. + _overlay_active = bool(getattr(args, "reuseOverlay", None)) if not _base: - warning("--reuse-policy relaxed needs --reuse-base (or defaults " - "reuse_base:); no packages will be grafted.") + if not _overlay_active: + warning("--reuse-policy relaxed needs --reuse-base (or defaults " + "reuse_base:); no packages will be grafted.") elif not _store.startswith("cvmfs://"): - warning("--reuse-policy relaxed needs a cvmfs:// --remote-store " - "(or --reuse-cvmfs); no packages will be grafted.") + if not _overlay_active: + warning("--reuse-policy relaxed needs a cvmfs:// --remote-store " + "(or --reuse-cvmfs); no packages will be grafted.") else: from bits_helpers.cvmfs_reuse import graftable_match _store_root = re.sub("^cvmfs://", "", _store) From a89cfb87dbbf7b7b2613e0acb79d296856eb6e4a Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 11:28:43 +0200 Subject: [PATCH 16/30] recipe: expand templated tag for tarball (sources:) recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolve_tag ran only under `if "source"` (git), so a tarball-only recipe with `tag: "v%(version)s"` kept the raw tag; it leaked into commit_hash and the SOURCES/// path, breaking the source copy. Resolve it on the sources path too — no-op for literal tags, and matches the git path. --- bits_helpers/build.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 677b601e..9a39fb4a 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -3085,9 +3085,14 @@ def _build_row(pkg): spec["commit_hash"] = "0" if "sources" in spec: + # Expand a templated tag (e.g. "v%(version)s") for tarball sources too. + # The git branch above only resolves it when a `source:` is present, so a + # tarball-only recipe kept the raw tag, which then leaked into commit_hash + # and the SOURCES/// path. No-op for literal tags. + spec["tag"] = resolve_tag(spec, defaultsMeta.get("variables")) for i, s in enumerate(spec["sources"]): resolved = resolveLocalPath(args.configDir, s) - spec["sources"][i] = resolved + spec["sources"][i] = resolved spec["commit_hash"] = spec["tag"] # Version may contain date params like tag, plus %(commit_hash)s, # %(short_hash)s and %(tag)s. From 9d5ef1ba38ffb2c123f1f5e294d60ac04ca00a17 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 13:50:40 +0200 Subject: [PATCH 17/30] reuse: correct reused deps' pkg-config prefix in init.sh A deployed .pc can bake a prefix= that doesn't match its CVMFS location (publish-time relocation), so pkg-config resolves a missing libdir and a consumer's find_package fails (xrootd -> Davix). Stage corrected .pc copies under $WORK_DIR with prefix set to the reused dep's real root, prepend to PKG_CONFIG_PATH. --- bits_helpers/build.py | 28 ++++++++++++++++++++++++++++ tests/test_build.py | 9 +++++++++ 2 files changed, 37 insertions(+) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 9a39fb4a..f54dcec3 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -1215,6 +1215,34 @@ def _reused_dep_lines(d): lines.extend(_reused_dep_lines(d)) else: lines.append(_dep_init_path(d)) + # A reused CVMFS package may ship a pkg-config .pc whose baked `prefix=` does + # not match its deployed location (publish-time relocation can misplace it), + # breaking find_package via pkg-config for a consumer (e.g. xrootd → Davix). + # The reuse anchoring already resolved each dep's real root into _ROOT, + # so stage corrected .pc copies (prefix rewritten to that root) in a writable + # dir and prepend it to PKG_CONFIG_PATH. Reads from read-only /cvmfs, writes + # under $WORK_DIR; a no-op for reused deps that ship no .pc. + _reused_roots = " ".join('"${%s_ROOT:-}"' % pkg_to_shell_id(d) + for d in _order if d in _reused_set) + lines.extend([ + '_bits_rpc="${WORK_DIR:-.}/reuse-pkgconfig"; mkdir -p "$_bits_rpc"', + 'for _bits_root in %s; do' % _reused_roots, + ' [ -n "$_bits_root" ] || continue', + ' for _bits_pcd in "$_bits_root/lib64/pkgconfig" "$_bits_root/lib/pkgconfig"; do', + ' [ -d "$_bits_pcd" ] || continue', + ' for _bits_pc in "$_bits_pcd"/*.pc; do', + ' [ -e "$_bits_pc" ] || continue', + ' sed "s|^prefix=.*|prefix=$_bits_root|" "$_bits_pc" > "$_bits_rpc/${_bits_pc##*/}"', + ' done', + ' done', + 'done', + # Prepend once — init.sh may be sourced repeatedly; avoid unbounded growth. + 'case ":${PKG_CONFIG_PATH:-}:" in', + ' *":$_bits_rpc:"*) ;;', + ' *) export PKG_CONFIG_PATH="$_bits_rpc${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" ;;', + 'esac', + 'unset _bits_rpc _bits_root _bits_pcd _bits_pc', + ]) else: lines.extend(_dep_init_path(dep) for dep in _reqs) diff --git a/tests/test_build.py b/tests/test_build.py index b51663f8..38e2010d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -685,9 +685,18 @@ def test_initdotsh_reuse_modules(self) -> None: self.assertIn('"$WORK_DIR/$BITS_ARCH_PREFIX"/Boost/1.90.0-1/etc/profile.d/init.sh', out) + # A reused dep's pkg-config .pc is re-staged with a corrected prefix (a + # deployed .pc can bake a wrong prefix=). The staging loop covers the + # REUSED dep (CMake) via its _ROOT, not the locally-built Boost. + self.assertIn('reuse-pkgconfig', out) + self.assertIn('"${CMAKE_ROOT:-}"', out) + self.assertNotIn('"${BOOST_ROOT:-}"', out) + self.assertIn('export PKG_CONFIG_PATH="$_bits_rpc', out) + # Dormant safety: marker set but no reuse base -> identical to baseline. dormant = generate_initdotsh("App", specs, "slc7_x86-64", post_build=False) self.assertEqual(dormant, baseline) + self.assertNotIn('reuse-pkgconfig', baseline) def test_initdotsh_reuse_orders_prereq_before_reused(self) -> None: """A local prerequisite of a reused dep must be sourced BEFORE it: the From b9bbe22f0dabcc66ec1608a6985ff2d0dd84c5dc Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 13:55:56 +0200 Subject: [PATCH 18/30] build: downgrade dangling-symlink notice to debug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The revision scan reports a TARS symlink whose store tarball is gone (leftover from a failed build or a cleanup) then skips it and rebuilds — self-healing, not actionable. It fired on every build as WARNING noise; make it debug. The unparseable-symlink case stays a warning. --- bits_helpers/build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index f54dcec3..e1cd7a49 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -3605,7 +3605,10 @@ def _build_row(pkg): # readlink() succeeds even for dangling symlinks, so we must check # existence explicitly. if not os.path.isfile(symlink_path): - warning("Ignoring dangling symlink in tarball directory: %s", symlink_path) + # Benign and self-healing: a leftover from a failed build or a cleanup + # that removed the store tarball. The scan skips it and the build + # rebuilds, so this is diagnostic noise, not actionable — keep it debug. + debug("Ignoring dangling symlink in tarball directory: %s", symlink_path) continue realPath = readlink(symlink_path) # The revision group is optional ((?:-((?:local)?[0-9]+))?) to handle From 635538dd1c5a3abce5745e99253516aff4ec5805 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 14:22:21 +0200 Subject: [PATCH 19/30] reuse: accept --reuse-from :: as sugar for --reuse-policy A trailing ::relaxed/::strict on --reuse-from sets the reuse policy alongside the source (e.g. cvmfs::relaxed). --reuse-policy stays canonical: it wins, and if both are given they must agree (error otherwise). Absolute paths never contain ::, so the split is unambiguous. --- bits_helpers/args.py | 4 +++- bits_helpers/build.py | 18 +++++++++++++++--- bits_helpers/cvmfs_layout.py | 17 +++++++++++++++++ tests/test_cvmfs_layout.py | 27 ++++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/bits_helpers/args.py b/bits_helpers/args.py index 8f512051..02296a52 100644 --- a/bits_helpers/args.py +++ b/bits_helpers/args.py @@ -748,7 +748,9 @@ def doParseArgs(): "absolute modules-tree path (distinct from --remote-store, which is the " "tarball store). The literal 'cvmfs' resolves the exact location from the " "defaults `system:` layout (module_dir under cvmfs_dir); fails if that is " - "not configured.")) + "not configured. A trailing '::relaxed' or '::strict' also sets the reuse " + "policy (e.g. 'cvmfs::relaxed'); it must agree with --reuse-policy if both " + "are given.")) build_remote.add_argument("--build-local", dest="buildLocal", metavar="PKG[,PKG...]", default="", help=("Comma-separated packages to always build locally even under " "--reuse-policy relaxed (e.g. a package you need patched), rather than " diff --git a/bits_helpers/build.py b/bits_helpers/build.py index e1cd7a49..96b1a079 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -2471,9 +2471,12 @@ def defaultsReader(): # Resolve --reuse-from into an absolute modules-tree path ('cvmfs' -> the # defaults system: layout module_path). Nothing consumes it yet (later step). - from bits_helpers.cvmfs_layout import resolve_reuse_from + from bits_helpers.cvmfs_layout import resolve_reuse_from, split_reuse_policy + # Sugar: a trailing '::relaxed'/'::strict' on --reuse-from sets the reuse + # policy alongside the source (reconciled with --reuse-policy below). + _reuse_src, _reuse_from_policy = split_reuse_policy(getattr(args, "reuseFrom", None)) try: - args.reuseFrom = resolve_reuse_from(getattr(args, "reuseFrom", None), _cvmfs) + args.reuseFrom = resolve_reuse_from(_reuse_src, _cvmfs) except ValueError as exc: dieOnError(True, str(exc)) # Build the local, re-anchored overlay from the deployment named by @@ -2675,8 +2678,17 @@ def _truthy(v): # the two above. Precedence: explicit --reuse-policy/--reuse-base > defaults # system.reuse_policy / reuse_base > strict / none. Default strict keeps the # simple aliBuild case bit-for-bit unchanged. + # An explicit --reuse-policy is canonical; the --reuse-from '::policy' suffix + # is sugar. If both are given they must agree; otherwise the suffix fills in. + # Precedence: --reuse-policy > --reuse-from ::policy > defaults > strict. + if getattr(args, "reusePolicy", None) is not None and _reuse_from_policy \ + and args.reusePolicy != _reuse_from_policy: + dieOnError(True, + "--reuse-policy %s conflicts with --reuse-from ...::%s; " + "specify the policy once." % (args.reusePolicy, _reuse_from_policy)) if getattr(args, "reusePolicy", None) is None: - args.reusePolicy = str(_system_opt("reuse_policy", "strict")).strip().lower() + args.reusePolicy = _reuse_from_policy \ + or str(_system_opt("reuse_policy", "strict")).strip().lower() if args.reusePolicy not in ("strict", "relaxed"): args.reusePolicy = "strict" if getattr(args, "reuseBase", None) is None: diff --git a/bits_helpers/cvmfs_layout.py b/bits_helpers/cvmfs_layout.py index 03b24dc3..c83fabd7 100644 --- a/bits_helpers/cvmfs_layout.py +++ b/bits_helpers/cvmfs_layout.py @@ -42,6 +42,23 @@ def _render(template, subst): return _VAR_RE.sub(lambda m: str(subst.get(m.group(1), m.group(0))), template) +def split_reuse_policy(reuse_from): + """Split an optional trailing ``::`` off a ``--reuse-from`` value. + + Sugar so ``--reuse-from cvmfs::relaxed`` (or ``::relaxed``) can set the + reuse policy alongside the source. Returns ``(source, policy)`` where policy + is ``"strict"``/``"relaxed"`` or ``None`` when no valid suffix is present. + Absolute paths never contain ``::``, so splitting on the last ``::`` is + unambiguous. Pure; the caller reconciles it with an explicit --reuse-policy. + """ + if not reuse_from: + return reuse_from, None + head, sep, tail = reuse_from.rpartition("::") + if sep and tail.strip().lower() in ("strict", "relaxed"): + return head, tail.strip().lower() + return reuse_from, None + + def resolve_reuse_from(reuse_from, layout): """Resolve ``--reuse-from`` to an absolute modules-tree path, or None. diff --git a/tests/test_cvmfs_layout.py b/tests/test_cvmfs_layout.py index 566b4981..33a254c1 100644 --- a/tests/test_cvmfs_layout.py +++ b/tests/test_cvmfs_layout.py @@ -13,7 +13,7 @@ from bits_helpers.cvmfs_layout import resolve_cvmfs_templates as RT from bits_helpers.cvmfs_layout import ( resolve_release, path_release, bake_release, _declared_release, - resolve_reuse_from) + resolve_reuse_from, split_reuse_policy) ARCH = "ubuntu2510_x86-64-gcc15-dbg" @@ -272,5 +272,30 @@ def test_cvmfs_without_layout_fails(self): resolve_reuse_from("cvmfs", {}) # layout present but no module_path +class SplitReusePolicyTest(unittest.TestCase): + + def test_no_suffix(self): + self.assertEqual(split_reuse_policy("cvmfs"), ("cvmfs", None)) + self.assertEqual(split_reuse_policy("/cvmfs/x/modulefiles"), + ("/cvmfs/x/modulefiles", None)) + + def test_none_and_empty(self): + self.assertEqual(split_reuse_policy(None), (None, None)) + self.assertEqual(split_reuse_policy(""), ("", None)) + + def test_policy_suffix(self): + self.assertEqual(split_reuse_policy("cvmfs::relaxed"), ("cvmfs", "relaxed")) + self.assertEqual(split_reuse_policy("cvmfs::strict"), ("cvmfs", "strict")) + self.assertEqual(split_reuse_policy("/cvmfs/x/modulefiles::relaxed"), + ("/cvmfs/x/modulefiles", "relaxed")) + + def test_case_insensitive(self): + self.assertEqual(split_reuse_policy("cvmfs::RELAXED"), ("cvmfs", "relaxed")) + + def test_unknown_suffix_is_not_a_policy(self): + # A non-policy trailing token is left as part of the source, untouched. + self.assertEqual(split_reuse_policy("cvmfs::loose"), ("cvmfs::loose", None)) + + if __name__ == "__main__": unittest.main() From 3f850bd86938d70dadb1886d3b42b79c66a84a65 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 15:13:33 +0200 Subject: [PATCH 20/30] reuse: derive --reuse-from cvmfs module path from cvmfs_modules_template When the defaults declare no module layout (module_dir/cvmfs_dir), --reuse-from cvmfs now derives the modulefiles base from the group's cvmfs_modules_template (one declaration drives publish and reuse). Expanded with the deployed (raw) arch so it matches where packages live, not the build-qualified family. --- bits_helpers/build.py | 15 +++++++++++++-- bits_helpers/cvmfs_layout.py | 27 +++++++++++++++++++++++++-- tests/test_cvmfs_layout.py | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 96b1a079..a44595ab 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -2471,12 +2471,23 @@ def defaultsReader(): # Resolve --reuse-from into an absolute modules-tree path ('cvmfs' -> the # defaults system: layout module_path). Nothing consumes it yet (later step). - from bits_helpers.cvmfs_layout import resolve_reuse_from, split_reuse_policy + from bits_helpers.cvmfs_layout import (resolve_reuse_from, split_reuse_policy, + reuse_module_path_from_templates) # Sugar: a trailing '::relaxed'/'::strict' on --reuse-from sets the reuse # policy alongside the source (reconciled with --reuse-policy below). _reuse_src, _reuse_from_policy = split_reuse_policy(getattr(args, "reuseFrom", None)) + # --reuse-from cvmfs prefers the system: layout module_path; if that is not + # declared, fall back to the group's cvmfs_modules_template (one declaration + # drives both publish and reuse). Expanded with raw_architecture — the DEPLOYED + # arch — not the build-qualified family, so it matches the deployment. + _reuse_layout = _cvmfs + if _reuse_src == "cvmfs" and not (_cvmfs and _cvmfs.get("module_path")): + _mp = reuse_module_path_from_templates( + defaultsMeta, raw_architecture, os.environ.get("BITS_CVMFS_PREFIX") or None) + if _mp: + _reuse_layout = dict(_cvmfs or {}, module_path=_mp) try: - args.reuseFrom = resolve_reuse_from(_reuse_src, _cvmfs) + args.reuseFrom = resolve_reuse_from(_reuse_src, _reuse_layout) except ValueError as exc: dieOnError(True, str(exc)) # Build the local, re-anchored overlay from the deployment named by diff --git a/bits_helpers/cvmfs_layout.py b/bits_helpers/cvmfs_layout.py index c83fabd7..2b7b5a95 100644 --- a/bits_helpers/cvmfs_layout.py +++ b/bits_helpers/cvmfs_layout.py @@ -73,8 +73,9 @@ def resolve_reuse_from(reuse_from, layout): module_path = layout.get("module_path") if layout else None if not module_path: raise ValueError( - "--reuse-from cvmfs needs a modules layout (module_dir/cvmfs_dir) " - "in the defaults system: section; none is configured.") + "--reuse-from cvmfs needs the modules location in the defaults " + "system: section — either a layout (module_dir/cvmfs_dir) or a " + "cvmfs_modules_template; none is configured.") return module_path if not os.path.isabs(reuse_from): raise ValueError( @@ -83,6 +84,28 @@ def resolve_reuse_from(reuse_from, layout): return reuse_from +def reuse_module_path_from_templates(defaults_meta, architecture, injected_prefix=None): + """Derive the modulefiles BASE dir from the group's ``cvmfs_modules_template``. + + Lets ``--reuse-from cvmfs`` work off the single publish template a group + already declares (which also drives publishing), instead of duplicating it as + ``cvmfs_dir`` + ``module_dir``. Expands ``{prefix}`` and ``{platform}`` and + strips the trailing per-package leaf (``…/{pkg}``), yielding the base under + which per-package modulefiles live. ``architecture`` MUST be the DEPLOYED arch + (the raw ``-a`` value / ``abi_tag``), not the build-qualified family, so the + path matches where the packages actually live. Returns None when no modules + template (or no prefix) is configured. Pure. + """ + templates = resolve_cvmfs_templates(defaults_meta, injected_prefix) + if not templates or not templates.get("modules"): + return None + # Drop the trailing "/{token}" run (the per-package leaf, e.g. /{pkg} or + # /{pkg}/{tag}) to get the fixed base the modulefiles live under. + base = re.sub(r"(?:/\{[^}]*\})+$", "", templates["modules"]) + return (base.replace("{prefix}", templates["prefix"]) + .replace("{platform}", architecture)) + + def resolve_cvmfs_layout(defaults_meta, architecture): """Return the resolved CVMFS layout dict, or None when not configured. diff --git a/tests/test_cvmfs_layout.py b/tests/test_cvmfs_layout.py index 33a254c1..fd69d51c 100644 --- a/tests/test_cvmfs_layout.py +++ b/tests/test_cvmfs_layout.py @@ -13,7 +13,7 @@ from bits_helpers.cvmfs_layout import resolve_cvmfs_templates as RT from bits_helpers.cvmfs_layout import ( resolve_release, path_release, bake_release, _declared_release, - resolve_reuse_from, split_reuse_policy) + resolve_reuse_from, split_reuse_policy, reuse_module_path_from_templates) ARCH = "ubuntu2510_x86-64-gcc15-dbg" @@ -297,5 +297,36 @@ def test_unknown_suffix_is_not_a_policy(self): self.assertEqual(split_reuse_policy("cvmfs::loose"), ("cvmfs::loose", None)) +class ReuseModulePathFromTemplatesTest(unittest.TestCase): + + def test_derives_base_from_declared_template(self): + meta = {"system": { + "prefix": "/cvmfs/sft.cern.ch/lcg/bits", + "cvmfs_modules_template": "{prefix}/{platform}/Modules/modulefiles/{pkg}", + }} + # Uses the DEPLOYED (raw) arch and strips the trailing /{pkg}. + self.assertEqual( + reuse_module_path_from_templates(meta, "x86_64-el9-gcc14-opt"), + "/cvmfs/sft.cern.ch/lcg/bits/x86_64-el9-gcc14-opt/Modules/modulefiles") + + def test_default_template_when_only_prefix(self): + # No explicit modules template -> resolve_cvmfs_templates supplies the + # conventional default, which we still reduce to the base. + meta = {"system": {"prefix": "/cvmfs/r"}} + self.assertEqual(reuse_module_path_from_templates(meta, "el9"), + "/cvmfs/r/el9/Modules/modulefiles") + + def test_injected_prefix_wins(self): + meta = {"system": {"cvmfs_modules_template": + "{prefix}/{platform}/Modules/modulefiles/{pkg}"}} + self.assertEqual( + reuse_module_path_from_templates(meta, "el9", injected_prefix="/cvmfs/inj"), + "/cvmfs/inj/el9/Modules/modulefiles") + + def test_none_when_no_prefix_or_template(self): + self.assertIsNone(reuse_module_path_from_templates({}, "el9")) + self.assertIsNone(reuse_module_path_from_templates(None, "el9")) + + if __name__ == "__main__": unittest.main() From f20b9297ee44a4fe146ef5f9e29c4415467525e3 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 15:26:34 +0200 Subject: [PATCH 21/30] reuse: remove legacy cvmfs:// build_id-graft wiring from doBuild (5/1) Retire the ADR-0001 graft path now that --reuse-from module overlay is the proven mechanism. Drops the build_id auto-select, the graftable_match performCvmfsMatch callback, the reuseBase default, and the reuseCvmfs -> remote-store=cvmfs:// wiring. Overlay reuse and the relaxed publish guard are untouched. getPackageList's now-unused callback param goes next. --- bits_helpers/build.py | 92 ++++--------------------------------------- 1 file changed, 7 insertions(+), 85 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index a44595ab..788a6c2c 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -2447,14 +2447,9 @@ def defaultsReader(): # ── CVMFS layout (templated dirs from defaults-release) ──────────────────── # When defaults declare cvmfs_dir / install_dir / module_dir (templates that - # may use %(architecture)s), resolve them and use them to default the build/ - # reuse flags so the whole CVMFS chain can be driven from one declaration: - # * reuse deployed -> --remote-store = cvmfs:// (with --reuse-cvmfs) - # Docker builds are relocatable by default (built in WORK_DIR with padded - # placeholders + relocate-me.sh), so the tarballs can be reused anywhere and are - # relocated into CVMFS on publish. Pass --cvmfs-prefix explicitly only for an - # in-place, non-relocatable build (skips relocation on publish, but the result - # is NOT reusable outside that exact CVMFS path). + # may use %(architecture)s), resolve them so create_provenance_info can record + # the tree paths in each package's .meta.json. Reusing deployed components is + # driven by --reuse-from (module overlay), not the remote/tarball store. from bits_helpers.cvmfs_layout import resolve_cvmfs_layout _cvmfs = resolve_cvmfs_layout(defaultsMeta, args.architecture) # Stash the resolved layout so create_provenance_info can record it in each @@ -2465,9 +2460,6 @@ def defaultsReader(): if _cvmfs: info("CVMFS layout: install=%s modules=%s views=%s", _cvmfs["install_path"], _cvmfs["module_path"], _cvmfs["views_path"]) - if getattr(args, "reuseCvmfs", False) and not args.remoteStore and _cvmfs["cvmfs_dir"]: - args.remoteStore = "cvmfs://" + _cvmfs["cvmfs_dir"] - info("Reusing deployed components: --remote-store %s", args.remoteStore) # Resolve --reuse-from into an absolute modules-tree path ('cvmfs' -> the # defaults system: layout module_path). Nothing consumes it yet (later step). @@ -2685,10 +2677,9 @@ def _truthy(v): args.criticalPathSchedule = _cp if isinstance(_cp, bool) \ else str(_cp).strip().lower() in ("1", "true", "yes", "on") - # Relaxed CVMFS reuse policy (ADR-0001). Non-hashed build-host policy, like - # the two above. Precedence: explicit --reuse-policy/--reuse-base > defaults - # system.reuse_policy / reuse_base > strict / none. Default strict keeps the - # simple aliBuild case bit-for-bit unchanged. + # Reuse policy (strict/relaxed) for the --reuse-from module overlay. Strict + # reuses only on exact content-hash match (publishable); relaxed matches any + # version in the one-release overlay (loose provenance, non-publishable). # An explicit --reuse-policy is canonical; the --reuse-from '::policy' suffix # is sugar. If both are given they must agree; otherwise the suffix fills in. # Precedence: --reuse-policy > --reuse-from ::policy > defaults > strict. @@ -2702,43 +2693,6 @@ def _truthy(v): or str(_system_opt("reuse_policy", "strict")).strip().lower() if args.reusePolicy not in ("strict", "relaxed"): args.reusePolicy = "strict" - if getattr(args, "reuseBase", None) is None: - args.reuseBase = _system_opt("reuse_base", "") or "" - # Relaxed reuse: auto-select a build_id from the CVMFS store when none was - # given (or the sentinels "latest" / "latest-common"), so the user need not - # copy an id by hand. Anchors on the build target for "latest"; requires all - # requested packages to share it for "latest-common". Announced loudly; an - # explicit --reuse-base always wins. Uses the (combined) architecture, which - # must match the deployed /Packages dir name. - if args.reusePolicy == "relaxed" \ - and str(args.reuseBase).strip().lower() in ("", "latest", "latest-common"): - _strategy = "latest-common" if str(args.reuseBase).strip().lower() == "latest-common" \ - else "latest" - _store = args.remoteStore or "" - # These warnings describe the legacy build_id graft path only. Silence them - # when --reuse-from overlay reuse is active: that path grafts on its own and - # the "packages will be built" message would contradict it. - _overlay_active = bool(getattr(args, "reuseOverlay", None)) - if not _store.startswith("cvmfs://"): - if not _overlay_active: - warning("relaxed reuse: auto-select needs a cvmfs:// --remote-store; " - "no build_id selected, packages will be built.") - args.reuseBase = "" - else: - from bits_helpers.cvmfs_reuse import select_build_id - _root = re.sub("^cvmfs://", "", _store) - _bid, _cov = select_build_id(packages, args.architecture, _root, _strategy) - if _bid: - banner("relaxed reuse: auto-selected build_id '%s' (%s)\n" - " from %s/%s/Packages, covering %d/%d requested package(s)", - _bid, _strategy, _root, args.architecture, - len(_cov.get(_bid, ())), len(packages)) - args.reuseBase = _bid - else: - if not _overlay_active: - warning("relaxed reuse: no %s build_id found under %s/%s/Packages; " - "packages will be built.", _strategy, _root, args.architecture) - args.reuseBase = "" # Publish guard: relaxed builds are loose-provenance (their closure includes # unverified deployed binaries) and must never reach a write store / publish # pipeline. Refuse early and clearly. @@ -2872,37 +2826,6 @@ def performPreferCheckWithTempDir(pkg, cmd): with tempfile.TemporaryDirectory(prefix=f"bits_prefer_check_{pkg['package']}_") as temp_dir: return getstatusoutput_docker(cmd, cwd=temp_dir) - # Relaxed CVMFS graft callback (ADR-0001). Active only under --reuse-policy - # relaxed with a cvmfs:// remote store and a --reuse-base build_id; None in - # every other case → strict behaviour, no graft (simple aliBuild path - # unaffected). Uses the combined architecture (args.architecture) — the arch - # recorded in the deployed packages' .meta.json — not raw_architecture. - _cvmfs_match = None - if getattr(args, "reusePolicy", "strict") == "relaxed": - _base = getattr(args, "reuseBase", "") or "" - _store = args.remoteStore or "" - # Silence these legacy-graft warnings when --reuse-from overlay reuse is - # active: the overlay grafts independently, so "no packages will be - # grafted" would be misleading. - _overlay_active = bool(getattr(args, "reuseOverlay", None)) - if not _base: - if not _overlay_active: - warning("--reuse-policy relaxed needs --reuse-base (or defaults " - "reuse_base:); no packages will be grafted.") - elif not _store.startswith("cvmfs://"): - if not _overlay_active: - warning("--reuse-policy relaxed needs a cvmfs:// --remote-store " - "(or --reuse-cvmfs); no packages will be grafted.") - else: - from bits_helpers.cvmfs_reuse import graftable_match - _store_root = re.sub("^cvmfs://", "", _store) - _build_local = set(getattr(args, "buildLocal", []) or []) - def _cvmfs_match(spec, _root=_store_root, _bid=_base, - _arch=args.architecture, _bl=_build_local): - if spec["package"] in _bl: - return None - return graftable_match(spec["package"], _arch, _bid, _root) - systemPackages, ownPackages, failed, validDefaults = \ getPackageList(packages = packages, specs = specs, @@ -2920,8 +2843,7 @@ def _cvmfs_match(spec, _root=_store_root, _bid=_base, taps = taps, log = debug, provider_dirs = provider_dirs, - defaults_meta = defaultsMeta, - performCvmfsMatch = _cvmfs_match) + defaults_meta = defaultsMeta) _bad_defaults, _missing_flavor = incompatibleFlavorDefaults(validDefaults, args.defaults, defaultsMeta) dieOnError(bool(_bad_defaults) or _missing_flavor, From 1d3ac136b05a85eac139ea471807cc681bc027d2 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 15:41:46 +0200 Subject: [PATCH 22/30] reuse: remove legacy cvmfs:// build_id-graft resolver (5/2) Drop the performCvmfsMatch graft branch from getPackageList, the from_cvmfs hash-adoption in storeHashes, and the from_cvmfs closure-loose provenance check (nothing sets from_cvmfs after 5/1). Legacy graft tests removed (test_reuse_resolver stubbed pending git rm); provenance loose is now driven solely by untracked_requires. --- bits_helpers/build.py | 40 ++---------- bits_helpers/utilities.py | 32 +-------- tests/test_provenance.py | 17 +++-- tests/test_reuse_resolver.py | 121 ++--------------------------------- 4 files changed, 21 insertions(+), 189 deletions(-) diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 788a6c2c..2d9226a3 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -819,24 +819,6 @@ def storeHashes(package, specs, considerRelocation): # subsequent calculations. return - # Relaxed CVMFS graft (ADR-0001): a grafted package adopts the *deployed* - # artifact's hash. The existing reuse path (CVMFSRemoteSync.fetch_symlinks + - # the reuse decision) then materialises and symlinks the deployed tree under - # that hash instead of building, and consumers hash against the real deployed - # dependency — so no separate build-skip branch is needed. Only triggers when - # the resolver tagged the spec from_cvmfs (relaxed mode); never in strict. - if spec.get("from_cvmfs") and spec.get("cvmfs_hash"): - _h = spec["cvmfs_hash"] - spec["remote_revision_hash"] = _h - spec["local_revision_hash"] = _h - spec["remote_hashes"] = [_h] - spec["local_hashes"] = [_h] - spec["hash"] = _h - # The grafted package has no followed dependencies; set deps_hash too (the - # normal path always sets it, and DEPS_HASH is read via spec.get downstream). - spec.setdefault("deps_hash", "") - return - # For now, all the hashers share data -- they'll be split below. h_all = Hasher() @@ -1509,21 +1491,11 @@ def dependency_list(key): from bits_helpers.provenance import ( compute_build_id, compute_abi_tag, recipe_tools_ref, ) - # Contagious provenance (ADR-0001): a locally-built package is "loose" when its - # dependency closure contains a package grafted from CVMFS (adopted by - # name/build_id, not verified hash). Grafted packages are not built, so this - # function only ever runs for local builds. - def _closure_grafted(): - for _key in ("full_build_requires", "full_runtime_requires"): - for _dep in specs[package].get(_key, ()): - _ds = specs.get(_dep) - if isinstance(_ds, dict) and _ds.get("from_cvmfs"): - return True - return False - # A build is also loose if its closure decoupled a dependency via - # untracked_requires: this package, or one below it, was hashed as if that - # dependency never changed, so its identity no longer certifies its full input - # closure. Contagious upward like grafted provenance. + # Contagious provenance: a build is "loose" when its closure decoupled a + # dependency via untracked_requires — this package, or one below it, was hashed + # as if that dependency never changed, so its identity no longer certifies its + # full input closure. (Relaxed --reuse-from builds are kept non-publishable by + # the reuse-policy publish guard, so they never reach a published record.) def _closure_untracked(): if specs[package].get("untracked_requires"): return True @@ -1534,7 +1506,7 @@ def _closure_untracked(): return True return False _untracked = list(specs[package].get("untracked_requires", ())) - _provenance = "loose" if (_closure_grafted() or _closure_untracked()) else "pure" + _provenance = "loose" if _closure_untracked() else "pure" return json.dumps({ "comment": args.annotate.get(package), "bits_version": __version__, diff --git a/bits_helpers/utilities.py b/bits_helpers/utilities.py index 9d1082f1..40e5d689 100644 --- a/bits_helpers/utilities.py +++ b/bits_helpers/utilities.py @@ -1591,7 +1591,7 @@ def recipeSourceLabel(pkgdir, provider_dirs=None): def getPackageList(packages, specs, configDir, preferSystem, noSystem, architecture, disable, defaults, performPreferCheck, performRequirementCheck, performValidateDefaults, overrides, taps, log, force_rebuild=(), - provider_dirs=None, defaults_meta=None, performCvmfsMatch=None): + provider_dirs=None, defaults_meta=None): """Resolve the full set of packages required by *packages*. *provider_dirs* is an optional ``dict`` returned by @@ -1953,36 +1953,6 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, # the legacy install layout //-. spec["pkg_family"] = resolve_pkg_family(defaults_meta or {}, spec["package"]) - # Relaxed CVMFS graft (ADR-0001). performCvmfsMatch is wired by build.py only - # under --reuse-policy relaxed (None otherwise → no effect, strict unchanged). - # It returns a descriptor for a package already deployed in the blessed - # release, or None. On a match we keep the spec in `specs` so consumers still - # source its deployed init.sh, but PRUNE its subtree — the grafted package's - # own dependencies are provided from /cvmfs via that init.sh cascade — and - # tag it (from_cvmfs/cvmfs_*) so the build step symlinks it instead of - # compiling (Stage 1c). The deployed hash is recorded for the hashing step. - if performCvmfsMatch is not None: - _m = performCvmfsMatch(spec) - if _m: - spec["from_cvmfs"] = True - spec["cvmfs_path"] = _m.get("path") - spec["cvmfs_build_id"] = _m.get("build_id") - if _m.get("hash"): - spec["cvmfs_hash"] = _m["hash"] - # Adopt the DEPLOYED version/revision (base versions win, ADR-0001) so the - # install path and the consumer's reuse-decision regex both match the - # deployed tarball. version/revision come from the deployed .meta.json. - if _m.get("version"): - spec["version"] = _m["version"] - spec["tag"] = _m["version"] - if _m.get("revision") is not None: - spec["revision"] = _m["revision"] - spec["requires"] = [] - spec["build_requires"] = [] - spec["runtime_requires"] = [] - specs[spec["package"]] = spec - continue - specs[spec["package"]] = spec packages += spec["requires"] diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 8fa4e964..448ddb39 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -131,19 +131,22 @@ def _record_specs(self, specs): finally: os.environ.pop("BITS_DIST_HASH", None) - def test_provenance_loose_when_closure_has_graft(self): + def test_provenance_pure_when_closure_is_clean(self): + # No untracked_requires anywhere in the closure -> pure. (The legacy + # cvmfs:// graft that also produced "loose" was removed in Step 5.) specs = { "a": _spec("a", full_runtime_requires=["dep"], full_build_requires=[]), - "dep": _spec("dep", from_cvmfs=True), + "dep": _spec("dep"), # locally built } - self.assertEqual(self._record_specs(specs)["provenance"], "loose") + self.assertEqual(self._record_specs(specs)["provenance"], "pure") - def test_provenance_pure_when_no_graft_in_closure(self): + def test_provenance_loose_when_closure_has_untracked(self): + # untracked_requires decouples a dependency from the hash -> loose. specs = { - "a": _spec("a", full_runtime_requires=["dep"], full_build_requires=[]), - "dep": _spec("dep"), # locally built, not grafted + "a": _spec("a", untracked_requires=["dep"]), + "dep": _spec("dep"), } - self.assertEqual(self._record_specs(specs)["provenance"], "pure") + self.assertEqual(self._record_specs(specs)["provenance"], "loose") class TestBuildIdFromManifest(unittest.TestCase): diff --git a/tests/test_reuse_resolver.py b/tests/test_reuse_resolver.py index 007a994e..7ec495c0 100644 --- a/tests/test_reuse_resolver.py +++ b/tests/test_reuse_resolver.py @@ -1,120 +1,7 @@ -#!/usr/bin/env python3 # SPDX-FileCopyrightText: 2015-2026 CERN # SPDX-License-Identifier: GPL-3.0-or-later -"""Tests for the relaxed-reuse frontier-cut in getPackageList (ADR-0001 Stage 1b). - -A `performCvmfsMatch` callback that returns a deployed-package descriptor causes -that package to be grafted: kept in `specs` (so consumers still depend on it and -source its deployed init.sh) but with its dependency subtree pruned and tagged -`from_cvmfs`. With the default callback (None) the resolver is unchanged. -""" -import os -import sys -import unittest -from unittest.mock import patch - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - -from bits_helpers.utilities import getPackageList - - -RECIPES = { - "myapp": "package: myapp\nversion: v1\nrequires:\n - mydep\n---\n", - "mydep": "package: mydep\nversion: v1\nrequires:\n - mysubdep\n---\n", - "mysubdep": "package: mysubdep\nversion: v1\n---\n", - "defaults-release": "package: defaults-release\nversion: v1\n---\n", -} - - -def _resolve(perform_cvmfs_match): - specs = {} - - def fake_resolveFilename(taps, pkg, configDir, genPkgs): - return (pkg + ".sh", "/pkgdir") - - def fake_getRecipeReader(filename, *a, **k): - pkg = filename.replace(".sh", "") - content = RECIPES.get(pkg, "package: {p}\nversion: v1\n---\n".format(p=pkg)) - return lambda: content - - with patch("bits_helpers.utilities.resolveFilename", side_effect=fake_resolveFilename), \ - patch("bits_helpers.utilities.getRecipeReader", side_effect=fake_getRecipeReader), \ - patch("bits_helpers.utilities.getGeneratedPackages", return_value={"/pkgdir": {}}), \ - patch("bits_helpers.utilities.load_for_spec", return_value=None), \ - patch("bits_helpers.utilities.merge_into_spec", return_value=None): - getPackageList( - packages=["myapp"], specs=specs, configDir="/fake", - preferSystem=False, noSystem="*", architecture="slc9_x86-64", - disable=[], defaults=["release"], - performPreferCheck=lambda pkg, cmd: (1, ""), - performRequirementCheck=lambda pkg, cmd: (0, ""), - performValidateDefaults=lambda spec: (True, "", None), - overrides={"defaults-release": {}}, taps={}, - log=lambda *a, **k: None, defaults_meta=None, - performCvmfsMatch=perform_cvmfs_match, - ) - return specs - - -class TestRelaxedFrontierCut(unittest.TestCase): - - def test_no_callback_is_unchanged(self): - specs = _resolve(None) - for p in ("myapp", "mydep", "mysubdep"): - self.assertIn(p, specs) - self.assertNotIn("from_cvmfs", specs[p]) - - def test_graft_prunes_subtree_but_keeps_consumer_dep(self): - # Graft 'mydep' from a blessed release: it stays in specs, its subtree - # ('mysubdep') is pruned, and the consumer ('myapp') still depends on it. - def match(spec): - if spec["package"] == "mydep": - return {"path": "/cvmfs/rel/mydep/v1", "build_id": "LCG_109-x", - "hash": "deadbeef", "version": "v1"} - return None - - specs = _resolve(match) - self.assertIn("mydep", specs) - self.assertTrue(specs["mydep"]["from_cvmfs"]) - self.assertEqual(specs["mydep"]["cvmfs_path"], "/cvmfs/rel/mydep/v1") - self.assertEqual(specs["mydep"]["cvmfs_hash"], "deadbeef") - self.assertEqual(specs["mydep"]["requires"], []) - # subtree pruned - self.assertNotIn("mysubdep", specs) - # consumer still present and still depends on the grafted package - self.assertIn("myapp", specs) - self.assertNotIn("from_cvmfs", specs["myapp"]) - self.assertIn("mydep", specs["myapp"]["requires"]) - - def test_no_match_builds_everything(self): - specs = _resolve(lambda spec: None) - for p in ("myapp", "mydep", "mysubdep"): - self.assertIn(p, specs) - self.assertNotIn("from_cvmfs", specs[p]) - - -class TestStoreHashesGraft(unittest.TestCase): - """A from_cvmfs spec adopts the deployed hash so the existing reuse path - symlinks it instead of building (ADR-0001 Stage 1c).""" - - def test_from_cvmfs_adopts_deployed_hash(self): - from bits_helpers.build import storeHashes - specs = { - "ROOT": { - "package": "ROOT", "version": "v1", "revision": "1", - "from_cvmfs": True, "cvmfs_hash": "deadbeef", - "requires": [], "pkg_family": "", - }, - } - storeHashes("ROOT", specs, considerRelocation=False) - s = specs["ROOT"] - self.assertEqual(s["hash"], "deadbeef") - self.assertEqual(s["remote_revision_hash"], "deadbeef") - self.assertEqual(s["local_revision_hash"], "deadbeef") - self.assertEqual(s["remote_hashes"], ["deadbeef"]) - self.assertEqual(s["local_hashes"], ["deadbeef"]) - - -if __name__ == "__main__": - unittest.main() +# Legacy ADR-0001 cvmfs:// build_id-graft tests removed with the graft path +# (Step 5 cleanup). Reuse is now driven by --reuse-from (module overlay), whose +# tests live in test_cvmfs_import.py / test_build.py / test_cvmfs_layout.py. +# This file is scheduled for `git rm`. From d52670017aad57870941c9293118d2b68c5e3af5 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 15:48:05 +0200 Subject: [PATCH 23/30] reuse: remove legacy CVMFSRemoteSync; reject cvmfs:// --remote-store (5/3) The cvmfs:// remote-store reader (synthesized-tarball reuse) is retired. remote_from_url now errors on a cvmfs:// --remote-store, pointing at --reuse-from; the http+write dual path and all other backends are unchanged. Legacy CVMFSRemoteSync tests removed (fetch_symlinks stub pending git rm). --- bits_helpers/sync.py | 126 +++-------------------------- tests/test_async_build.py | 15 ---- tests/test_cvmfs_fetch_symlinks.py | 58 +------------ tests/test_source_cache.py | 49 +---------- tests/test_sync.py | 20 ++--- 5 files changed, 26 insertions(+), 242 deletions(-) diff --git a/bits_helpers/sync.py b/bits_helpers/sync.py index 7d8898f6..ccf2db47 100644 --- a/bits_helpers/sync.py +++ b/bits_helpers/sync.py @@ -259,11 +259,8 @@ def remote_from_url(read_url, write_url, architecture, work_dir, insecure=False, # different backend, so pair the read-only reader with a writer built from # write_url (see DualRemoteSync). Without this, the write store was silently # dropped below and nothing was ever uploaded. - if write_url and (read_url.startswith("cvmfs://") or read_url.startswith("http")): - if read_url.startswith("http"): - reader = HttpRemoteSync(read_url, architecture, work_dir, insecure) - else: - reader = CVMFSRemoteSync(read_url, None, architecture, work_dir) + if write_url and read_url.startswith("http"): + reader = HttpRemoteSync(read_url, architecture, work_dir, insecure) return DualRemoteSync(reader, _writer_from_url(write_url, architecture, work_dir)) if read_url.startswith("http"): @@ -273,7 +270,11 @@ def remote_from_url(read_url, write_url, architecture, work_dir, insecure=False, if read_url.startswith("b3://"): return Boto3RemoteSync(read_url, write_url, architecture, work_dir) if read_url.startswith("cvmfs://"): - return CVMFSRemoteSync(read_url, None, architecture, work_dir) + dieOnError(True, + "--remote-store cvmfs:// is no longer supported. --remote-store " + "is the tarball store (s3://, b3://, rsync:, http). To reuse " + "components already deployed on CVMFS, use " + "--reuse-from |cvmfs instead.") if read_url: return RsyncRemoteSync(read_url, write_url, architecture, work_dir) return NoRemoteSync() @@ -339,10 +340,10 @@ def _first_supporting(self, method, default, *args, **kwargs): """Call *method* on the first backend that implements it, reader before writer. Store metadata (ADR-0005 rev-index markers, content-object listings) lives in - the STORE, and a read-only reader (HttpRemoteSync, CVMFSRemoteSync) may not + the STORE, and a read-only reader (e.g. HttpRemoteSync) may not implement the lookup at all. In the common ``--remote-store https://…::rw`` - setup both sides are the very same bucket (http read URL + b3 writer); in the - cvmfs-read/s3-write setup the metadata only ever exists on the writer. + setup both sides are the very same bucket (http read URL + b3 writer); in an + http-read/s3-write setup the metadata only ever exists on the writer. Delegating to the reader alone silently returned the empty default, so the revision counter never saw the markers: it could not reuse the recorded @@ -820,113 +821,6 @@ def upload_source(self, local_path, url_checksum, filename) -> None: dieOnError(err, "Unable to upload source archive to store.") -class CVMFSRemoteSync: - """ Sync packages build directory from CVMFS or similar - FS based deployment. The tarball will be created on the fly with a single - symlink to the remote store in it, so that unpacking really - means unpacking the symlink to the wanted package. - """ - - def __init__(self, remoteStore, writeStore, architecture, workdir) -> None: - self.remoteStore = re.sub("^cvmfs://", "", remoteStore) - # We do not support uploading directly to CVMFS, for obvious - # reasons. - assert(writeStore is None) - self.writeStore = None - self.architecture = architecture - self.workdir = workdir - - def fetch_tarball(self, spec) -> None: - arch = effective_arch(spec, self.architecture) - info("Downloading tarball for %s@%s-%s, if available", spec["package"], spec["version"], spec["revision"]) - # If we already have a tarball with any equivalent hash, don't check S3. - for pkg_hash in spec["remote_hashes"] + spec["local_hashes"]: - store_path = resolve_store_path(arch, pkg_hash) - pattern = os.path.join(self.workdir, store_path, "%s-*.tar.gz" % spec["package"]) - # Use os.path.isfile() to skip dangling symlinks that glob would otherwise return. - if any(os.path.isfile(t) for t in glob.glob(pattern)): - info("Reusing existing tarball for %s@%s", spec["package"], pkg_hash) - return - info("Could not find prebuilt tarball for %s@%s-%s, will be rebuilt", - spec["package"], spec["version"], spec["revision"]) - - def fetch_symlinks(self, spec) -> None: - # When using CVMFS, we create the symlinks grass by reading the . - info("Fetching available build hashes for %s, from %s", spec["package"], self.remoteStore) - arch = effective_arch(spec, self.architecture) - links_path = resolve_links_path(arch, spec["package"]) - os.makedirs(os.path.join(self.workdir, links_path), exist_ok=True) - - cvmfs_architecture = re.sub(r"slc(\d+)_x86-64", r"el\1-x86_64", self.architecture) - err = execute(r"""\ - set -x - # Exit without error in case we do not have any package published - test -d "{remote_store}/{cvmfs_architecture}/Packages/{package}" || exit 0 - mkdir -p "{workDir}/{links_path}" - for install_path in $(find "{remote_store}/{cvmfs_architecture}/Packages/{package}" -mindepth 1 -maxdepth 1 -type d); do - full_version="${{install_path##*/}}" - tarball={package}-$full_version.{architecture}.tar.gz - pkg_hash=$(cat "${{install_path}}/.build-hash" || jq -r '.package.hash' <${{install_path}}/.meta.json) - if [ "X$pkg_hash" = X ]; then - continue - fi - # POSIX 2-char prefix: this script runs under /bin/sh (dash), where a - # bash substring expansion would give "Bad substitution". - pref=$(printf %s "$pkg_hash" | cut -c1-2) - ln -sf ../../{architecture}/store/$pref/$pkg_hash/$tarball "{workDir}/{links_path}/$tarball" - # Create the dummy tarball, if it does not exists - test -f "{workDir}/{architecture}/store/$pref/$pkg_hash/$tarball" && continue - # Build the tree the reuse-unpack expects: ///... (the - # $full_version dir is the version-revision segment). Matches PKGPATH for - # the family-less deployed layout (Packages// has no family). - pkgroot="{workDir}/INSTALLROOT/$pkg_hash/{architecture}/{package}/$full_version" - mkdir -p "$pkgroot" - find "{remote_store}/{cvmfs_architecture}/Packages/{package}/$full_version" -mindepth 1 -maxdepth 1 ! -name etc -exec ln -sf {{}} "$pkgroot/" \; - cp -fr "{remote_store}/{cvmfs_architecture}/Packages/{package}/$full_version/etc" "$pkgroot/etc" - mkdir -p "{workDir}/TARS/{architecture}/store/$pref/$pkg_hash" - tar -C "{workDir}/INSTALLROOT/$pkg_hash" -czf "{workDir}/TARS/{architecture}/store/$pref/$pkg_hash/$tarball" . - rm -rf "{workDir}/INSTALLROOT/$pkg_hash" - done - """.format( - workDir=self.workdir, - architecture=arch, - cvmfs_architecture=cvmfs_architecture, - package=spec["package"], - remote_store=self.remoteStore, - links_path=links_path, - )) - print(f"fetch_symlink: maybe something wrong? {err}") - - def upload_symlinks_and_tarball(self, spec) -> None: - dieOnError(True, "CVMFS backend does not support uploading directly") - - def upload_shell_command(self, spec): - """Return None: CVMFS backend is read-only.""" - return None - - def fetch_source(self, url_checksum, filename, dest_dir) -> bool: - """Try to fetch a source archive from the CVMFS filesystem mount. - - The CVMFS remote store is a read-only filesystem path; we attempt a - plain file copy from the mirrored SOURCES/cache subtree. - """ - remote_path = os.path.join(self.remoteStore, - _source_remote_path(url_checksum, filename)) - dest = os.path.join(dest_dir, filename) - if not os.path.exists(remote_path): - return False - os.makedirs(dest_dir, exist_ok=True) - import shutil - try: - shutil.copy2(remote_path, dest) - return True - except OSError: - return False - - def upload_source(self, local_path, url_checksum, filename) -> None: - pass # CVMFS backend does not support uploading directly - - class S3RemoteSync: """Sync package build directory from and to S3 using s3cmd. diff --git a/tests/test_async_build.py b/tests/test_async_build.py index 122f18b3..16c22e3c 100644 --- a/tests/test_async_build.py +++ b/tests/test_async_build.py @@ -60,21 +60,6 @@ def test_returns_none(self): self.assertIsNone(sync.upload_shell_command(GOOD_SPEC)) -class CVMFSRemoteSyncUploadCmdTest(unittest.TestCase): - """CVMFSRemoteSync is read-only — upload_shell_command returns None.""" - - def test_returns_none(self): - from bits_helpers.sync import CVMFSRemoteSync - # CVMFSRemoteSync asserts writeStore is None (no write support). - sync = CVMFSRemoteSync( - remoteStore="cvmfs://repo", - writeStore=None, - architecture=ARCH, - workdir=WORKDIR, - ) - self.assertIsNone(sync.upload_shell_command(GOOD_SPEC)) - - class RsyncRemoteSyncUploadCmdTest(unittest.TestCase): """RsyncRemoteSync returns None without write store, shell cmd with one.""" diff --git a/tests/test_cvmfs_fetch_symlinks.py b/tests/test_cvmfs_fetch_symlinks.py index e43c6341..b7131183 100644 --- a/tests/test_cvmfs_fetch_symlinks.py +++ b/tests/test_cvmfs_fetch_symlinks.py @@ -1,57 +1,7 @@ # SPDX-FileCopyrightText: 2015-2026 CERN # SPDX-License-Identifier: GPL-3.0-or-later -""" -Integration test for CVMFSRemoteSync.fetch_symlinks tarball synthesis. - -The reuse-unpack in build_template.sh does `mv $TMP/$PKGHASH/$PKGPATH ...` -where PKGPATH = //, so the synthesized dummy tarball MUST -carry that version-revision directory level. Runs the real shell (needs -jq/tar/find), so it is skipped where those are absent. -""" - -import json -import os -import shutil -import tarfile -import tempfile -import unittest - -from bits_helpers.sync import CVMFSRemoteSync - -_TOOLS = all(shutil.which(t) for t in ("jq", "tar", "find")) -ARCH = "x86_64-el9-gcc15-opt" # the slc->el rewrite does not touch this name -HASH = "26681160ad2eec00361f7df05dd5a94f6cbdf9e6" - - -@unittest.skipUnless(_TOOLS, "needs jq/tar/find") -class FetchSymlinksLayoutTest(unittest.TestCase): - - def setUp(self): - self.root = tempfile.mkdtemp() - self.work = tempfile.mkdtemp() - pkg = os.path.join(self.root, ARCH, "Packages", "CMake", "3.30.6-1") - os.makedirs(os.path.join(pkg, "bin")) - os.makedirs(os.path.join(pkg, "etc")) - open(os.path.join(pkg, "bin", "cmake"), "w").close() - with open(os.path.join(pkg, ".meta.json"), "w") as fh: - json.dump({"package": {"hash": HASH, "version": "3.30.6", - "revision": "1"}}, fh) - - def test_tarball_has_version_revision_level(self): - sync = CVMFSRemoteSync("cvmfs://" + self.root, None, ARCH, self.work) - sync.fetch_symlinks({"package": "CMake", "version": "3.30.6", - "revision": "1", "remote_hashes": [HASH], - "local_hashes": []}) - tb = os.path.join(self.work, "TARS", ARCH, "store", HASH[:2], HASH, - "CMake-3.30.6-1.%s.tar.gz" % ARCH) - self.assertTrue(os.path.isfile(tb), "synthesized tarball missing") - names = set(tarfile.open(tb).getnames()) - # PKGPATH tail that build_template.sh's `mv` will look for. - verrev = "./%s/CMake/3.30.6-1" % ARCH - self.assertIn(verrev, names) - self.assertIn(verrev + "/bin", names) - - -if __name__ == "__main__": - unittest.main() +# Tests for CVMFSRemoteSync.fetch_symlinks, removed with the legacy cvmfs:// +# remote-store reader (Step 5 cleanup). Deployed reuse is now driven by +# --reuse-from (module overlay); see test_cvmfs_import.py / test_build.py. +# This file is scheduled for `git rm`. diff --git a/tests/test_source_cache.py b/tests/test_source_cache.py index a741afe8..1bc09e88 100644 --- a/tests/test_source_cache.py +++ b/tests/test_source_cache.py @@ -7,7 +7,7 @@ * ``_source_remote_path()`` — canonical remote path helper * ``NoRemoteSync``, ``HttpRemoteSync``, ``RsyncRemoteSync``, - ``S3RemoteSync``, ``Boto3RemoteSync``, ``CVMFSRemoteSync`` — + ``S3RemoteSync``, ``Boto3RemoteSync`` — ``fetch_source()`` / ``upload_source()`` methods * ``download()`` — ``sync_helper`` integration (local-cache hit, remote-store hit, upstream download with subsequent archive upload) @@ -369,53 +369,6 @@ def test_upload_skipped_with_no_write_store(self): syncer.s3.upload_file.assert_not_called() -# --------------------------------------------------------------------------- -# CVMFSRemoteSync -# --------------------------------------------------------------------------- - -class CVMFSRemoteSyncSourceTest(unittest.TestCase): - """CVMFSRemoteSync.fetch_source reads from the filesystem mount.""" - - def _make_syncer(self, remote_path): - return sync.CVMFSRemoteSync( - remoteStore="cvmfs://{}".format(remote_path), - writeStore=None, - architecture="slc9_x86-64", - workdir="/sw", - ) - - def test_fetch_success(self): - with tempfile.TemporaryDirectory() as tmp: - # Lay out the file at the expected remote filesystem path. - remote_file = os.path.join( - tmp, _source_remote_path(TEST_URL_HASH, TEST_FILENAME), - ) - _write_fake_file(remote_file) - - syncer = self._make_syncer(tmp) - dest_dir = os.path.join(tmp, "dest") - result = syncer.fetch_source(TEST_URL_HASH, TEST_FILENAME, dest_dir) - - self.assertTrue(result) - dest_file = os.path.join(dest_dir, TEST_FILENAME) - self.assertTrue(os.path.isfile(dest_file)) - with open(dest_file, "rb") as fh: - self.assertEqual(fh.read(), _FAKE_CONTENT) - - def test_fetch_miss(self): - with tempfile.TemporaryDirectory() as tmp: - syncer = self._make_syncer(tmp) # remote dir empty - result = syncer.fetch_source(TEST_URL_HASH, TEST_FILENAME, - os.path.join(tmp, "dest")) - self.assertFalse(result) - - def test_upload_is_noop(self): - with tempfile.TemporaryDirectory() as tmp: - syncer = self._make_syncer(tmp) - # Must not raise even though CVMFS is read-only. - syncer.upload_source("/tmp/libfoo.tar.gz", TEST_URL_HASH, TEST_FILENAME) - - # --------------------------------------------------------------------------- # download() — sync_helper integration # --------------------------------------------------------------------------- diff --git a/tests/test_sync.py b/tests/test_sync.py index e006c67c..206ef3bd 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -479,32 +479,34 @@ def test_tarball_upload_spec_with_architecture_key(self) -> None: @patch("bits_helpers.sync.Boto3RemoteSync._s3_init", new=MagicMock()) class DualRemoteSyncTestCase(unittest.TestCase): - """Cross-backend stores: recall from CVMFS, upload freshly-built to S3.""" + """Cross-backend stores: recall from an HTTP mirror, upload freshly-built to S3/B3.""" - READ = "cvmfs:///cvmfs/sft-nightlies-test.cern.ch/lcg/bits/" + READ = "http://mirror.example/tars/" WRITE = "b3://bucketofpieces" # ── remote_from_url dispatch ───────────────────────────────────────────── - def test_cvmfs_read_plus_write_returns_dual(self): + def test_http_read_plus_write_returns_dual(self): helper = sync.remote_from_url(self.READ, self.WRITE, ARCHITECTURE, "/work") self.assertIsInstance(helper, sync.DualRemoteSync) - self.assertIsInstance(helper.reader, sync.CVMFSRemoteSync) + self.assertIsInstance(helper.reader, sync.HttpRemoteSync) self.assertIsInstance(helper.writer, sync.Boto3RemoteSync) # Writer targets the write store for both its read-back and its uploads. self.assertEqual(helper.writer.writeStore, "bucketofpieces") self.assertEqual(helper.writer.remoteStore, "bucketofpieces") - def test_cvmfs_read_without_write_is_unchanged(self): - # No write store -> the old read-only CVMFS helper, not a Dual. - helper = sync.remote_from_url(self.READ, "", ARCHITECTURE, "/work") - self.assertIsInstance(helper, sync.CVMFSRemoteSync) - def test_same_backend_is_unchanged(self): helper = sync.remote_from_url("b3://bucket", "b3://bucket", ARCHITECTURE, "/work") self.assertIsInstance(helper, sync.Boto3RemoteSync) + @patch("bits_helpers.sync.error", new=MagicMock()) + def test_cvmfs_read_store_is_rejected(self): + # cvmfs:// --remote-store is retired; deployed reuse is via --reuse-from. + with self.assertRaises(SystemExit): + sync.remote_from_url("cvmfs:///cvmfs/x/", "", ARCHITECTURE, "/work") + @patch("bits_helpers.sync.error", new=MagicMock()) def test_cvmfs_write_target_is_rejected(self): + # cvmfs is read-only: reject it as a --write-store (via the dual path). with self.assertRaises(SystemExit): sync.remote_from_url(self.READ, "cvmfs:///somewhere", ARCHITECTURE, "/work") From e1f96d7c2a99a45e65b511b51ccbc80b8424a0db Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 15:54:47 +0200 Subject: [PATCH 24/30] reuse: drop --reuse-base/--reuse-cvmfs and orphaned cvmfs_reuse (5/4) The build_id-graft flags and the select_build_id/graftable_match helpers they drove have no consumers after 5/1-5/3. Remove the flags, empty cvmfs_reuse.py (stub pending git rm), retarget the bits-import help at --reuse-from, and prune the legacy arg tests. --- bits_helpers/args.py | 15 +--- bits_helpers/cvmfs_reuse.py | 145 ++---------------------------------- tests/test_args.py | 4 +- tests/test_cvmfs_reuse.py | 143 +---------------------------------- 4 files changed, 12 insertions(+), 295 deletions(-) diff --git a/bits_helpers/args.py b/bits_helpers/args.py index 02296a52..a8169f23 100644 --- a/bits_helpers/args.py +++ b/bits_helpers/args.py @@ -380,8 +380,8 @@ def doParseArgs(): "manifest), closure-check the set, stamp it with one deterministic " "build_id, and generate a per-build_id overlay (build-sufficient bits " "modulefiles + module-side .meta.json + .cvmfscatalog) that " - "'bits build --reuse-policy relaxed --reuse-base ' can graft " - "without recompiling. See ADR-0001." + "'bits build --reuse-from |cvmfs' can reuse without " + "recompiling." ), ) import_parser.add_argument("-w", "--work-dir", dest="workDir", @@ -682,10 +682,6 @@ def doParseArgs(): is set to the same value). May be set to a default store on some architectures; use --no-remote-store to disable it in that case. """) - build_remote.add_argument("--reuse-cvmfs", dest="reuseCvmfs", action="store_true", - help=("Reuse already-deployed components from the CVMFS area declared by the " - "defaults `cvmfs_dir:` field. Sets --remote-store to cvmfs:// " - "when no remote store is given.")) build_remote.add_argument("--sign-manifest", dest="signManifest", default=None, metavar="KEY.pem", help=("After the build, sign the build manifest with this Ed25519 private key " "(PEM) so consumers can verify it for trusted reuse. Public verification " @@ -736,13 +732,6 @@ def doParseArgs(): "build_id) for fast local dev; the result is loose-provenance and is " "refused by the publish path. Falls back to the defaults `reuse_policy:` " "value, else 'strict'.")) - build_remote.add_argument("--reuse-base", dest="reuseBase", metavar="BUILD_ID", default=None, - help=("With --reuse-policy relaxed, the build_id of the deployed release to " - "graft packages from. Falls back to the defaults `reuse_base:` value. " - "Special values auto-select from the cvmfs:// store: 'latest' (newest " - "build_id for the build target) and 'latest-common' (newest build_id " - "shared by all requested packages); the same happens when left empty " - "under relaxed. The chosen build_id is announced.")) build_remote.add_argument("--reuse-from", dest="reuseFrom", metavar="PATH|cvmfs", default=None, help=("Reuse deployed components via their published modulefiles at this " "absolute modules-tree path (distinct from --remote-store, which is the " diff --git a/bits_helpers/cvmfs_reuse.py b/bits_helpers/cvmfs_reuse.py index d439366f..fe16dacb 100644 --- a/bits_helpers/cvmfs_reuse.py +++ b/bits_helpers/cvmfs_reuse.py @@ -1,143 +1,8 @@ # SPDX-FileCopyrightText: 2015-2026 CERN # SPDX-License-Identifier: GPL-3.0-or-later -""" -Relaxed-reuse matcher (ADR-0001, Stage 1b). - -Find a package already deployed in a blessed CVMFS release that can be grafted by -(name, architecture, build_id) — without the exact content-hash match that -``strict`` reuse requires. This is the read-only lookup the dependency resolver -calls when ``--reuse-policy relaxed`` is in effect; it never mutates the store. - -The deployed layout mirrors what ``CVMFSRemoteSync`` consumes: -``//Packages///.meta.json``, where -``.meta.json`` carries the ``build_id`` written by ``create_provenance_info`` -(Stage 0). A deployment that predates Stage 0 (no ``build_id``) simply never -matches, so relaxed reuse degrades to a normal build — safe by default. -""" - -import json -import os -from glob import glob - - -def _read_meta(path): - try: - with open(path) as fh: - return json.load(fh) - except (OSError, ValueError): - return None - - -def available_build_ids(package, architecture, store_root): - """Return ``{build_id: (version_dir, mtime)}`` for *package* on the store. - - Scans ``//Packages//*/.meta.json`` and - keeps, per build_id, the version directory with the newest ``.meta.json`` - mtime (the recency proxy used for "latest" — there is no build-time field in - the metadata). Defensive: a missing store or unreadable meta yields ``{}``. - """ - out = {} - if not (package and architecture and store_root): - return out - pkg_dir = os.path.join(store_root, architecture, "Packages", package) - if not os.path.isdir(pkg_dir): - return out - for ver_dir in glob(os.path.join(pkg_dir, "*")): - if not os.path.isdir(ver_dir): - continue - meta = _read_meta(os.path.join(ver_dir, ".meta.json")) - if not isinstance(meta, dict): - continue - bid = meta.get("build_id") - if not bid: - continue - try: - mtime = os.path.getmtime(os.path.join(ver_dir, ".meta.json")) - except OSError: - mtime = 0.0 - if bid not in out or mtime > out[bid][1]: - out[bid] = (ver_dir, mtime) - return out - - -def select_build_id(packages, architecture, store_root, strategy="latest"): - """Pick a build_id for relaxed reuse across the requested *packages*. - - ``latest`` : newest build_id available for the anchor (the last - requested package — the build target). A coherent release - shares one build_id across its whole closure, so the - target's newest build_id already covers its dependencies. - ``latest-common`` : newest build_id present for EVERY requested package. - - Only the top-level requested packages are known here (the resolved - dependency closure is not available until after this selection feeds the - resolver), so "common" is across the requested targets. Returns - ``(build_id | None, coverage)`` where coverage maps build_id -> set(packages). - """ - per_pkg = {p: available_build_ids(p, architecture, store_root) for p in packages} - coverage, newest = {}, {} - for pkg, ids in per_pkg.items(): - for bid, (_vd, mtime) in ids.items(): - coverage.setdefault(bid, set()).add(pkg) - newest[bid] = max(newest.get(bid, 0.0), mtime) - if not coverage: - return None, coverage - if strategy == "latest-common": - want = len(set(packages)) - common = [b for b, pkgs in coverage.items() if len(pkgs) == want] - if not common: - return None, coverage - return max(common, key=lambda b: newest[b]), coverage - # 'latest': anchor on the build target (last requested package) - anchor_ids = per_pkg.get(packages[-1], {}) if packages else {} - if not anchor_ids: - return None, coverage - return max(anchor_ids, key=lambda b: anchor_ids[b][1]), coverage - - -def graftable_match(package, architecture, build_id, store_root): - """Return a match descriptor for *package* under *build_id*, or None. - - Scans ``//Packages///.meta.json`` - and returns the first version whose recorded ``build_id`` matches and whose - ``architecture`` (when recorded) agrees: - - {"package", "version", "path", "hash", "build_id"} - - Defensive throughout: a missing store, an unreadable or legacy ``.meta.json``, - or any mismatch yields None (→ the package is built normally). - """ - if not (package and architecture and build_id and store_root): - return None - pkg_dir = os.path.join(store_root, architecture, "Packages", package) - if not os.path.isdir(pkg_dir): - return None - # Deterministic order so repeated resolutions agree; one version per package - # is expected within a coherent release, but be explicit anyway. - for ver_dir in sorted(glob(os.path.join(pkg_dir, "*"))): - if not os.path.isdir(ver_dir): - continue - meta = _read_meta(os.path.join(ver_dir, ".meta.json")) - if not isinstance(meta, dict): - continue - if meta.get("build_id") != build_id: - continue - meta_arch = meta.get("architecture") - if meta_arch and meta_arch != architecture: - continue - pkg_info = meta.get("package") if isinstance(meta.get("package"), dict) else {} - # Take version/revision from the deployed .meta.json (authoritative), NOT - # from the directory basename — that is "-" and cannot - # be split unambiguously (versions contain dashes). The consumer's reuse - # decision matches the deployed tarball by "--…", so the - # spec's version MUST equal the deployed version for the graft to fire. - return { - "package": package, - "version": pkg_info.get("version") or os.path.basename(ver_dir.rstrip("/")), - "revision": pkg_info.get("revision"), - "path": ver_dir, - "hash": pkg_info.get("hash"), - "build_id": build_id, - } - return None +# The legacy ADR-0001 cvmfs:// build_id-graft reuse (available_build_ids, +# select_build_id, graftable_match) was removed in the Step 5 cleanup. Deployed +# components are now reused via --reuse-from (module overlay); see +# bits_helpers/cvmfs_import.py and cvmfs_layout.py. This module is empty and +# scheduled for `git rm`. diff --git a/tests/test_args.py b/tests/test_args.py index bf07f144..de6b94c0 100644 --- a/tests/test_args.py +++ b/tests/test_args.py @@ -296,14 +296,12 @@ def test_defaults_are_inert(self): # resolves reusePolicy to "strict"); nothing changes. a = self._parse("build --force-unknown-architecture zlib") self.assertIsNone(a["reusePolicy"]) - self.assertIsNone(a["reuseBase"]) self.assertEqual(a["buildLocal"], []) def test_relaxed_flags_parse(self): a = self._parse("build --force-unknown-architecture --reuse-policy relaxed " - "--reuse-base LCG_109 --build-local p1,p2 zlib") + "--build-local p1,p2 zlib") self.assertEqual(a["reusePolicy"], "relaxed") - self.assertEqual(a["reuseBase"], "LCG_109") self.assertEqual(a["buildLocal"], ["p1", "p2"]) def test_initdotsh_flag_tristate(self): diff --git a/tests/test_cvmfs_reuse.py b/tests/test_cvmfs_reuse.py index 2ff475e9..9fb51a44 100644 --- a/tests/test_cvmfs_reuse.py +++ b/tests/test_cvmfs_reuse.py @@ -1,142 +1,7 @@ # SPDX-FileCopyrightText: 2015-2026 CERN # SPDX-License-Identifier: GPL-3.0-or-later -""" -Tests for bits_helpers/cvmfs_reuse.graftable_match() (ADR-0001 Stage 1b). - -The matcher is read-only and not yet wired into the resolver, so these tests -fully cover its behaviour against a faked deployed-store tree. -""" - -import json -import os -import tempfile -import unittest - -from bits_helpers.cvmfs_reuse import graftable_match, select_build_id - - -ARCH = "ubuntu2510_x86-64-gcc15-dbg" - - -def _deploy(root, pkg, version, revision="1", *, build_id=None, architecture=ARCH, - pkg_hash="h", write_meta=True): - """Create //Packages//-/ + .meta.json. - - The directory is named version-revision (as bits deploys); version/revision - are recorded authoritatively in .meta.json's package field. - """ - d = os.path.join(root, architecture, "Packages", pkg, "%s-%s" % (version, revision)) - os.makedirs(d, exist_ok=True) - if write_meta: - meta = {"architecture": architecture, - "package": {"hash": pkg_hash, "version": version, "revision": revision}} - if build_id is not None: - meta["build_id"] = build_id - with open(os.path.join(d, ".meta.json"), "w") as fh: - json.dump(meta, fh) - return d - - -class TestGraftableMatch(unittest.TestCase): - - def setUp(self): - self.root = tempfile.mkdtemp() - - def test_match_on_name_arch_build_id(self): - d = _deploy(self.root, "ROOT", "6.38.00", "1", build_id="LCG_109-abc") - m = graftable_match("ROOT", ARCH, "LCG_109-abc", self.root) - self.assertIsNotNone(m) - # version/revision come from .meta.json, not the dir basename - self.assertEqual(m["version"], "6.38.00") - self.assertEqual(m["revision"], "1") - self.assertEqual(m["path"], d) - self.assertEqual(m["hash"], "h") - self.assertEqual(m["build_id"], "LCG_109-abc") - - def test_no_match_wrong_build_id(self): - _deploy(self.root, "ROOT", "6.38.00-1", build_id="LCG_108-zzz") - self.assertIsNone(graftable_match("ROOT", ARCH, "LCG_109-abc", self.root)) - - def test_no_match_wrong_arch(self): - _deploy(self.root, "ROOT", "6.38.00-1", build_id="LCG_109-abc") - self.assertIsNone( - graftable_match("ROOT", "osx_arm64_gcc15", "LCG_109-abc", self.root)) - - def test_no_match_missing_store(self): - self.assertIsNone(graftable_match("ROOT", ARCH, "LCG_109-abc", - os.path.join(self.root, "nope"))) - - def test_legacy_deploy_without_build_id_never_matches(self): - _deploy(self.root, "ROOT", "6.38.00-1", build_id=None) # pre-Stage-0 - self.assertIsNone(graftable_match("ROOT", ARCH, "LCG_109-abc", self.root)) - - def test_no_meta_json_never_matches(self): - _deploy(self.root, "ROOT", "6.38.00-1", write_meta=False) - self.assertIsNone(graftable_match("ROOT", ARCH, "LCG_109-abc", self.root)) - - def test_picks_the_matching_version_among_several(self): - _deploy(self.root, "Boost", "1.88.0", "1", build_id="OTHER") - _deploy(self.root, "Boost", "1.90.0", "1", build_id="LCG_109-abc") - m = graftable_match("Boost", ARCH, "LCG_109-abc", self.root) - self.assertIsNotNone(m) - self.assertEqual(m["version"], "1.90.0") - self.assertEqual(m["revision"], "1") - - def test_empty_inputs_are_safe(self): - self.assertIsNone(graftable_match("", ARCH, "x", self.root)) - self.assertIsNone(graftable_match("ROOT", "", "x", self.root)) - self.assertIsNone(graftable_match("ROOT", ARCH, "", self.root)) - self.assertIsNone(graftable_match("ROOT", ARCH, "x", "")) - - -class TestSelectBuildId(unittest.TestCase): - - def setUp(self): - self.root = tempfile.mkdtemp() - - def _mtime(self, d, t): - os.utime(os.path.join(d, ".meta.json"), (t, t)) - - def test_latest_anchors_on_build_target(self): - # ROOT is the target (last requested). Newer build_id must win for it, - # regardless of what CMake carries. - old = _deploy(self.root, "ROOT", "6.0", "1", build_id="rel-old") - new = _deploy(self.root, "ROOT", "6.1", "1", build_id="rel-new") - self._mtime(old, 1000); self._mtime(new, 2000) - _deploy(self.root, "CMake", "3.30", "1", build_id="rel-old") - bid, cov = select_build_id(["CMake", "ROOT"], ARCH, self.root, "latest") - self.assertEqual(bid, "rel-new") - self.assertIn("ROOT", cov["rel-new"]) - - def test_latest_common_requires_all_packages(self): - # rel-new is newest but only ROOT has it; rel-old is shared by both. - r_old = _deploy(self.root, "ROOT", "6.0", "1", build_id="rel-old") - r_new = _deploy(self.root, "ROOT", "6.1", "1", build_id="rel-new") - c_old = _deploy(self.root, "CMake", "3.30", "1", build_id="rel-old") - self._mtime(r_old, 1000); self._mtime(r_new, 2000); self._mtime(c_old, 1000) - bid, _ = select_build_id(["CMake", "ROOT"], ARCH, self.root, "latest-common") - self.assertEqual(bid, "rel-old") - - def test_latest_common_tolerates_duplicate_packages(self): - # A duplicated target must not make the shared-by-all test unsatisfiable. - _deploy(self.root, "ROOT", "6.1", "1", build_id="rel-x") - bid, _ = select_build_id(["ROOT", "ROOT"], ARCH, self.root, "latest-common") - self.assertEqual(bid, "rel-x") - - def test_latest_common_none_when_no_shared_id(self): - _deploy(self.root, "ROOT", "6.1", "1", build_id="rel-a") - _deploy(self.root, "CMake", "3.30", "1", build_id="rel-b") - bid, _ = select_build_id(["CMake", "ROOT"], ARCH, self.root, "latest-common") - self.assertIsNone(bid) - - def test_none_when_store_empty_or_no_build_id(self): - self.assertEqual((None, {}), select_build_id(["ROOT"], ARCH, self.root, "latest")) - _deploy(self.root, "ROOT", "6.1", "1", build_id=None) # legacy: no build_id - bid, cov = select_build_id(["ROOT"], ARCH, self.root, "latest") - self.assertIsNone(bid) - self.assertEqual(cov, {}) - - -if __name__ == "__main__": - unittest.main() +# Tests for the legacy ADR-0001 graftable_match/select_build_id, removed in the +# Step 5 cleanup along with cvmfs_reuse.py. Deployed reuse is now driven by +# --reuse-from; its tests live in test_cvmfs_import.py / test_cvmfs_layout.py. +# This file is scheduled for `git rm`. From 1068b797afbc43ecb764ead7e006e6926163d06d Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 16:03:47 +0200 Subject: [PATCH 25/30] docs: retire cvmfs:// reuse; document --reuse-from (5/5) --- CHANGELOG.md | 2 + docs/REFERENCE.md | 118 ++++++++++++++++++++++------------------------ 2 files changed, 59 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c4a4921..92c3ae65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ Covers `bits`, `lcg.bits` (recipes), and `bits-recipe-tools`. Entries tagged **[ - **[Improvement]** `621dded` warn on provider version conflict; `4d92e95` point the "package not found" error at the provider mechanism; `ee78182` docs. ## CVMFS layout, merged views & relaxed reuse +- **[Feature]** `--reuse-from |cvmfs` reuses deployed components via their published modulefiles/`init.sh` (sourced in place from `/cvmfs`); `--reuse-policy strict|relaxed`, `::policy` sugar, per-dependency reuse, corrected pkg-config prefixes, and `cvmfs`-resolution from the `cvmfs_modules_template`. +- **[Change]** Removed the legacy `cvmfs://` `--remote-store` reuse: the `build_id` graft (`--reuse-base`/`--reuse-cvmfs`, `CVMFSRemoteSync`, `select_build_id`/`graftable_match`) is retired in favour of `--reuse-from`; a `cvmfs://` `--remote-store` now errors and points at `--reuse-from` (see ADR-0001, superseded). - **[Feature]** `0632f28` / `b70abac` / `b1dfc1d` / `e8d1222` / `ea04075` merged symlink-farm view: one-entry-per-var env, opt-in `enter/setenv --view`, view-aware `load`/`printenv` + age-based GC, path remap (fixes PyROOT). - **[Feature]** `58f13b6` / `c2d99be` / `f59a547` / `53e91d4` / `80b6a08` published per-`build_id` views on CVMFS, `bits publish --view`, per-tree pre-publish primitive, CVMFS layout recorded in `.meta.json`. - **[Feature]** `8f193fa`→`c52f5d7`, `cf19966` ADR-0001 import pipeline: modulefile harvest → classify → closure/`build_id` → overlay → `bits import` (build-sufficient from modulefiles). diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index baf64b19..fb652b36 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -107,10 +107,9 @@ bits resolves these to `install_path` / `module_path` and uses them to default: - **docker build:** `--cvmfs-prefix` ← `/`, so packages compile at their final CVMFS prefix and relocation on publish is a no-op (explicit `--cvmfs-prefix` still wins); -- **reuse:** with `--reuse-cvmfs`, `--remote-store` ← `cvmfs://`, so - already-deployed components are reused via the `CVMFSRemoteSync` store (which - matches a deployed package's recorded `.build-hash`/`.meta.json` against the - hash bits computes — reuse happens only on a hash match). +- **reuse:** `--reuse-from cvmfs` resolves the deployed modules tree from the + same layout, so already-deployed components are set up from their published + modulefiles (`--remote-store` stays the tarball store; it is never `cvmfs://`). Builds that don't set any of these fields are unaffected. @@ -143,9 +142,9 @@ wins — and each has its own root of trust: 1. **Local store on the build node** (`$WORK_DIR/TARS`, already-unpacked `INSTALLROOT`) — artifacts this node built or fetched earlier. Ultimately trusted (produced here) and cheapest, so it is consulted first. -2. **CVMFS**, if mounted (`--reuse-cvmfs` / `cvmfs://`) — the published read-only - tree. Trusted by CVMFS itself: the repository is signed at Stratum-0 and the - client verifies it against the repo key in `/etc/cvmfs/keys`. No extra +2. **CVMFS**, when reusing a deployed release (`--reuse-from`) — the published + read-only tree. Trusted by CVMFS itself: the repository is signed at Stratum-0 + and the client verifies it against the repo key in `/etc/cvmfs/keys`. No extra bits-level attestation is needed for these artifacts. 3. **Remote archive (S3/HTTP), verified against a signed manifest** — content- addressed tarballs. Integrity comes from the content hash + `tarball_sha256`; @@ -862,10 +861,9 @@ bits build [options] PACKAGE [PACKAGE ...] |--------|-------------| | `--defaults PROFILE` | Defaults profile(s); use `::` to combine (e.g. `release::myproject`). Default: `release`. | | `--flavour NAME[=VALUE]` | Set a build-wide flavour variable (repeatable, comma-separated). `NAME`→`true`, `NAME=VALUE`→`VALUE`, `!NAME`→`false`. Gates `(?NAME)` conditional requires/sources/patches and is exported into the build environment; overrides a defaults `variables:` entry of the same name. See [Flavours](#flavours). | -| `--reuse-cvmfs` | Reuse already-deployed components from the defaults `cvmfs_dir:` area: sets `--remote-store cvmfs://` when no store is given. See [CVMFS layout](#cvmfs-layout). | -| `--reuse-policy {strict,relaxed}` | How CVMFS reuse is matched. `strict` (default): reuse only on an exact content-hash match; the result is publishable. `relaxed`: also graft deployed packages of a blessed release matched by (name, architecture, `build_id`), for fast local dev on top of e.g. an LCG release — only the top of the stack is built. Relaxed artifacts are *loose-provenance* and are refused by the publish path. Falls back to the defaults `reuse_policy:` value. See [Relaxed CVMFS reuse](#relaxed-cvmfs-reuse). | -| `--reuse-base BUILD_ID` | With `--reuse-policy relaxed`, the `build_id` of the deployed release to graft from (the value `bits` records under `build_id` in each deployed package's `.meta.json`). Falls back to the defaults `reuse_base:` value. | -| `--build-local PKG[,PKG…]` | Packages to always build locally even under `--reuse-policy relaxed` (e.g. one you need patched), instead of grafting them from the base. | +| `--reuse-from PATH\|cvmfs` | Reuse deployed components via their published modulefiles at this absolute modules-tree path (distinct from `--remote-store`, which is the tarball store). The literal `cvmfs` resolves the location from the defaults `system:` layout (`module_dir`/`cvmfs_dir`) or the `cvmfs_modules_template`. A trailing `::relaxed`/`::strict` also sets the reuse policy (e.g. `cvmfs::relaxed`). See [Reusing deployed components](#relaxed-cvmfs-reuse). | +| `--reuse-policy {strict,relaxed}` | How a reused (`--reuse-from`) component is matched. `strict` (default): reuse only on an exact content-hash match; the result is publishable. `relaxed`: reuse any version present in the one-release overlay, for fast local dev on top of e.g. an LCG release — only the top of the stack is built. Relaxed builds are *loose-provenance* and are refused by the publish path. Falls back to the defaults `reuse_policy:` value. | +| `--build-local PKG[,PKG…]` | Packages to always build locally even when they could be reused (e.g. one you need patched), instead of taking them from `--reuse-from`. | | `-a ARCH`, `--architecture ARCH` | Target architecture. Default: auto-detected, or the `architecture:` template from defaults (see [§9](#9-architecture-overview)). An explicit value here overrides the template. | | `--force-unknown-architecture` | Proceed even if architecture is unrecognised. | | `-j N`, `--jobs N` | Parallel compilation jobs per package. Default: CPU count. | @@ -2696,18 +2694,22 @@ When either `--remote-store` or `--write-store` is given, bits automatically set | `http://` or `https://` | HTTP/HTTPS | ✓ | — | None (public) or TLS; use `--insecure` to skip cert check | | `s3://BUCKET/PATH` | Amazon S3 via `s3cmd` | ✓ | ✓ | `~/.s3cfg` config file | | `b3://BUCKET/PATH` | S3-compatible via `boto3` | ✓ | ✓ | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` env vars | -| `cvmfs://REPO/PATH` | CernVM File System | ✓ | — | None (read-only filesystem) | | `rsync://HOST/PATH` or `/local/path` | rsync | ✓ | ✓ | SSH keys (`~/.ssh/`) or filesystem permissions | +> `cvmfs://` is **not** a `--remote-store` backend. `--remote-store` is the +> tarball store; to reuse components already deployed on CVMFS use +> [`--reuse-from`](#relaxed-cvmfs-reuse). A `cvmfs://` `--remote-store` is +> rejected with an error pointing at `--reuse-from`. + #### Mixing a read-only remote with a separate write store -A read-only `--remote-store` (`cvmfs://` or `http(s)://`) can be paired with a writable `--write-store` of a different backend, e.g. recall pre-built packages from a CVMFS release and upload newly-built ones to S3: +A read-only `--remote-store` (`http(s)://`) can be paired with a writable `--write-store` of a different backend, e.g. recall pre-built packages from an HTTP mirror and upload newly-built ones to S3: ```bash -bits build ... --remote-store cvmfs:///cvmfs/.../bits/ --write-store b3://mybucket +bits build ... --remote-store https://mirror.example/bits/ --write-store b3://mybucket ``` -Reads (recall) go to the remote store; uploads go to the write store. **Only freshly-built packages are uploaded** — packages recalled from the read-only store keep their original provenance and are not re-published (a CVMFS-recalled package has only a synthetic tarball of symlinks into `/cvmfs`, so uploading it would publish a stub). This is *strict* reuse only; `--reuse-policy relaxed` (loose provenance) remains barred from any write store. +Reads (recall) go to the remote store; uploads go to the write store. **Only freshly-built packages are uploaded** — packages recalled from the read-only store keep their original provenance and are not re-published. #### HTTP / HTTPS @@ -2807,14 +2809,6 @@ runners; if the key must never touch shared infrastructure, register one dedicated protected runner (it only dials out — no standing server). A ready template lives in `bits-console/.gitlab/sign-manifest.yml`. -#### CernVM File System (`cvmfs://`) - -Read-only. Instead of unpacking a remote tarball, bits creates a small local tarball containing symlinks that point into the already-mounted CVMFS repository. The build environment is constructed from the CVMFS paths without copying data locally: - -```bash -bits build --remote-store cvmfs://cvmfs.example.cern.ch/sw ROOT -``` - #### rsync / local filesystem Supports both remote hosts (via SSH) and local paths. Useful for shared NFS or a build server accessible over SSH: @@ -2949,7 +2943,6 @@ SOURCES/cache/a1/a1b2c3d4.../libfoo-1.2.tar.gz | `RsyncRemoteSync` | ✓ | ✓ | Uses `rsync -vW`; skipped if `--write-store` is absent. | | `S3RemoteSync` | ✓ | ✓ | Uses `s3cmd get/put`; skipped if `--write-store` is absent. | | `Boto3RemoteSync` | ✓ | ✓ | Native boto3 API; skips upload if the key already exists. | -| `CVMFSRemoteSync` | ✓ | — | Read-only filesystem mount; upload not supported. | #### Enabling source archive caching @@ -3050,43 +3043,46 @@ Steps to investigate: ``` The next build run will re-record the current digest and warn instead of aborting. -### Relaxed CVMFS reuse - -Strict reuse (the default) only reuses a deployed package when bits recomputes -the **exact same content hash** — which guarantees reproducibility and keeps the -result publishable, but means that to build one package on top of a blessed -release (e.g. an LCG release on `/cvmfs`) you must reproduce every hashed input -of the whole stack. **Relaxed reuse** trades that for speed in local -development: it grafts deployed packages matched by **(name, architecture, -`build_id`)** instead of by hash, so only the top of the stack is built and -everything below it is symlinked from `/cvmfs`. - -`build_id` is the per-release coherence token bits records in each package's -`.meta.json` (and which is identical for every package built together). Matching -on it — together with the *combined* architecture string, which already encodes -OS, compiler and build type — means the grafted set is ABI-consistent by -construction, without re-verifying the dependency closure. - -```bash -bits build --reuse-policy relaxed \ - --reuse-base LCG_109- \ - --remote-store cvmfs:///cvmfs/sft.cern.ch/lcg/releases \ - --defaults dev4 key4hep -``` - -Grafted dependencies are logged as **Unpacking** (no recompilation); only the -requested top package is **Compiling**. Use `--build-local PKG[,PKG…]` to force -specific packages to build locally anyway (e.g. one you need patched). - -**Provenance.** Any package built on top of a relaxed graft is *loose* — its -closure includes binaries adopted by name/`build_id` rather than verified hash, -so its own hash no longer certifies reproducible inputs. bits records -`provenance: loose` in such a package's `.meta.json`, and the **publish path -refuses loose artifacts** (`--reuse-policy relaxed` with `--write-store` or -`--pipeline` is rejected). Relaxed reuse is therefore strictly a dev/iteration -accelerator; production builds use `strict`. The matcher requires a `cvmfs://` -`--remote-store` and a `--reuse-base`; without either, relaxed reuse warns and -falls back to a normal build. + +### Reusing deployed components (`--reuse-from`) + +To build on top of a release already deployed on CVMFS, point `--reuse-from` at +its published **modules tree** (or the literal `cvmfs`, which resolves the +location from the defaults `system:` layout / `cvmfs_modules_template`). Each +reused component is set up from its deployed modulefile / `init.sh` — sourced in +place from `/cvmfs`, not copied — so only the top of the stack is built and +everything below it is consumed from the deployment. `--reuse-from` is distinct +from `--remote-store`, which remains the tarball store. + +```bash +bits build --reuse-from cvmfs::relaxed \ + --docker --docker-image \ + --architecture x86_64-el9-gcc14-opt \ + --defaults lcg::release::gcc14::opt \ + --build-local xrootd xrootd +``` + +Reused components are logged as **Reuse: … (not built)**; only the requested top +package is **Compiling**. Use `--build-local PKG[,PKG…]` to force specific +packages to build locally anyway (e.g. one you need patched). + +**Policy.** `--reuse-policy strict` (default) reuses a component only on an exact +content-hash match, so the build stays reproducible and publishable. +`--reuse-policy relaxed` reuses any version present in the one-release overlay +(matched via its `build_id`) — faster for local iteration, but **loose +provenance**: the result is not reproducible from hash alone, so the **publish +path refuses it** (`--reuse-policy relaxed` with `--write-store` or `--pipeline` +is rejected). The `::relaxed`/`::strict` suffix on `--reuse-from` sets the +policy inline; an explicit `--reuse-policy` must agree with it. + +> The builder image must match the reused release's ABI (OS + compiler): reused +> binaries carry the toolchain they were built with, so run them in an image that +> provides a compatible runtime. + +> **Note.** Earlier versions reused deployed packages through a `cvmfs://` +> `--remote-store` and a `--reuse-base ` graft; that path has been +> removed in favour of `--reuse-from`. A `cvmfs://` `--remote-store` now errors +> and points here. --- From 910fa18e2c06e3a3cbf691c5d498c8624094c699 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 16:04:27 +0200 Subject: [PATCH 26/30] cleanup: git rm legacy reuse stubs --- bits_helpers/cvmfs_reuse.py | 8 -------- tests/test_cvmfs_fetch_symlinks.py | 7 ------- tests/test_reuse_resolver.py | 7 ------- 3 files changed, 22 deletions(-) delete mode 100644 bits_helpers/cvmfs_reuse.py delete mode 100644 tests/test_cvmfs_fetch_symlinks.py delete mode 100644 tests/test_reuse_resolver.py diff --git a/bits_helpers/cvmfs_reuse.py b/bits_helpers/cvmfs_reuse.py deleted file mode 100644 index fe16dacb..00000000 --- a/bits_helpers/cvmfs_reuse.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: 2015-2026 CERN -# SPDX-License-Identifier: GPL-3.0-or-later - -# The legacy ADR-0001 cvmfs:// build_id-graft reuse (available_build_ids, -# select_build_id, graftable_match) was removed in the Step 5 cleanup. Deployed -# components are now reused via --reuse-from (module overlay); see -# bits_helpers/cvmfs_import.py and cvmfs_layout.py. This module is empty and -# scheduled for `git rm`. diff --git a/tests/test_cvmfs_fetch_symlinks.py b/tests/test_cvmfs_fetch_symlinks.py deleted file mode 100644 index b7131183..00000000 --- a/tests/test_cvmfs_fetch_symlinks.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-FileCopyrightText: 2015-2026 CERN -# SPDX-License-Identifier: GPL-3.0-or-later - -# Tests for CVMFSRemoteSync.fetch_symlinks, removed with the legacy cvmfs:// -# remote-store reader (Step 5 cleanup). Deployed reuse is now driven by -# --reuse-from (module overlay); see test_cvmfs_import.py / test_build.py. -# This file is scheduled for `git rm`. diff --git a/tests/test_reuse_resolver.py b/tests/test_reuse_resolver.py deleted file mode 100644 index 7ec495c0..00000000 --- a/tests/test_reuse_resolver.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-FileCopyrightText: 2015-2026 CERN -# SPDX-License-Identifier: GPL-3.0-or-later - -# Legacy ADR-0001 cvmfs:// build_id-graft tests removed with the graft path -# (Step 5 cleanup). Reuse is now driven by --reuse-from (module overlay), whose -# tests live in test_cvmfs_import.py / test_build.py / test_cvmfs_layout.py. -# This file is scheduled for `git rm`. From 320de489e7900f8cf96f9a92317c33caee5a0f0a Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 16:05:31 +0200 Subject: [PATCH 27/30] cleanup: git rm legacy reuse stubs --- docs/adr/0001-cvmfs-relaxed-reuse.md | 355 --------------------------- 1 file changed, 355 deletions(-) delete mode 100644 docs/adr/0001-cvmfs-relaxed-reuse.md diff --git a/docs/adr/0001-cvmfs-relaxed-reuse.md b/docs/adr/0001-cvmfs-relaxed-reuse.md deleted file mode 100644 index 8d0c9349..00000000 --- a/docs/adr/0001-cvmfs-relaxed-reuse.md +++ /dev/null @@ -1,355 +0,0 @@ -# ADR-0001: Relaxed CVMFS package reuse via `build_id`, and importing foreign deployments - -**Status:** Proposed -**Date:** 2026-06-10 -**Deciders:** Predrag Buncic; bits maintainers; CVMFS/SFT release managers - -## Context - -bits reuses a published CVMFS package instead of recompiling it **only when the -full content hash matches** the hash bits computes for the current recipe -closure (recipe body — comments excluded — plus the `defaults` env, the -architecture string, and every transitive dependency's hash). This exact-hash -rule is correct and is what makes a bits artifact reproducible and safe to -publish: the hash certifies the *entire* input closure, so two packages that -share a hash are byte-for-byte interchangeable. - -Two real needs are not served by exact-hash-only reuse: - -1. **Fast local development on top of a blessed stack.** A developer wants to - build and test the *top* of a stack (e.g. key4hep, or one generator) on top - of an already-published, blessed release (an LCG release on - `/cvmfs/sft.cern.ch/...`) *without recompiling the world*. The deployed base - was built by a different pipeline/host, so its hashes will not match what - this developer's bits would compute — exact-hash reuse therefore rebuilds - everything, defeating the purpose. - -2. **Adopting pre-existing, non-bits CVMFS deployments.** Established releases - (LCG) are already deployed on CVMFS but lack bits-native metadata - (`.meta.json`, `build_id`) and may even lack a usable modulefile layout. We - want bits to *consume* such a deployment as a reuse source without rebuilding - or republishing it, and without write access to the (read-only, not-ours) - package tree. - -What bits already has (verified, current code): - -- `cvmfs://` remote store (`CVMFSRemoteSync`, `sync.py`): reads a deployed - `Packages////.build-hash` and grafts the deployed files into - the store so the normal reuse path consumes them. Read-only (no `writeStore`). -- Hash-gated reuse decision (`build.py` ~2090–2360) keyed on - `remote_hashes`/`local_hashes`; builds write `.build-hash`/`.meta.json`. -- The `islink && isdir` short-circuit that would let `sw///` - point straight at `/cvmfs` (true zero-copy) exists but is not wired for reuse. -- `prefer_system` machinery (a CLI/defaults-driven "this dep is provided - externally, prune it" hook in `getPackageList`). -- Qualified architecture strings that already encode the ABI-critical axes, - e.g. `ubuntu2510_x86-64-gcc15-dbg` (OS/glibc, compiler, build type). - -Forces: we must **not** weaken the integrity of the publish path (the -content-addressed store and the cvmfs-prepub transparency log depend on -hash purity), while making the dev path fast and able to adopt foreign releases. -**Hard constraint:** the simple case — a plain `bits build` / aliBuild with a -local alidist and no CVMFS — must remain bit-for-bit unaffected (identical -hashes, no forced rebuild, no schema break). Every decision below is therefore -opt-in and default-off, all new metadata is additive and defensively read, and -nothing in the hashed path (recipes, `MODULE_OPTIONS`, `bits-recipe-tools`) -changes. See the implementation plan's "Backward compatibility" section for the -invariants and the CI regression guard. - -## Decision - -Introduce an **opt-in, policy-gated relaxed reuse mode** plus a **`build_id` -coherence token** and an **importer** for foreign deployments. Concretely: - -**D1 — `--reuse-policy strict|relaxed` (CLI flag and `reuse_policy:` defaults -variable; default `strict`).** It is *not* a recipe field, because the same -recipes serve standard builds; the relaxation is a property of *this invocation*, -not of the package. `strict` is today's behaviour (exact-hash reuse only; -artifacts are hash-pure and publish-eligible). `relaxed` enables D2–D4. - -**D2 — Relaxed match key = (name, qualified-arch, `build_id`), not the hash.** -Because lcg.bits recipes do not pin dependency versions, "version" reduces to -"whatever the blessed set deploys": in `relaxed` mode **the base's versions -win**, overriding the recipe/`defaults` version intent. A package the top layer -genuinely needs built differently (newer version, a patch the base lacks) is -forced out of the graft with `--build-local [,…]` and built locally -above the frontier. The qualified arch supplies ABI safety (compiler + build -type + OS). `build_id` supplies *closure* safety: see D3. - -**D3 — `build_id`: a per-release coherence token stamped on every package built -together.** Same `build_id` ⇒ the packages were built as one coherent set ⇒ any -subset is mutually ABI-consistent **by construction**, so the resolver does not -have to walk and verify the dependency closure. `build_id` is **content-derived -and deterministic** (a human-readable release label, e.g. `LCG_109`, plus a -hash of the manifest = the sorted set of `{name, version, path, deps}`), so -independent or repeated imports of the same release agree and the id is -verifiable. Stored in `.meta.json` (`package.build_id`) and surfaced in the -modulefile as a `module-whatis "build_id: …"` line (not a `setenv`, to avoid -leaking it into the loaded environment). A later `base_build_id` lineage field -extends this to "built *against* release X in a separate run" (see Open -Questions). - -The `build_id` **manifest doubles as the strict-reproducibility spec.** Besides -the package set, it records the inputs that determine bits' content hashes — -the recipe-repo commits (`lcg.bits`/`stacks.bits`), the `bits-recipe-tools` -version, the `defaults`, and an explicit **`abi_tag`** (compiler, libstdc++ ABI -/ `_GLIBCXX_USE_CXX11_ABI`, `CXXSTD`, glibc/OS). So one object serves both -modes: the *coherence token* for `relaxed`, and the exact "what to check out to -get matching hashes" recipe for `strict`. This is what makes the publish-grade -path ("reuse a CVMFS package whose hash matches") dependable on a fresh host -rather than best-effort, and it lets cross-`build_id` (`base_build_id`) -compatibility be **checked** against `abi_tag` instead of assumed. - -**D4 — Frontier-cut at resolution time + zero-copy graft.** Reusing the existing -`prefer_system` hook in `getPackageList`: for each `requires`, if a deployed/ -overlay package matches (name via the alias map of D7, qualified-arch, and the -selected `build_id`), **prune that node and its entire subtree** from the build -graph, wire its modulefile environment into the consumers, and zero-copy -symlink `sw///` and `MODULES//` at `/cvmfs` (wiring -the dormant `islink && isdir` short-circuit). Everything *above* the frontier is -built locally against the grafted deps. - -**D5 — Provenance qualification, and it is contagious upward.** A `relaxed` -graft taints provenance. Crucially, taint **propagates up the closure**: any -package built above a relaxed graft is itself loose, because its inputs include -unverified binaries. Provenance is therefore "pure" only if the *entire* closure -is pure (hash-verified or locally built from pure inputs). Each artifact records -its provenance (`pure` | `loose`, plus the `build_id`(s) it borrowed) in -`.meta.json`. **The publish pipeline accepts only `pure` artifacts** and checks -the whole closure, not just the directly-grafted nodes. - -**D6 — Out-of-tree module + metadata overlay.** bits learns to read package -metadata from a *module-adjacent* location, not only from inside the package -tree. Native published packages keep `.meta.json` **with the package** (the -artifact stays self-describing and signable). Foreign/imported deployments get a -separate, bits-owned overlay (generated modulefiles + module-side metadata) that -*points into* the read-only foreign tree via `BASEDIR`/symlinks. The policy -selects the source: `strict` reads the package-side `.meta.json` (the binary's -own hash); `relaxed` reads the module-side `build_id`. - -**D7 — Importer (`bits import`).** Convert a foreign -deployment into a bits-consumable overlay: - -1. **Harvest** the deployed modulefiles by *evaluating* them - (`MODULEPATH= modulecmd sh display /`) rather than parsing raw - Tcl, capturing the resolved operations into one JSON **corpus**. Fallback - when no modulefiles exist (manifest-only deployments): synthesise the env - from the manifest's path info plus the matched recipe template. -2. **Classify** each entry's ops into three buckets: env ops that fit the - generic `BitsModule` template (`bin→PATH`, `lib/lib64→LD_LIBRARY_PATH`+ - `CMAKE_PREFIX_PATH`, `python→PYTHONPATH`, `pkgconfig→PKG_CONFIG_PATH`); - **`module load`/`prereq`/`depends-on` → a structured `deps:[…]` list** - (so names are remappable and so they form the release graph edges); and any - leftover `setenv`/etc. carried **verbatim**. ~90% are pure generic+deps; the - rest keep verbatim extras. -3. **Factor the base prefix** out of every path so generation is pure - substitution over (prefix, version, `build_id`). -4. **Closure-check** the `deps` edge set — refuse to stamp one `build_id` on a - set with dangling edges — then assign the deterministic `build_id` of D3. -5. **Generate** the bits overlay modulefiles (retargeted prefix + `build_id` + - remapped deps) and the module-side metadata of D6, with **path validation** - (the generated paths must exist in the CVMFS tree) and per-package overrides. - The modulefile is made **build-sufficient** — it adds the build hooks - (`CMAKE_PREFIX_PATH`/`PKG_CONFIG_PATH`/`CPATH`/`_ROOT`, guarded on the - tree) so a grafted dep's build env comes from *loading the modulefile* (via - `bits printenv`/module), the same mechanism as bits-native deps. No separate - `init.sh` is synthesised. - -The corpus is simultaneously the import source, the bits-native manifest for the -release, the template library, and the name corpus. - -**D8 — Name-alias map is the only fuzzy human input.** lcg.bits vocabulary and -the foreign deployment's vocabulary (e.g. `ROOT` vs `root`, `-local2` vs -hash-revisions) are reconciled once via an alias table, used **only** at -reuse-resolution time to match a recipe's `requires` to an overlay module. Env, -deps, paths, and the closure check all come from the corpus — not the recipe. - -**D9 — The published module-overlay directory is its own CVMFS nested -catalog.** When the generated modulefiles are published, the overlay root -carries a `.cvmfscatalog` marker so that subtree becomes an independent nested -catalog. This makes module enumeration use bits' **existing fast catalog path** -(`cvmfs_catalog.py` / `bitsModules`: read `user.catalog_counters`, fetch the -catalog, query SQLite — one HTTP round-trip instead of a per-file FUSE walk), -which is exactly how `bits q` already lists modules quickly on CVMFS. Both the -runtime listing (`bits q`/`avail`) and the reuse resolver scanning the overlay -for available `build_id` packages then enumerate in O(1 catalog fetch) rather -than walking the tree. The catalog boundary also lets the whole module set be -fetched/cached as a unit and snapshotted independently of the (much larger) -package payload. The importer emits the marker; native module publishing should -do the same for its `MODULES//` root. - -**D10 — Module sets are per-`build_id`; package payload is deduplicated; -multiple modulefiles may reference one package under owner attestation.** Every -package in a coherent release is published with its own modulefile, **even when -its content is unchanged** and CVMFS CAS deduplicates the underlying payload — -the modulefile is a tiny per-release entry, so a full per-release module set is -essentially free. The module overlay is therefore namespaced by `build_id` -(layout `MODULES///…`, one nested catalog each per D9), which -also removes the collision when several releases share a `MODULEPATH` (two -`Boost/1.90` from different releases are distinct module *files* pointing at the -same deduplicated payload). A single CVMFS package may be referenced by several -`(version, build_id)` modulefiles — i.e. an already-published binary can be -adopted into a *new* coherent set without rebuild. Because that breaks the -"built-together ⇒ coherent" guarantee of D3 (the binary was built in a -*different* run), it is valid **only as an explicit act of the repository owner, -who attests the package works within the new set**; the attestation (who, when, -which `base_build_id`/sets) is recorded in the new `build_id` manifest and is the -provenance for that owner-asserted reuse. bits trusts the attestation; it does -not re-derive coherence for cross-set references. - -## Options Considered - -### Option A: Exact-hash reuse only (status quo) -| Dimension | Assessment | -|-----------|------------| -| Complexity | Low (already built) | -| Reproducibility | Perfect | -| Dev iteration on blessed stacks | Poor — rebuilds everything | -| Adopt foreign (LCG) releases | Not possible | - -**Pros:** Strong integrity; nothing to build. **Cons:** Fails both target use cases. - -### Option B: Loose match on (name, version) only, no `build_id` -| Dimension | Assessment | -|-----------|------------| -| Complexity | Low | -| ABI safety | **Unsafe** — can mix incompatible closures (diamond ABI) | -| Reproducibility | None | - -**Pros:** Trivial. **Cons:** Silent ABI corruption when a consumer links a -grafted dep alongside a differently-built sub-dep. Rejected. - -### Option C: Relaxed match on (name, qualified-arch, `build_id`) + frontier-cut (CHOSEN) -| Dimension | Assessment | -|-----------|------------| -| Complexity | Medium | -| ABI safety | Safe *by construction* (same `build_id` ⇒ built together) | -| Dev iteration | Fast (build only the top) | -| Adopt foreign releases | Yes, via importer overlay | -| Reproducibility | Loose (explicitly gated out of publish) | - -**Pros:** Safe, fast, adopts foreign sets, clean strict/relaxed split. -**Cons:** Loose artifacts not reproducible (by design); needs `build_id`, -importer, alias map. - -### Option D: Regenerate modulefiles from lcg.bits recipes (rejected for env) -**Pros:** Recipe knows bespoke env. **Cons:** Depends on a matching recipe; -assumes the foreign layout matches the recipe's `MODULE_OPTIONS`; re-derives env -that already exists and is validated in the deployed modulefile. Rejected in -favour of harvesting the deployed modulefile (D7); the recipe is reduced to the -name-alias role (D8). - -### Option E: Relocate `.meta.json` globally to the module side (rejected) -**Pros:** One place; convenient for the importer. **Cons:** Breaks the -self-contained, signable native artifact (binary separated from its provenance). -Rejected in favour of the role-split overlay (D6). - -### Option F: Mutate the foreign package tree to inject bits metadata (rejected) -**Cons:** The foreign tree is read-only and not ours. Rejected; the overlay (D6) -adds compatibility without touching it. - -## Trade-off Analysis - -The core trade is **reproducibility for speed/adoptability**, made safe by two -disciplines that cost almost nothing because bits already has the hooks: - -- **ABI safety is bought by the qualified arch + `build_id`, not by closure - analysis.** This is the central bet: rather than verify that a grafted - subtree is internally consistent, we *trust a token that certifies it was - built together*. That collapses the hard part (graph consistency) into a - string compare, but it means `build_id` must be assigned only to genuinely - coherent sets (hence the closure-check in D7.4 and the determinism in D3). -- **Integrity is preserved by making loose-ness contagious and publish-blocked - (D5).** The strict path is untouched; the only way a loose binary reaches the - store is a pipeline bug, which the closure-wide provenance check guards. - -The residual risk is *misplaced trust*: if a `build_id` is stamped on an -incoherent set, or the qualified arch fails to capture an ABI axis, relaxed -reuse can produce broken binaries. Both are dev-only blast radius (never -published) and both are mitigated (closure-check; arch discipline). - -## Consequences - -**Easier:** developer iteration on blessed stacks (build only the top, instant -zero-copy base); adopting existing LCG releases as reuse sources without rebuild -or republish; reasoning about reuse safety (one token, not a graph walk); -finally wiring the true zero-copy symlink path. - -**Harder / new burden:** maintaining the name-alias map; keeping the importer's -env fidelity correct (harvest + validation + per-package overrides); ensuring -`build_id` is stamped only on closed/coherent sets; two metadata locations to -keep from drifting; the publish pipeline must enforce the closure-wide -provenance check. - -**To revisit:** `base_build_id` lineage for cross-build "work together"; -optionally tying `build_id` to the cvmfs-prepub signing/transparency log so a -blessed set's integrity is verifiable before adoption; GC/lifetime of relaxed -builds (they break if the blessed release is removed). - -## Open Questions / Gaps surfaced on review - -1. **Relaxed reuse presupposes "build against modules" — and the env must be - *build*-sufficient, not just runtime.** To compile the top layer against - grafted `/cvmfs` deps, the build environment must come from the deps' - **modulefiles**, not from bits' usual `INSTALLROOT` layout. The catch: a - harvested modulefile is runtime-oriented (`PATH`, `LD_LIBRARY_PATH`), whereas - compilation needs `CMAKE_PREFIX_PATH`, include/lib dirs, `pkg-config`, - `*_ROOT`. This is exactly the build-vs-runtime env drift that is already a - recurring bug class. So (a) the "build env == runtime modulefile env" - capability is a **prerequisite**, not an optional follow-on, for D4; and - (b) the importer must **validate that each grafted module is build-sufficient** - (or augment it) — this is the first place a relaxed build (e.g. key4hep on - LCG) will fail, at compile/link time, not runtime. -2. **ABI tag beyond the arch string.** Within one `build_id` ABI is automatic. - Across `build_id`s (the `base_build_id` lineage), compatibility must compare - ABI-relevant config (compiler, libstdc++ ABI / `_GLIBCXX_USE_CXX11_ABI`, - `CXXSTD`, glibc/OS). Record an explicit `abi_tag` with each `build_id` so - cross-build compatibility is *checked*, not assumed. -3. **Importer input modes.** The original motivating case is a manifest with - **no** modulefiles; `module show` harvest is impossible there. The importer - needs the manifest-only fallback (env from manifest paths + recipe template), - same corpus schema, different env source. -4. **Per-package "force local" escape.** A developer may need a package built - the way *their* recipe specifies (a patch/feature the blessed build lacks) - while grafting everything else. Support `relaxed` with an exclude list - (`--build-local pkgA,pkgB`), CLI/defaults, not a recipe field. -5. **Frontier/release selection.** When several `build_id`s are deployed, which - base to graft from must be explicit (`--reuse-base LCG_109`), with a sane - default (newest satisfying the most deps). -6. **Discoverability.** `bits q`/`avail` could filter by `build_id` so a user - can see "what the blessed set offers." -7. **Trust.** Relaxed reuse trusts the deployment; optional hardening is to - verify the `build_id` manifest signature (cvmfs-prepub transparency log) - before adoption. - -## Rollout / Sequencing - -The design couples two independently large efforts; decouple them to de-risk. - -- **Stage 0 — `build_id` only.** Add `build_id` (+ `abi_tag` + reproducibility - inputs, D3) to `.meta.json` and a `module-whatis` line. Small, immediately - useful for provenance/discoverability, and the foundation everything keys on. -- **Stage 1 — relaxed reuse of *bits-native* releases.** `--reuse-policy`, - frontier-cut, zero-copy graft, provenance propagation, **build-against-modules** - — validated against a release bits itself published (no importer needed; the - metadata already exists). This proves the reuse machinery and surfaces the - build-sufficient-env problem on a controlled surface. -- **Stage 2 — import foreign deployments.** The importer (D7), name-alias map - (D8), per-`build_id` overlay + nested catalog (D9/D10), manifest-only - fallback. This is where LCG adoption lands, on top of proven machinery. -- **Stage 3 — hardening.** `base_build_id` lineage + `abi_tag` compatibility - checks; signed-manifest verification; `bits q --build-id` filter. - -## Action Items - -1. [ ] Accept/iterate this ADR with release managers (esp. D5 publish-guard and D10 attestation). -2. [ ] Add `build_id` (+ `abi_tag`) to `.meta.json` and a `module-whatis` line via `MakeModule`; make it deterministic (D3). -3. [ ] Add `--reuse-policy strict|relaxed` (+ `reuse_policy:` defaults) and provenance fields (`pure|loose`, borrowed `build_id`s) in `.meta.json`. -4. [ ] Implement the resolution-time frontier-cut on the `prefer_system` hook; wire the zero-copy `islink && isdir` symlink path. -5. [ ] Implement closure-wide provenance propagation + the publish-pipeline `pure`-only guard. -6. [ ] Add out-of-tree overlay metadata reading (D6). -7. [ ] Build the importer (D7): `module show` harvest → corpus (prefix-factored, classified, deps as edges) → closure-check → deterministic `build_id` → generate overlay + validation + overrides; plus the manifest-only fallback. -8. [ ] Seed the lcg.bits ↔ foreign name-alias map (D8); report unmatched packages. -8a. [ ] Emit a `.cvmfscatalog` at the published module-overlay root (D9) so enumeration uses the fast catalog path; do the same for native `MODULES//`. -9. [ ] **Prerequisite track:** "build against modules" (build env == runtime modulefile env) — required by D4. -10. [ ] Defer: `base_build_id` lineage; signed-manifest verification; `bits q --build-id` filter. From c045897e3586baf6800c260a1395ff613602df42 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 17:03:02 +0200 Subject: [PATCH 28/30] =?UTF-8?q?cli:=20add=20'bits=20use'=20=E2=80=94=20s?= =?UTF-8?q?ave=20a=20per-command=20arg=20profile=20in=20.bitscmd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bits use [section] records reusable CLI args (INI-style: [common] plus per-command sections) so repeated commands stay short. The wrapper injects [common]+[] right after the action token before parsing, so build and the module commands (q/enter) both see it and explicit args still win. New bits_use.py (rewrite_argv/--rewrite0) + tests. --- bits | 21 ++++ bits_helpers/bits_use.py | 208 +++++++++++++++++++++++++++++++++++++++ tests/test_bits_use.py | 133 +++++++++++++++++++++++++ 3 files changed, 362 insertions(+) create mode 100644 bits_helpers/bits_use.py create mode 100644 tests/test_bits_use.py diff --git a/bits b/bits index adb49592..98ce0c72 100755 --- a/bits +++ b/bits @@ -5,6 +5,19 @@ BITSDIR="$(dirname "$0")" +# `.bitscmd` saved-arg profile (see `bits use`): inject the [common] + [] +# tokens right after the action, BEFORE anything is parsed, so both the build +# path and the module commands (q/enter/…, which read ARGV below) see them. +# Runs python only when a profile exists; `use` opts out; explicit args stay last +# (they win on single-value options). Safe no-op if the action isn't first. +if [[ -f .bitscmd ]]; then + _bits_prof=() + while IFS= read -r -d '' _bits_t; do _bits_prof+=("$_bits_t"); done \ + < <(BITS_SELF="${BITSDIR}" python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.bits_use import main; sys.exit(main())' --rewrite0 -- "$@") + [[ ${#_bits_prof[@]} -gt 0 ]] && set -- "${_bits_prof[@]}" + unset _bits_prof _bits_t +fi + ARGV=("$@"); ARGC=$# # ARGC must be a plain integer, not an array # Optional phase timing for diagnosing slow load/enter: set BITS_TIMING=1 to @@ -375,6 +388,14 @@ if [[ "${1:-}" == "store" ]]; then exec "$BITSDIR/bitsStore" "$@" fi +# `bits use …` → save/show/clear a per-command arg profile in ./.bitscmd, so +# repeated commands stay short (the injection at the top of this script consumes +# it). Handled here, before work-dir/module setup: it only touches ./.bitscmd. +if [[ "${1:-}" == "use" ]]; then + shift + BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.bits_use import main; sys.exit(main())' "$@" +fi + # `bits cvmfs-stage …` → producer-side CVMFS staging (ADR-0011). # Prepares a package into a staging S3 prefix with the canonical publisher and # names the catalog prepub must graft. Handled here, before work-dir/module diff --git a/bits_helpers/bits_use.py b/bits_helpers/bits_use.py new file mode 100644 index 00000000..3869ac4e --- /dev/null +++ b/bits_helpers/bits_use.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""`bits use` — save reusable command-line args in ``./.bitscmd`` so repeated +commands stay short. Distinct from ``.bitsrc`` (typed key=value settings): +``.bitscmd`` holds raw CLI tokens, structured by the command they apply to. + +A ``[common]`` section is injected into *every* command (e.g. ``--architecture``, +which ``build`` AND ``q``/``enter`` all need); a per-command section +(``[build]``, ``[q]``, …) adds command-specific args. Injected BEFORE the user's +own args, so those override single-value options. Example ``.bitscmd``:: + + [common] + --architecture x86_64-el9-gcc14-opt --defaults lcg::release::gcc14::opt + + [build] + --docker --docker-image reg/alma9 --sandbox off --reuse-from cvmfs::relaxed + +Prototype — runs standalone:: + + python3 -m bits_helpers.bits_use common --architecture x86_64-el9-gcc14-opt + python3 -m bits_helpers.bits_use build --docker --sandbox off + python3 -m bits_helpers.bits_use # show all sections + python3 -m bits_helpers.bits_use --clear [SECTION] +""" + +import os +import shlex +import sys + +PROFILE = ".bitscmd" +COMMON = "common" # section injected into every command ('global' alias) + + +def _join(tokens): + try: + return shlex.join(tokens) # Python 3.8+ + except AttributeError: # pragma: no cover + return " ".join(shlex.quote(t) for t in tokens) + + +def read_all(path=PROFILE): + """Parse the profile into an ordered ``{section: [tokens]}`` dict. + + ``[name]`` opens a section; lines before any header belong to ``common``; + ``#`` comments and blank lines are ignored. Each section's lines are joined + and shlex-split into tokens. + """ + sections, cur, buf = {}, COMMON, [] + if not os.path.exists(path): + return sections + + def _flush(): + if buf: + sections.setdefault(cur, []) + sections[cur] += shlex.split(" ".join(buf)) + try: + with open(path) as fh: + for raw in fh: + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("[") and line.endswith("]"): + _flush(); buf = [] + cur = line[1:-1].strip().lower() + cur = COMMON if cur in ("global", COMMON) else cur + sections.setdefault(cur, []) + continue + buf.append(line) + _flush() + except (OSError, ValueError): + # ValueError: a malformed token (e.g. an unbalanced quote) in the + # profile. Fail safe — ignore the profile rather than crash a build. + return {} + return sections + + +def write_section(section, tokens, path=PROFILE): + """Replace *section* with *tokens*, preserving the other sections.""" + section = COMMON if section in ("global", COMMON) else section.lower() + sections = read_all(path) + sections[section] = list(tokens) + _write_all(sections, path) + return path + + +def clear_section(section=None, path=PROFILE): + """Clear one section, or the whole profile when *section* is None.""" + if section is None: + try: + os.unlink(path); return True + except OSError: + return False + sections = read_all(path) + section = COMMON if section in ("global", COMMON) else section.lower() + if section in sections: + del sections[section] + _write_all(sections, path) + return True + return False + + +def _write_all(sections, path=PROFILE): + order = [COMMON] + [s for s in sections if s != COMMON] + with open(path, "w") as fh: + for s in order: + toks = sections.get(s) + if not toks: + continue + fh.write("[%s]\n%s\n\n" % (s, _join(toks))) + + +def merged_argv(command, user_args, path=PROFILE): + """Args to run for *command*: ``[common]`` then ``[command]`` then the + user's own args (which come last and win on single-value options).""" + sec = read_all(path) + return sec.get(COMMON, []) + sec.get((command or "").lower(), []) + list(user_args) + + +# Top-level flags that may precede the action (from the root argparse parser); +# skipped when locating the action token. `use` never injects into itself. +TOP_FLAGS = {"-d", "--debug", "-n", "--dry-run"} +NO_INJECT = {"use"} + + +def _find_action(argv): + """Index of the action token in *argv* (first non-top-flag word), or None.""" + for i, tok in enumerate(argv): + if tok in TOP_FLAGS: + continue + if tok.startswith("-"): + return None # an option before any action → leave as-is + return i + return None + + +def rewrite_argv(argv, path=PROFILE): + """Return *argv* with the ``.bitscmd`` profile injected right after the action + token: ``[common]`` for every command, plus ``[]`` for that action. + A no-op when there is no profile, no action, or the action opts out + (``use``). This is the single entry point the wrapper calls at startup. + """ + argv = list(argv) + sec = read_all(path) + if not sec: + return argv + ai = _find_action(argv) + if ai is None: + return argv + action = argv[ai].lower() + if action in NO_INJECT: + return argv + inject = sec.get(COMMON, []) + sec.get(action, []) + if not inject: + return argv + return argv[:ai + 1] + inject + argv[ai + 1:] + + +# ── CLI ────────────────────────────────────────────────────────────────────── + +def _show(path=PROFILE): + sec = read_all(path) + if not sec: + print("no .bitscmd in %s" % os.getcwd()); return 0 + for s, toks in sec.items(): + if toks: + print("[%s] %s" % (s, _join(toks))) + return 0 + + +def main(argv=None): + argv = list(sys.argv[1:] if argv is None else argv) + # Wrapper hook: emit the profile-injected argv, NUL-separated, for the args + # after '--'. `mapfile -t -d '' arr < <(bits_use --rewrite0 -- "$@")`. + if argv and argv[0] == "--rewrite0": + rest = argv[2:] if len(argv) > 1 and argv[1] == "--" else argv[1:] + out = rewrite_argv(rest) + if out != rest: # injection happened → tell the user + n = len(out) - len(rest) + sys.stderr.write("using .bitscmd (+%d arg%s)\n" % (n, "" if n == 1 else "s")) + # NUL-terminate EVERY token (trailing NUL included) so the wrapper's + # `while read -d ''` loop captures the final token — an unterminated last + # field is assigned but the loop exits before appending it (dropping it). + sys.stdout.write("".join(t + "\0" for t in out)) + return 0 + if not argv: + return _show() + if argv[0] in ("--clear", "clear"): + sect = argv[1] if len(argv) > 1 else None + ok = clear_section(sect) + print(("cleared %s" % (sect or "all")) if ok else "nothing to clear") + return 0 + # Optional leading section name (a bare word); otherwise default to common. + if argv[0] and not argv[0].startswith("-"): + section, rest = argv[0], argv[1:] + else: + section, rest = COMMON, argv + if not rest: + print("no args given for [%s]" % section); return 1 + write_section(section, rest) + print("saved to [%s] in %s: %s" % ( + COMMON if section in ("global", COMMON) else section.lower(), + PROFILE, _join(rest))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_bits_use.py b/tests/test_bits_use.py new file mode 100644 index 00000000..bec8583c --- /dev/null +++ b/tests/test_bits_use.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Tests for bits_helpers/bits_use — the .bitscmd saved-arg-profile.""" + +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from bits_helpers import bits_use as U + + +class BitsUseTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.p = os.path.join(self.tmp, ".bitscmd") + self._cwd = os.getcwd() + + def tearDown(self): + import shutil + os.chdir(self._cwd) + shutil.rmtree(self.tmp, ignore_errors=True) + + # ── sections round-trip ────────────────────────────────────────────────── + def test_write_read_sections(self): + U.write_section("common", ["--architecture", "x86_64-el9-gcc14-opt"], self.p) + U.write_section("build", ["--docker", "--sandbox", "off"], self.p) + sec = U.read_all(self.p) + self.assertEqual(sec["common"], ["--architecture", "x86_64-el9-gcc14-opt"]) + self.assertEqual(sec["build"], ["--docker", "--sandbox", "off"]) + + def test_global_aliases_common(self): + U.write_section("global", ["-a", "X"], self.p) + self.assertEqual(U.read_all(self.p).get("common"), ["-a", "X"]) + + def test_headerless_preamble_is_common(self): + with open(self.p, "w") as fh: + fh.write("--architecture X\n[build]\n--docker\n") + sec = U.read_all(self.p) + self.assertEqual(sec["common"], ["--architecture", "X"]) + self.assertEqual(sec["build"], ["--docker"]) + + def test_quoting_roundtrips(self): + U.write_section("build", ["--env", "A=b c", "--x", "a::b"], self.p) + self.assertEqual(U.read_all(self.p)["build"], ["--env", "A=b c", "--x", "a::b"]) + + def test_clear_one_and_all(self): + U.write_section("common", ["-a", "X"], self.p) + U.write_section("build", ["--docker"], self.p) + self.assertTrue(U.clear_section("build", self.p)) + self.assertNotIn("build", U.read_all(self.p)) + self.assertIn("common", U.read_all(self.p)) + self.assertTrue(U.clear_section(None, self.p)) + self.assertFalse(os.path.exists(self.p)) + + # ── merge / rewrite ────────────────────────────────────────────────────── + def _profile(self): + U.write_section("common", ["--architecture", "A", "--defaults", "D"], self.p) + U.write_section("build", ["--docker", "--reuse-from", "cvmfs::relaxed"], self.p) + + def test_merged_build_gets_common_and_section(self): + self._profile() + self.assertEqual( + U.merged_argv("build", ["xrootd"], self.p), + ["--architecture", "A", "--defaults", "D", + "--docker", "--reuse-from", "cvmfs::relaxed", "xrootd"]) + + def test_merged_q_gets_common_only(self): + self._profile() + self.assertEqual(U.merged_argv("q", ["ROOT"], self.p), + ["--architecture", "A", "--defaults", "D", "ROOT"]) + + def test_rewrite_injects_after_action(self): + self._profile() + self.assertEqual( + U.rewrite_argv(["build", "xrootd"], self.p), + ["build", "--architecture", "A", "--defaults", "D", + "--docker", "--reuse-from", "cvmfs::relaxed", "xrootd"]) + + def test_rewrite_skips_top_flags(self): + self._profile() + out = U.rewrite_argv(["-d", "build", "xrootd"], self.p) + self.assertEqual(out[:2], ["-d", "build"]) + self.assertIn("--docker", out) + + def test_rewrite_use_opts_out(self): + self._profile() + self.assertEqual(U.rewrite_argv(["use", "build", "--docker"], self.p), + ["use", "build", "--docker"]) + + def test_rewrite_no_profile_is_noop(self): + self.assertEqual(U.rewrite_argv(["build", "x"], self.p), ["build", "x"]) + + def test_rewrite_user_arg_comes_after_injected(self): + # A user's explicit --architecture must land AFTER the injected one so + # argparse last-wins lets it override. + self._profile() + out = U.rewrite_argv(["build", "--architecture", "B", "x"], self.p) + self.assertLess(out.index("A"), out.index("B")) + + def test_rewrite_option_before_action_is_noop(self): + self._profile() + self.assertEqual(U.rewrite_argv(["--weird", "x"], self.p), + ["--weird", "x"]) + + # ── --rewrite0 wire contract (wrapper reads with `while read -d ''`) ────── + def test_rewrite0_nul_terminated_last_token_survives(self): + import io, contextlib + self._profile() + os.chdir(self.tmp) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + U.main(["--rewrite0", "--", "build", "xrootd"]) + s = buf.getvalue() + self.assertTrue(s.endswith("\0")) # trailing NUL present + # `while read -d ''` keeps only fully-terminated fields; splitting on NUL + # and dropping the trailing empty must reproduce every token, incl. last. + toks = s.split("\0")[:-1] + self.assertEqual(toks[0], "build") + self.assertEqual(toks[-1], "xrootd") + + def test_malformed_profile_ignored_not_crash(self): + with open(self.p, "w") as fh: + fh.write('[build]\n--foo "unbalanced\n') # bad quoting + self.assertEqual(U.read_all(self.p), {}) # fail safe, no raise + self.assertEqual(U.rewrite_argv(["build", "x"], self.p), ["build", "x"]) + + +if __name__ == "__main__": + unittest.main() From 67cc1d8994fdbbeb99f9457924d84efa103c81a2 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 17:17:56 +0200 Subject: [PATCH 29/30] cli: add 'bits cvmfs' inspect; gate bits-use injection to arch-aware cmds --- bits | 9 ++ bits_helpers/bits_use.py | 38 ++++-- bits_helpers/cvmfs_inspect.py | 242 ++++++++++++++++++++++++++++++++++ tests/test_bits_use.py | 10 +- tests/test_cvmfs_inspect.py | 91 +++++++++++++ 5 files changed, 375 insertions(+), 15 deletions(-) create mode 100644 bits_helpers/cvmfs_inspect.py create mode 100644 tests/test_cvmfs_inspect.py diff --git a/bits b/bits index 98ce0c72..30689662 100755 --- a/bits +++ b/bits @@ -396,6 +396,15 @@ if [[ "${1:-}" == "use" ]]; then BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.bits_use import main; sys.exit(main())' "$@" fi +# `bits cvmfs {platforms,show,summary} …` → read-only inspection of a deployed +# CVMFS bits tree: platforms + host compatibility, a package's build/provenance +# from .meta.json (no jq), and a per-build_id summary. Reads the tree only, so it +# needs no work dir; distinct from cvmfs-stage/cvmfs-publish (producer side). +if [[ "${1:-}" == "cvmfs" ]]; then + shift + BITS_SELF="${BITSDIR}" exec python3 -c 'import os,sys; sys.path.insert(0, os.environ.get("BITS_SELF","")); from bits_helpers.cvmfs_inspect import main; sys.exit(main())' "$@" +fi + # `bits cvmfs-stage …` → producer-side CVMFS staging (ADR-0011). # Prepares a package into a staging S3 prefix with the canonical publisher and # names the catalog prepub must graft. Handled here, before work-dir/module diff --git a/bits_helpers/bits_use.py b/bits_helpers/bits_use.py index 3869ac4e..22c82e03 100644 --- a/bits_helpers/bits_use.py +++ b/bits_helpers/bits_use.py @@ -5,16 +5,18 @@ commands stay short. Distinct from ``.bitsrc`` (typed key=value settings): ``.bitscmd`` holds raw CLI tokens, structured by the command they apply to. -A ``[common]`` section is injected into *every* command (e.g. ``--architecture``, -which ``build`` AND ``q``/``enter`` all need); a per-command section -(``[build]``, ``[q]``, …) adds command-specific args. Injected BEFORE the user's -own args, so those override single-value options. Example ``.bitscmd``:: +A ``[common]`` section is injected into arch-aware commands (put only broadly +accepted args here, i.e. ``--architecture``, which ``build`` AND ``q``/``enter``/ +``clean`` all take); a per-command section (``[build]``, ``[q]``, …) adds args +that command accepts — e.g. ``--defaults`` belongs in ``[build]``, NOT +``[common]`` (module commands and ``clean``/``import`` don't accept it). Injected +BEFORE the user's own args, so those override single-value options. Example:: [common] - --architecture x86_64-el9-gcc14-opt --defaults lcg::release::gcc14::opt + --architecture x86_64-el9-gcc14-opt [build] - --docker --docker-image reg/alma9 --sandbox off --reuse-from cvmfs::relaxed + --defaults lcg::release::gcc14::opt --docker --sandbox off --reuse-from cvmfs::relaxed Prototype — runs standalone:: @@ -118,9 +120,20 @@ def merged_argv(command, user_args, path=PROFILE): # Top-level flags that may precede the action (from the root argparse parser); -# skipped when locating the action token. `use` never injects into itself. +# skipped when locating the action token. TOP_FLAGS = {"-d", "--debug", "-n", "--dry-run"} -NO_INJECT = {"use"} + +# The profile is injected ONLY into arch-aware commands (they all accept +# `--architecture`, the intended `[common]` content). Meta commands (use, cvmfs, +# store, version, help, cvmfs-stage/publish) and `verify` (accepts neither +# --architecture nor --defaults) take a different option set and are excluded. +# Note: several of these accept --architecture but NOT --defaults, so --defaults +# belongs in per-command sections ([build], …), never in [common]. +INJECT_ACTIONS = { + "build", "deps", "doctor", "status", "clean", "cleanup", "gc", + "import", "publish", "certify", "compliance", + "q", "query", "enter", "setenv", "printenv", "load", "unload", +} def _find_action(argv): @@ -136,9 +149,10 @@ def _find_action(argv): def rewrite_argv(argv, path=PROFILE): """Return *argv* with the ``.bitscmd`` profile injected right after the action - token: ``[common]`` for every command, plus ``[]`` for that action. - A no-op when there is no profile, no action, or the action opts out - (``use``). This is the single entry point the wrapper calls at startup. + token: ``[common]`` plus ``[]``, for arch-aware actions only + (``INJECT_ACTIONS``). A no-op when there is no profile, no action, or the + action is a meta command (``use``/``cvmfs``/``store``/…). This is the single + entry point the wrapper calls at startup. """ argv = list(argv) sec = read_all(path) @@ -148,7 +162,7 @@ def rewrite_argv(argv, path=PROFILE): if ai is None: return argv action = argv[ai].lower() - if action in NO_INJECT: + if action not in INJECT_ACTIONS: return argv inject = sec.get(COMMON, []) + sec.get(action, []) if not inject: diff --git a/bits_helpers/cvmfs_inspect.py b/bits_helpers/cvmfs_inspect.py new file mode 100644 index 00000000..f00f92dd --- /dev/null +++ b/bits_helpers/cvmfs_inspect.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Read-only inspection of a deployed CVMFS bits tree: discover platforms, mark +host compatibility, and present a package's build/provenance from .meta.json +without jq. Prototype — runs standalone as:: + + python3 -m bits_helpers.cvmfs_inspect platforms --cvmfs + python3 -m bits_helpers.cvmfs_inspect show Davix --cvmfs --arch --deps + python3 -m bits_helpers.cvmfs_inspect summary --cvmfs --arch + +`` is the directory holding the per-arch trees, i.e. it contains +``/Packages///.meta.json``. +""" + +import argparse +import json +import os +import sys + +try: + from bits_helpers.utilities import ( + detectArchComponents, arch_machine_token, arch_distro_token) +except Exception: # standalone / partial import fallback + detectArchComponents = None + arch_machine_token = arch_distro_token = lambda s: None + +# Compatibility marks (state -> glyph); ASCII-safe fallbacks are easy to add. +_MARK = {"native": "✓", "container": "⚠", "incompatible": "✗"} + + +# ── discovery ──────────────────────────────────────────────────────────────── + +def list_platforms(cvmfs_root): + """ dirs under *cvmfs_root* that actually hold a Packages/ tree.""" + out = [] + try: + for name in sorted(os.listdir(cvmfs_root)): + if os.path.isdir(os.path.join(cvmfs_root, name, "Packages")): + out.append(name) + except OSError: + pass + return out + + +def _packages_root(cvmfs_root, arch): + return os.path.join(cvmfs_root, arch, "Packages") + + +def list_packages(cvmfs_root, arch): + """{pkg: [verrev, ...]} under /Packages (sorted).""" + root = _packages_root(cvmfs_root, arch) + out = {} + try: + for pkg in sorted(os.listdir(root)): + pdir = os.path.join(root, pkg) + if not os.path.isdir(pdir): + continue + out[pkg] = sorted(v for v in os.listdir(pdir) + if os.path.isdir(os.path.join(pdir, v))) + except OSError: + pass + return out + + +def resolve_verrev(cvmfs_root, arch, pkg, ver=None): + """Pick a verrev: prefix-match *ver* if given, else the newest present.""" + vers = list_packages(cvmfs_root, arch).get(pkg, []) + if not vers: + return None + if ver: + for v in vers: + if v == ver or v.startswith(ver + "-"): + return v + return None + return vers[-1] + + +def read_meta(cvmfs_root, arch, pkg, verrev): + with open(os.path.join(_packages_root(cvmfs_root, arch), pkg, verrev, + ".meta.json")) as fh: + return json.load(fh) + + +# ── host compatibility (advisory, three-state) ─────────────────────────────── + +def _norm_machine(tok): + return (tok or "").replace("-", "_") + + +def classify_platform(arch, host=None): + """(state, note) with state in {native, container, incompatible}. + + Honest and layered: machine must match (else incompatible); a differing distro + token means the binaries were built for another OS/toolchain and want a + matching container (the gcc15-in-alma9 lesson), not the bare host. + """ + host = host or (detectArchComponents() if detectArchComponents else {}) + h_machine = _norm_machine(host.get("_machine") or host.get("machine")) + h_os = host.get("os") or "" + p_machine = _norm_machine(arch_machine_token(arch)) + p_os = arch_distro_token(arch) or "" + if p_machine and h_machine and p_machine != h_machine: + return "incompatible", "needs %s (host is %s)" % (p_machine, h_machine) + if p_os and h_os and p_os != h_os: + return "container", ("built for %s; host is %s — run in a matching image" + % (p_os, h_os)) + return "native", "machine/OS match host" + + +# ── presentation ───────────────────────────────────────────────────────────── + +def format_meta(meta, deps=False, provenance_only=False): + pkg = meta.get("package", {}) or {} + L = ["%s %s-%s" % (pkg.get("name", "?"), pkg.get("version", "?"), + pkg.get("revision", "?"))] + L.append(" hash: %s" % pkg.get("hash", "")) + L.append(" architecture: %s" % meta.get("architecture", "")) + L.append(" abi_tag: %s" % meta.get("abi_tag", "")) + L.append(" build_id: %s" % meta.get("build_id", "")) + L.append(" provenance: %s" % meta.get("provenance", "")) + L.append(" reuse_policy: %s" % meta.get("reuse_policy", "")) + L.append(" defaults: %s" % " ".join(meta.get("defaults", []) or [])) + L.append(" bits: %s dist %s" + % (meta.get("bits_version", ""), + ((meta.get("dist") or {}).get("commit", "") or "")[:12])) + if provenance_only: + return "\n".join(L) + if deps: + d = meta.get("dependencies", {}) or {} + for scope in ("direct", "recursive"): + sd = d.get(scope, {}) or {} + for kind in ("build", "runtime"): + items = sd.get(kind, []) or [] + if items: + L.append(" %s %s (%d):" % (scope, kind, len(items))) + for it in items: + L.append(" - %s %s-%s" % (it.get("name"), + it.get("version"), it.get("revision"))) + return "\n".join(L) + + +# ── summary ────────────────────────────────────────────────────────────────── + +def summarize(cvmfs_root, arch): + """(pkgs, build_ids): pkgs={pkg:[verrev]}, build_ids={bid:[pkg/verrev]}.""" + pkgs = list_packages(cvmfs_root, arch) + build_ids = {} + for pkg, vers in pkgs.items(): + for v in vers: + try: + m = read_meta(cvmfs_root, arch, pkg, v) + except (OSError, ValueError): + continue + build_ids.setdefault(m.get("build_id") or "(none)", []).append( + "%s/%s" % (pkg, v)) + return pkgs, build_ids + + +# ── CLI ────────────────────────────────────────────────────────────────────── + +def _cmd_platforms(a): + plats = list_platforms(a.cvmfs) + if a.json: + print(json.dumps([{"arch": p, "state": classify_platform(p)[0]} + for p in plats], indent=2)) + return 0 + if not plats: + print("no platforms found under %s" % a.cvmfs); return 1 + for p in plats: + state, note = classify_platform(p) + print(" %s %-32s %s" % (_MARK.get(state, "?"), p, note)) + print("\n %s native %s needs container %s incompatible" + % (_MARK["native"], _MARK["container"], _MARK["incompatible"])) + return 0 + + +def _cmd_show(a): + arch = a.arch or (list_platforms(a.cvmfs) or [None])[0] + if not arch: + print("no arch (pass --arch)"); return 1 + pkg, _, ver = a.package.partition("/") + verrev = resolve_verrev(a.cvmfs, arch, pkg, ver or None) + if not verrev: + print("%s%s not found under %s/%s/Packages" + % (pkg, "/" + ver if ver else "", a.cvmfs, arch)); return 1 + try: + meta = read_meta(a.cvmfs, arch, pkg, verrev) + except (OSError, ValueError) as exc: + print("cannot read %s/%s/%s/.meta.json: %s" % (arch, pkg, verrev, exc)) + return 1 + if a.json: + print(json.dumps(meta, indent=2)); return 0 + print(format_meta(meta, deps=a.deps, provenance_only=a.provenance)) + return 0 + + +def _cmd_summary(a): + arch = a.arch or (list_platforms(a.cvmfs) or [None])[0] + if not arch: + print("no arch (pass --arch)"); return 1 + pkgs, build_ids = summarize(a.cvmfs, arch) + if a.json: + print(json.dumps({"arch": arch, "packages": len(pkgs), + "build_ids": {b: len(v) for b, v in build_ids.items()}}, + indent=2)); return 0 + print("Platform %s: %d packages, %d build_id(s)" + % (arch, len(pkgs), len(build_ids))) + for bid, members in sorted(build_ids.items()): + print(" %s (%d packages)" % (bid, len(members))) + return 0 + + +def main(argv=None): + # Shared options live on a parent parser so they work AFTER the subcommand + # too (`bits cvmfs platforms --cvmfs ROOT`), not only before it. + common = argparse.ArgumentParser(add_help=False) + common.add_argument("--cvmfs", required=True, + help="root holding /Packages///.meta.json") + common.add_argument("--json", action="store_true", help="machine-readable output") + + ap = argparse.ArgumentParser(prog="bits cvmfs", + description="Inspect a deployed CVMFS bits tree.") + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("platforms", parents=[common], + help="list platforms + host compatibility") + sp = sub.add_parser("show", parents=[common], + help="print a package's build/provenance") + sp.add_argument("package", help="PKG or PKG/VERSION") + sp.add_argument("--arch", help="platform (default: first found)") + sp.add_argument("--deps", action="store_true", help="include dependency tree") + sp.add_argument("--provenance", action="store_true", help="provenance fields only") + ss = sub.add_parser("summary", parents=[common], + help="per-platform package + build_id summary") + ss.add_argument("--arch", help="platform (default: first found)") + a = ap.parse_args(argv) + return {"platforms": _cmd_platforms, "show": _cmd_show, + "summary": _cmd_summary}[a.cmd](a) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_bits_use.py b/tests/test_bits_use.py index bec8583c..0270e339 100644 --- a/tests/test_bits_use.py +++ b/tests/test_bits_use.py @@ -86,10 +86,14 @@ def test_rewrite_skips_top_flags(self): self.assertEqual(out[:2], ["-d", "build"]) self.assertIn("--docker", out) - def test_rewrite_use_opts_out(self): + def test_rewrite_meta_commands_opt_out(self): + # Meta commands (use/cvmfs/store/version) take a different option set and + # must NOT receive the profile, or their argparse would reject it. self._profile() - self.assertEqual(U.rewrite_argv(["use", "build", "--docker"], self.p), - ["use", "build", "--docker"]) + for meta in (["use", "build", "--docker"], ["cvmfs", "platforms"], + ["store", "ls"], ["version"], + ["verify", "--from-manifest", "m.json"]): # accepts neither flag + self.assertEqual(U.rewrite_argv(list(meta), self.p), list(meta)) def test_rewrite_no_profile_is_noop(self): self.assertEqual(U.rewrite_argv(["build", "x"], self.p), ["build", "x"]) diff --git a/tests/test_cvmfs_inspect.py b/tests/test_cvmfs_inspect.py new file mode 100644 index 00000000..8b11a068 --- /dev/null +++ b/tests/test_cvmfs_inspect.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: 2015-2026 CERN +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Tests for bits_helpers/cvmfs_inspect — read-only CVMFS tree inspection.""" + +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from bits_helpers import cvmfs_inspect as I + + +def _meta(arch, name, ver, rev, build_id="bid-1", provenance="pure", deps=None): + return { + "architecture": arch, "abi_tag": arch, "build_id": build_id, + "defaults": ["lcg", "release"], "reuse_policy": "strict", + "provenance": provenance, "bits_version": "0.1", "dist": {"commit": "c" * 40}, + "package": {"name": name, "version": ver, "revision": rev, "hash": "h" + name}, + "dependencies": deps or {"direct": {"build": [], "runtime": []}, + "recursive": {"build": [], "runtime": []}}, + } + + +class CvmfsInspectTest(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self._pkg("x86_64-el9-gcc14-opt", "Davix", "0.8.10-1", _meta( + "x86_64-el9-gcc14-opt", "Davix", "0.8.10", "1", + deps={"direct": {"build": [{"name": "bits-recipe-tools", "version": "0.0.32", "revision": "1"}], + "runtime": [{"name": "Boost", "version": "1.90.0", "revision": "1"}]}, + "recursive": {"build": [], "runtime": []}})) + self._pkg("x86_64-el9-gcc14-opt", "Boost", "1.90.0-1", + _meta("x86_64-el9-gcc14-opt", "Boost", "1.90.0", "1")) + self._pkg("aarch64-el9-gcc14-opt", "Davix", "0.8.10-1", + _meta("aarch64-el9-gcc14-opt", "Davix", "0.8.10", "1", build_id="bid-arm")) + + def tearDown(self): + import shutil + shutil.rmtree(self.root, ignore_errors=True) + + def _pkg(self, arch, name, verrev, meta): + d = os.path.join(self.root, arch, "Packages", name, verrev) + os.makedirs(d) + with open(os.path.join(d, ".meta.json"), "w") as fh: + json.dump(meta, fh) + + def test_list_platforms(self): + self.assertEqual(I.list_platforms(self.root), + ["aarch64-el9-gcc14-opt", "x86_64-el9-gcc14-opt"]) + + def test_list_packages_and_resolve(self): + pk = I.list_packages(self.root, "x86_64-el9-gcc14-opt") + self.assertEqual(sorted(pk), ["Boost", "Davix"]) + self.assertEqual(I.resolve_verrev(self.root, "x86_64-el9-gcc14-opt", "Davix"), + "0.8.10-1") + self.assertEqual( + I.resolve_verrev(self.root, "x86_64-el9-gcc14-opt", "Davix", "0.8.10"), + "0.8.10-1") + self.assertIsNone(I.resolve_verrev(self.root, "x86_64-el9-gcc14-opt", "Nope")) + + def test_classify_three_states(self): + host = {"os": "el9", "_machine": "x86_64", "machine": "x86_64"} + self.assertEqual(I.classify_platform("x86_64-el9-gcc14-opt", host)[0], "native") + # right machine, wrong OS -> needs a container + self.assertEqual(I.classify_platform("x86_64-el8-gcc14-opt", host)[0], "container") + # wrong machine -> incompatible + self.assertEqual(I.classify_platform("aarch64-el9-gcc14-opt", host)[0], "incompatible") + + def test_format_meta_deps(self): + m = I.read_meta(self.root, "x86_64-el9-gcc14-opt", "Davix", "0.8.10-1") + out = I.format_meta(m, deps=True) + self.assertIn("Davix 0.8.10-1", out) + self.assertIn("build_id:", out) + self.assertIn("provenance: pure", out) + self.assertIn("bits-recipe-tools 0.0.32-1", out) # dep tree present + # provenance_only drops the dep tree + self.assertNotIn("bits-recipe-tools", I.format_meta(m, provenance_only=True)) + + def test_summarize_groups_by_build_id(self): + pkgs, bids = I.summarize(self.root, "x86_64-el9-gcc14-opt") + self.assertEqual(sorted(pkgs), ["Boost", "Davix"]) + self.assertEqual(sorted(bids), ["bid-1"]) + self.assertEqual(len(bids["bid-1"]), 2) + + +if __name__ == "__main__": + unittest.main() From 7170b4333506642f83256637c52e4d548cc7e700 Mon Sep 17 00:00:00 2001 From: Predrag Buncic Date: Tue, 25 Aug 2026 17:49:18 +0200 Subject: [PATCH 30/30] cli: bits cvmfs show - list all platforms; handle legacy trees without .meta.json --- bits_helpers/cvmfs_inspect.py | 99 +++++++++++++++++++++++++--- tests/test_cvmfs_inspect.py | 117 ++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 10 deletions(-) diff --git a/bits_helpers/cvmfs_inspect.py b/bits_helpers/cvmfs_inspect.py index f00f92dd..67239425 100644 --- a/bits_helpers/cvmfs_inspect.py +++ b/bits_helpers/cvmfs_inspect.py @@ -82,6 +82,34 @@ def read_meta(cvmfs_root, arch, pkg, verrev): return json.load(fh) +def _legacy_meta(arch, pkg, verrev): + """Minimal record inferred from the path for a pre-bits tree with no + .meta.json. Best-effort guess from directory names, marked as legacy; only + name, architecture, and the verrev dir (kept in `version`) are real — + everything else is unknown/blank.""" + return { + "architecture": arch, "abi_tag": "", "build_id": "", + "provenance": "legacy (no .meta.json)", "reuse_policy": "", + "defaults": [], "bits_version": "", "dist": {}, + "package": {"name": pkg, "version": verrev, "revision": "", "hash": ""}, + "dependencies": {"direct": {"build": [], "runtime": []}, + "recursive": {"build": [], "runtime": []}}, + "_legacy": True, + } + + +def meta_or_legacy(cvmfs_root, arch, pkg, verrev): + """Read .meta.json; when it is simply absent, synthesize a legacy record from + the path. `lexists` so only a truly-missing file is legacy: a present-but- + unreadable file, or a dangling symlink, still raises (open fails) and is + surfaced as an error rather than masked as legacy.""" + path = os.path.join(_packages_root(cvmfs_root, arch), pkg, verrev, ".meta.json") + if not os.path.lexists(path): + return _legacy_meta(arch, pkg, verrev) + with open(path) as fh: + return json.load(fh) + + # ── host compatibility (advisory, three-state) ─────────────────────────────── def _norm_machine(tok): @@ -110,10 +138,15 @@ def classify_platform(arch, host=None): # ── presentation ───────────────────────────────────────────────────────────── +def _verrev(pkg): + """version-revision, dropping the dash when there is no revision (legacy).""" + ver, rev = pkg.get("version", "?"), pkg.get("revision", "") + return "%s-%s" % (ver, rev) if rev else ver + + def format_meta(meta, deps=False, provenance_only=False): pkg = meta.get("package", {}) or {} - L = ["%s %s-%s" % (pkg.get("name", "?"), pkg.get("version", "?"), - pkg.get("revision", "?"))] + L = ["%s %s" % (pkg.get("name", "?"), _verrev(pkg))] L.append(" hash: %s" % pkg.get("hash", "")) L.append(" architecture: %s" % meta.get("architecture", "")) L.append(" abi_tag: %s" % meta.get("abi_tag", "")) @@ -175,26 +208,72 @@ def _cmd_platforms(a): return 0 -def _cmd_show(a): - arch = a.arch or (list_platforms(a.cvmfs) or [None])[0] - if not arch: - print("no arch (pass --arch)"); return 1 - pkg, _, ver = a.package.partition("/") - verrev = resolve_verrev(a.cvmfs, arch, pkg, ver or None) +def _show_one(a, arch, pkg, ver, verrev=None): + """Full build/provenance detail for one platform.""" + verrev = verrev or resolve_verrev(a.cvmfs, arch, pkg, ver) if not verrev: print("%s%s not found under %s/%s/Packages" % (pkg, "/" + ver if ver else "", a.cvmfs, arch)); return 1 try: - meta = read_meta(a.cvmfs, arch, pkg, verrev) + meta = meta_or_legacy(a.cvmfs, arch, pkg, verrev) except (OSError, ValueError) as exc: print("cannot read %s/%s/%s/.meta.json: %s" % (arch, pkg, verrev, exc)) return 1 if a.json: print(json.dumps(meta, indent=2)); return 0 print(format_meta(meta, deps=a.deps, provenance_only=a.provenance)) + if meta.get("_legacy"): + print(" (legacy tree: no .meta.json — fields inferred from the path)") return 0 +def _show_across(a, pkg, hits): + """One compact line per platform that has *pkg*; --arch drills into detail.""" + rows = [] + for arch, verrev in hits: + try: + m = meta_or_legacy(a.cvmfs, arch, pkg, verrev) + except (OSError, ValueError): + # Present but unreadable/corrupt — keep it visible, don't drop it. + rows.append((arch, verrev, "", "", "unreadable .meta.json")) + continue + p = m.get("package", {}) or {} + rows.append((arch, _verrev(p), + m.get("build_id", "") or "", (p.get("hash", "") or "")[:8], + m.get("provenance", "") or "")) + if a.json: + print(json.dumps([{"arch": r[0], "verrev": r[1], "build_id": r[2], + "hash": r[3], "provenance": r[4]} for r in rows], indent=2)) + return 0 + print("%s — %d platform%s" % (pkg, len(rows), "" if len(rows) == 1 else "s")) + for arch, verrev, bid, h, prov in rows: + print(" %-30s %-10s %-38s %-8s %s" % (arch, verrev, bid, h, prov)) + print(" (add --arch for full build/provenance + --deps)") + return 0 + + +def _cmd_show(a): + pkg, _, ver = a.package.partition("/") + ver = ver or None + if a.arch: + return _show_one(a, a.arch, pkg, ver) + # No --arch: don't silently pick the first platform — show the package on + # every platform that has it, so provenance across the tree is visible. + hits = [] + for arch in list_platforms(a.cvmfs): + verrev = resolve_verrev(a.cvmfs, arch, pkg, ver) + if verrev: + hits.append((arch, verrev)) + if not hits: + print("%s%s not found on any platform under %s" + % (pkg, "/" + ver if ver else "", a.cvmfs)); return 1 + # JSON keeps one stable shape for no-arch — always the per-platform list, + # regardless of hit count. Text shows full detail when there's a single hit. + if not a.json and len(hits) == 1: + return _show_one(a, hits[0][0], pkg, ver, hits[0][1]) + return _show_across(a, pkg, hits) + + def _cmd_summary(a): arch = a.arch or (list_platforms(a.cvmfs) or [None])[0] if not arch: @@ -227,7 +306,7 @@ def main(argv=None): sp = sub.add_parser("show", parents=[common], help="print a package's build/provenance") sp.add_argument("package", help="PKG or PKG/VERSION") - sp.add_argument("--arch", help="platform (default: first found)") + sp.add_argument("--arch", help="platform (default: all platforms that have it)") sp.add_argument("--deps", action="store_true", help="include dependency tree") sp.add_argument("--provenance", action="store_true", help="provenance fields only") ss = sub.add_parser("summary", parents=[common], diff --git a/tests/test_cvmfs_inspect.py b/tests/test_cvmfs_inspect.py index 8b11a068..5ff1e35b 100644 --- a/tests/test_cvmfs_inspect.py +++ b/tests/test_cvmfs_inspect.py @@ -86,6 +86,123 @@ def test_summarize_groups_by_build_id(self): self.assertEqual(sorted(bids), ["bid-1"]) self.assertEqual(len(bids["bid-1"]), 2) + # ── `show` platform selection ──────────────────────────────────────────── + class _Args: + def __init__(self, root, package, arch=None, deps=False, + provenance=False, json=False): + self.cvmfs, self.package, self.arch = root, package, arch + self.deps, self.provenance, self.json = deps, provenance, json + + def test_show_no_arch_lists_every_platform(self): + # Davix exists on both platforms -> compact multi-platform view (JSON). + import io, contextlib + a = self._Args(self.root, "Davix", json=True) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + got = json.loads(buf.getvalue()) + self.assertEqual(sorted(r["arch"] for r in got), + ["aarch64-el9-gcc14-opt", "x86_64-el9-gcc14-opt"]) + self.assertTrue(all(r["verrev"] == "0.8.10-1" for r in got)) + + def test_show_no_arch_single_hit_is_full_detail(self): + # Boost is on one platform only -> falls through to full detail. + import io, contextlib + a = self._Args(self.root, "Boost") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + out = buf.getvalue() + self.assertIn("Boost 1.90.0-1", out) + self.assertIn("build_id:", out) # detail block, not a table row + + def test_show_with_arch_is_full_detail(self): + import io, contextlib + a = self._Args(self.root, "Davix", arch="aarch64-el9-gcc14-opt") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + self.assertIn("bid-arm", buf.getvalue()) + + def test_show_not_found_anywhere(self): + import io, contextlib + a = self._Args(self.root, "Nope") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 1) + self.assertIn("not found on any platform", buf.getvalue()) + + # ── legacy (pre-bits) trees without .meta.json ─────────────────────────── + def _legacy_pkg(self, arch, name, verrev): + # a package dir with NO .meta.json, like /cvmfs/alice.cern.ch/... + os.makedirs(os.path.join(self.root, arch, "Packages", name, verrev)) + + def test_meta_or_legacy_absent_is_synthesized(self): + self._legacy_pkg("el5-x86_64", "xrootd", "v3.3.3") + m = I.meta_or_legacy(self.root, "el5-x86_64", "xrootd", "v3.3.3") + self.assertTrue(m.get("_legacy")) + self.assertEqual(m["package"]["name"], "xrootd") + self.assertEqual(m["package"]["version"], "v3.3.3") + self.assertEqual(m["architecture"], "el5-x86_64") + + def test_meta_or_legacy_corrupt_still_raises(self): + d = os.path.join(self.root, "el5-x86_64", "Packages", "bad", "v1") + os.makedirs(d) + with open(os.path.join(d, ".meta.json"), "w") as fh: + fh.write("{ not json") + with self.assertRaises(ValueError): + I.meta_or_legacy(self.root, "el5-x86_64", "bad", "v1") + + def test_show_no_arch_json_is_list_even_for_single_hit(self): + # Stable JSON shape: no-arch --json always returns a per-platform list, + # so a caller sees the same shape whether 1 or N platforms match. + import io, contextlib + a = self._Args(self.root, "Boost", json=True) # Boost: single platform + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + got = json.loads(buf.getvalue()) + self.assertIsInstance(got, list) + self.assertEqual(len(got), 1) + self.assertEqual(got[0]["arch"], "x86_64-el9-gcc14-opt") + + def test_show_across_keeps_unreadable_platform_visible(self): + # A corrupt .meta.json on one platform must not silently vanish. + d = os.path.join(self.root, "aarch64-el9-gcc14-opt", "Packages", + "Boost", "1.90.0-1") + os.makedirs(d) + with open(os.path.join(d, ".meta.json"), "w") as fh: + fh.write("{ broken") + import io, contextlib + a = self._Args(self.root, "Boost", json=True) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + got = {r["arch"]: r for r in json.loads(buf.getvalue())} + self.assertEqual(len(got), 2) # both platforms present + self.assertEqual(got["aarch64-el9-gcc14-opt"]["provenance"], + "unreadable .meta.json") + + def test_meta_or_legacy_dangling_symlink_raises(self): + d = os.path.join(self.root, "el5-x86_64", "Packages", "xrootd", "v3.3.3") + os.makedirs(d) + os.symlink("/no/such/target", os.path.join(d, ".meta.json")) + with self.assertRaises(OSError): # not treated as legacy + I.meta_or_legacy(self.root, "el5-x86_64", "xrootd", "v3.3.3") + + def test_show_legacy_detail_notes_missing_meta(self): + import io, contextlib + self._legacy_pkg("el5-x86_64", "xrootd", "v3.3.3") + a = self._Args(self.root, "xrootd", arch="el5-x86_64") + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.assertEqual(I._cmd_show(a), 0) + out = buf.getvalue() + self.assertIn("xrootd v3.3.3", out) + self.assertNotIn("v3.3.3-", out) # no trailing dash when no revision + self.assertIn("legacy", out) + self.assertIn("no .meta.json", out) + if __name__ == "__main__": unittest.main()