From c9d83a2d62a56fc3bc98d1686da9f7beab6646cb Mon Sep 17 00:00:00 2001 From: Thomas Schmelzer Date: Wed, 2 Sep 2026 19:55:32 +0400 Subject: [PATCH] feat: a folder entry can exclude a checkout by name A folder is only usable for a directory that is *entirely* yours, and the one in front of me was not: ~/repos/jebel-quant held 26 of my own repos and a clone of upstream numpy, which has no business on a fleet board - its own PR budget, a line count of a tree that size read through the Docker mount, a template drift column for a repo that never heard of the template. Listing 26 repos to leave out the 27th is the terseness the folder entry was for. - folder: ~/repos/jebel-quant exclude: [numpy] Excludes name the folder's own subdirectories. That is the one thing about a child that is knowable without reading its git config, and the thing you see when you list the folder - and the two do come apart: ~/repos/tschm/ rhiza_projects/funnel is VanekPetr/funnel. So which of the two is meant had to be settled rather than guessed, and an `owner/name` written there is refused rather than quietly matching nothing. The interesting decision is what a spent exclude does. Every other mistake in repos.yml is fatal, and this one is a warning, because the harm the refusals guard against is a repo silently *missing* from the board. An exclude exists to keep a repo off it, so a folder with no such checkout in it already satisfies the request: the outcome is the one the file asked for and only the line is spent. That is what deleting a clone you never wanted looks like - a normal thing to do, which must not take the board down. It is also what a misspelt exclude looks like, and that one does put an unwanted repo on the board, so it is logged. But it lands as a row you can see next to the warning, not as the absent row you notice weeks later, and that is the difference between the two. Written after `numpy` was deleted from that folder mid-change and the first draft, which refused, would have stopped the collector over a directory whose removal was the point. Co-Authored-By: Claude Opus 5 (1M context) --- collector/jq_collector/repos.py | 75 ++++++++++++++++++++++++++-- collector/tests/test_fleet.py | 86 +++++++++++++++++++++++++++++++++ docs/configuration.md | 37 +++++++++++++- repos.example.yml | 8 +++ 4 files changed, 201 insertions(+), 5 deletions(-) diff --git a/collector/jq_collector/repos.py b/collector/jq_collector/repos.py index ab286a6..4367065 100644 --- a/collector/jq_collector/repos.py +++ b/collector/jq_collector/repos.py @@ -92,6 +92,35 @@ def _declared_forge(item: dict, index: int) -> str | None: return declared +def _excluded(item: dict, index: int, raw: str) -> frozenset[str]: + """A folder's ``exclude:`` list, as a set of directory names. + + A single name may be written on its own - ``exclude: numpy`` - because a + one-element list is the common case and the brackets are noise. + """ + declared = item.get("exclude") + if declared is None: + return frozenset() + if isinstance(declared, str): + declared = [declared] + if not isinstance(declared, list) or not all(isinstance(name, str) for name in declared): + raise FleetError( + f"entry {index}: `exclude` on folder {raw} must be a directory name " + f"or a list of them, not {declared!r}" + ) + + names = {name.strip().strip("/") for name in declared} + if nested := sorted(name for name in names if "/" in name): + # `exclude: acme/web` reads as a repo name, and a folder is scanned one + # level deep, so there is nothing at that path for it to mean. + raise FleetError( + f"entry {index}: exclude {', '.join(nested)} on folder {raw} is not a " + "directory name - excludes name the folder's own subdirectories, " + "not paths and not `owner/name`" + ) + return frozenset(names - {""}) + + def _folder(item: dict, index: int, host_root: str, claimed: frozenset[str]) -> list[dict]: """A ``folder:`` entry -> one ``path:`` entry per checkout inside it. @@ -107,6 +136,15 @@ def _folder(item: dict, index: int, host_root: str, claimed: frozenset[str]) -> path rather than on ``namespace/name`` is what makes a ``repo:`` override work, since the whole point of one is that the name comes out different. + ``exclude:`` names the folder's children to leave out, by directory name - + the one thing about a child that is knowable without reading its git + config, and the thing you can see when you list the folder. It is what + makes a folder usable for a directory that is *nearly* all yours: a clone + of somebody else's repo kept alongside your own does not belong on your + fleet board, and naming the exception is shorter than listing the rest. + An exclude with nothing to exclude is a warning rather than a refusal, for + the reason given where it is logged. + A folder that is not there is refused, exactly as an unreachable ``path`` is, and so is one holding no checkouts at all. Both look identical to a fleet that is quietly short a folder's worth of repos, and the point of @@ -123,6 +161,7 @@ def _folder(item: dict, index: int, host_root: str, claimed: frozenset[str]) -> # Validated here so a bad `forge:` on the folder is refused once, against # the line that was actually written, rather than per checkout found. forge = _declared_forge(item, index) + excluded = _excluded(item, index, raw) folder = resolve_path(raw, host_root) try: @@ -141,10 +180,31 @@ def _folder(item: dict, index: int, host_root: str, claimed: frozenset[str]) -> "A folder is scanned one level deep, so name the folder the " "checkouts are directly in." ) + if missed := excluded - {os.path.basename(child) for child in checkouts}: + # Said out loud, but not fatal, and the asymmetry is deliberate. An + # exclude exists to keep something off the board; nothing there to + # exclude means that has already happened, so the outcome is the one + # the file asked for and the line is merely spent - which is what + # deleting a clone you did not want looks like, a normal thing to do + # that must not take the board down. It is also what a misspelt + # exclude looks like, and that one does put an unwanted repo on the + # board - but as a row you can see, next to this line in the log, + # rather than as the silently missing repo the refusals here guard. + log.warning( + "folder %s: nothing called %s to exclude - the line has nothing left to do", + raw, + ", ".join(sorted(missed)), + ) + # The emptiness check is on what is in the folder, not on what is left - # after the claimed ones go: a folder whose every checkout has an entry of - # its own is a redundant line, not a mistake worth refusing to start over. - taken = [child for child in checkouts if child not in claimed] + # after the excluded and claimed ones go: a folder whose every checkout has + # an entry of its own is a redundant line, not a mistake worth refusing to + # start over. + taken = [ + child + for child in checkouts + if child not in claimed and os.path.basename(child) not in excluded + ] if len(taken) == len(checkouts): log.info("folder %s: %d checkouts", raw, len(taken)) else: @@ -168,6 +228,15 @@ def _expand( """ if isinstance(item, dict) and item.get("folder"): return [(found, True) for found in _folder(item, index, host_root, claimed)] + if isinstance(item, dict) and item.get("exclude"): + # An entry that is one repo has nothing to exclude from, and reading + # the key as decoration would leave somebody believing a repo was off + # the board while it sat on it. + raise FleetError( + f"entry {index} ({item!r}): `exclude` belongs to a `folder`, which " + "this entry is not - it names one repo, so there is nothing to " + "leave out of it." + ) return [(item, False)] diff --git a/collector/tests/test_fleet.py b/collector/tests/test_fleet.py index 16135d2..ee4dfdc 100644 --- a/collector/tests/test_fleet.py +++ b/collector/tests/test_fleet.py @@ -903,3 +903,89 @@ def test_naming_a_repo_the_folder_also_holds_is_one_entry(tmp_path, outright_fir assert fleet == ("org/alpha",) assert paths == {"org/alpha": str(path)} assert forges == {"org/alpha": "github"} + + +def test_a_folder_can_exclude_one_of_its_checkouts(tmp_path): + """What makes a folder usable for a directory that is nearly all yours. + + A clone of somebody else's repo kept alongside your own does not belong on + your fleet board, and naming the one exception is shorter than listing + everything else. + """ + make_checkout(tmp_path, "org", "alpha") + make_checkout(tmp_path, "org", "numpy", origin="git@github.com:numpy/numpy.git") + body = f" - folder: {tmp_path / 'org'}\n exclude: [numpy]\n" + + fleet, paths, _forges = repos.load(write_fleet(tmp_path, body)) + + assert fleet == ("org/alpha",) + assert "numpy/numpy" not in paths + + +def test_one_exclude_needs_no_brackets(tmp_path): + """`exclude: numpy` is the common case and the brackets are noise.""" + make_checkout(tmp_path, "org", "alpha") + make_checkout(tmp_path, "org", "numpy", origin="git@github.com:numpy/numpy.git") + body = f" - folder: {tmp_path / 'org'}\n exclude: numpy\n" + + assert repos.load(write_fleet(tmp_path, body))[0] == ("org/alpha",) + + +def test_excludes_name_directories_not_repos(tmp_path): + """The directory name is the one thing about a child that is knowable + without reading its git config, and the thing you see when you list the + folder. A checkout's name need not match its repo's - the folder `funnel` + is `VanekPetr/funnel` - so which of the two is meant has to be settled.""" + make_checkout(tmp_path, "org", "alpha") + make_checkout(tmp_path, "org", "funnel", origin="git@github.com:VanekPetr/funnel.git") + body = f" - folder: {tmp_path / 'org'}\n exclude: [funnel]\n" + + assert repos.load(write_fleet(tmp_path, body))[0] == ("org/alpha",) + + +def test_an_exclude_written_as_a_repo_name_is_refused(tmp_path): + """`exclude: VanekPetr/funnel` would otherwise silently exclude nothing.""" + make_checkout(tmp_path, "org", "alpha") + make_checkout(tmp_path, "org", "funnel", origin="git@github.com:VanekPetr/funnel.git") + body = f" - folder: {tmp_path / 'org'}\n exclude: [VanekPetr/funnel]\n" + + with pytest.raises(repos.FleetError, match="not a directory name"): + repos.load(write_fleet(tmp_path, body)) + + +def test_an_exclude_with_nothing_to_exclude_is_said_out_loud(tmp_path, caplog): + """Deleting the clone you did not want must not take the board down. + + Every other mistake in this file is refused, and this one is not, because + an exclude exists to keep a repo off the board: nothing there to exclude + means that already holds, so the outcome is the one the file asked for and + only the line is spent. Said out loud all the same - it looks identical to + a misspelt exclude, which does put an unwanted repo on the board. + """ + make_checkout(tmp_path, "org", "alpha") + body = f" - folder: {tmp_path / 'org'}\n exclude: [nupmy]\n" + + with caplog.at_level("WARNING"): + fleet, _paths, _forges = repos.load(write_fleet(tmp_path, body)) + + assert fleet == ("org/alpha",) + assert "nothing called nupmy to exclude" in caplog.text + + +@pytest.mark.parametrize("declared", ["42", "{a: b}", "[numpy, 42]"]) +def test_a_malformed_exclude_is_refused(tmp_path, declared): + make_checkout(tmp_path, "org", "alpha") + body = f" - folder: {tmp_path / 'org'}\n exclude: {declared}\n" + + with pytest.raises(repos.FleetError, match="must be a directory name"): + repos.load(write_fleet(tmp_path, body)) + + +def test_exclude_on_an_entry_that_is_not_a_folder_is_refused(tmp_path): + """One repo has nothing to exclude from, and reading the key as decoration + would leave you believing a repo was off the board while it sat on it.""" + path = make_checkout(tmp_path, "org", "alpha") + body = f" - path: {path}\n exclude: [numpy]\n" + + with pytest.raises(repos.FleetError, match="belongs to a `folder`"): + repos.load(write_fleet(tmp_path, body)) diff --git a/docs/configuration.md b/docs/configuration.md index 878e0a0..195d767 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -85,8 +85,40 @@ The rules, all of which exist so the board cannot go quietly short: - **`forge: gitlab` on a folder covers every checkout in it**, though with checkouts to read the origin off it is not needed at all. -`JQ_IGNORE` is the way to drop one repo a folder sweeps up without listing the -rest by hand. +### Excluding one repo from a folder + +A directory that is *nearly* all yours is the common case — a clone of somebody +else's repo kept alongside your own, which has no business on your fleet board. +Name the exception rather than listing everything else: + +```yaml +repos: + - folder: ~/repos/jebel-quant + exclude: [numpy] # a clone of upstream numpy, not ours +``` + +`exclude` names the folder's own **subdirectories**, which is the one thing +about a child that is knowable without reading its git config and the thing you +see when you list the folder — a checkout's directory need not be named after +its repo, so `funnel` excludes the directory `funnel` whatever its origin says +it is. A single name needs no brackets. Writing an `owner/name` there is +refused, since a folder is one level deep and there is nothing at that path for +it to mean. + +**An exclude with nothing to exclude is a warning, not a refusal** — the one +mistake in this file that is not fatal. An exclude exists to keep a repo off the +board, so a folder with no such checkout in it already satisfies the request: +the outcome is the one you asked for and only the line is spent, which is what +deleting a clone you never wanted looks like. It is logged, because a *misspelt* +exclude looks identical from here and that one does put an unwanted repo on the +board — as a row you can see, next to the warning, rather than as the silently +missing repo every other refusal here guards against. + +`exclude` on an entry that is not a folder *is* refused — one repo has nothing +to exclude from, and reading the key as decoration would leave you believing a +repo was off the board while it sat on it. + +`JQ_IGNORE` remains the way to drop a repo without editing this file at all. Edit `repos.yml`, `docker restart jq-fleet`, and the fleet is whatever you just wrote. Both halves of the collector read the same list, so the GitHub panels @@ -165,6 +197,7 @@ alarming. | `path` | | A checkout on this machine, written as you would write it yourself. `~` is your home directory — which the container sees as the single `-v "$HOME:/host:ro"` mount — and a relative path is relative to it too. | | `repo` | | `owner/name`. Optional next to a `path` — it overrides the origin, which is what you want for a fork whose board should follow upstream. On its own it monitors a repo you have not cloned: GitHub panels are gathered, the working-copy panels stay empty for that row. | | `folder` | | A directory full of checkouts. Every checkout directly inside it joins the fleet, one level deep and no further — see [A folder of repos](#a-folder-of-repos). Cannot be combined with `path` or `repo`, which describe one repo each. | +| `exclude` | | Only on a `folder`: subdirectory names to leave out, as a list or a single name. One with nothing to exclude is warned about, not refused — see [Excluding one repo from a folder](#excluding-one-repo-from-a-folder). | A bare string is shorthand for `path`. Duplicate entries, a path that is not a checkout, a folder that is missing or empty of checkouts, and an entry with diff --git a/repos.example.yml b/repos.example.yml index f1abf4a..cadd529 100644 --- a/repos.example.yml +++ b/repos.example.yml @@ -28,6 +28,14 @@ repos: # start, because you asked for the repos in it and there are none. # - folder: ~/repos/cvxgrp + # A folder that is nearly all yours: `exclude` names the children to leave + # out, by directory name, which is what you see when you list the folder. A + # single name needs no brackets. An exclude with nothing left to exclude is + # warned about rather than refused: deleting a clone you never wanted is a + # normal thing to do, and it leaves the board exactly as you asked for it. + # - folder: ~/repos/jebel-quant + # exclude: [numpy] # a clone of upstream numpy, not ours + # Only some of a large shared org is yours, so name those repos one by one. - path: ~/repos/cvxgrp/cvxsimulator