From 3f1fb0c5b91e8e49a1b4d188587102818162513a Mon Sep 17 00:00:00 2001 From: Garrett Allen <98648590+Gerrrt@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:41:33 +0000 Subject: [PATCH 1/3] fix(ci): pin the gitleaks image in compose.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one image reference left outside compose.yaml, sitting directly beneath the comment in ci.yml saying image versions are not duplicated there. It carried a tag and no digest, so it contradicted the claim in README.md and SECURITY.md that every image is pinned by tag and digest. Being outside compose.yaml is what made it unfixable in place: Dependabot watches that file, and `make pin-digests` reads that file, so a pin anywhere else is one nothing bumps and nothing digests. It also survived all three of CI's image guards — not `:latest`, not a prom/ or grafana/ prefix, not an `image:` line — which is the same gap #65 was opened about. So it moves to compose.yaml as a profile-gated stub, the way `archiver` and `renderer` already are, and ci.yml resolves it with scripts/image-for.sh. The digest is what scripts/pin-digests.sh resolved from the registry, not a hand-copied one. `make up` still starts six services. Refs #65 Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 14 ++++++++++++-- stacks/observability/compose.yaml | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dfc393..e54f786 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,9 @@ env: # Image versions are NOT duplicated here. They are resolved from compose.yaml # at run time by scripts/image-for.sh, because Dependabot only updates # compose.yaml — hardcoded copies went stale silently and CI ended up - # validating v3.1.0 configs against a stack running v3.13.2. - GITLEAKS_IMAGE: zricethezav/gitleaks:v8.24.0 + # validating v3.1.0 configs against a stack running v3.13.2. gitleaks was the + # last exception to that and is now a profile-gated service in compose.yaml + # like every other image (#65). jobs: # --------------------------------------------------------------------------- @@ -239,6 +240,15 @@ jobs: with: fetch-depth: 0 + # Same single source of truth as the validate job: whatever compose.yaml + # pins is what runs. Repeated here rather than shared because jobs do not + # share $GITHUB_ENV — it is the resolution that is duplicated, not the + # version. + - name: Resolve the pinned gitleaks image from compose.yaml + run: | + echo "GITLEAKS_IMAGE=$(./scripts/image-for.sh gitleaks)" >> "$GITHUB_ENV" + ./scripts/image-for.sh gitleaks + # Two scans, because they answer different questions. # # The working-tree scan asks "is there a secret in the code as it stands" diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index d966e2b..049ba55 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -402,6 +402,28 @@ services: image: debian:13-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132 command: ["true"] + # --------------------------------------------------------------------------- + # gitleaks for the secret-scan CI job. NOT part of the running stack: it sits + # behind the `scan` profile, so `make up` neither starts it nor pulls it, and + # `docker compose ps` shows six services as before. + # + # Here for the same reason `archiver` above is. This one was a version pinned + # in ci.yml's env: block — directly underneath the comment saying image + # versions are not duplicated there — carrying a tag and no digest. Dependabot + # watches this file, so it never saw it; `make pin-digests` reads this file, + # so it could not re-resolve it. And it survived all three of CI's image + # guards: it is not `:latest`, it was not a prom/ or grafana/ prefix, and it + # was not an `image:` line here. That is the second half of #65. + # + # `command: ["version"]` rather than ["true"]: this image's entrypoint IS + # gitleaks, so `true` would be read as a gitleaks subcommand. `version` is the + # harmless no-op if it is ever started by accident. + # --------------------------------------------------------------------------- + gitleaks: + profiles: ["scan"] + image: zricethezav/gitleaks:v8.24.0@sha256:2bcceac45179b3a91bff11a824d0fb952585b429e54fc928728b1d4d5c3e5176 + command: ["version"] + networks: observability: driver: bridge From 593bea97b365ebde79d5f3bea293ae9e48fbe8f1 Mon Sep 17 00:00:00 2001 From: Garrett Allen <98648590+Gerrrt@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:41:41 +0000 Subject: [PATCH 2/3] fix(docs): resolve the agent image from compose.yaml The runbook for onboarding a Linux host pinned grafana/alloy by hand. It was correct on the day it was written and would have been wrong from the next Dependabot bump onward, at which point a newly onboarded host would run a different Alloy from the one the monitoring host runs. check_docs.py bans image versions in prose for exactly this reason (#73), but it only inspects inline code spans, so a version inside a fenced block was invisible to it. A runbook is a shell recipe that a human executes, and this one had no more claim to a hand-written pin than a script does. Refs #65 Co-Authored-By: Claude Opus 5 --- docs/runbooks/add-monitored-device.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/runbooks/add-monitored-device.md b/docs/runbooks/add-monitored-device.md index e324212..2cd3502 100644 --- a/docs/runbooks/add-monitored-device.md +++ b/docs/runbooks/add-monitored-device.md @@ -26,7 +26,7 @@ sudo docker run -d \ -v /var/log:/var/log:ro \ -v /:/rootfs:ro \ -p 127.0.0.1:12345:12345 \ - grafana/alloy:v1.18.1 \ + "$(/path/to/HomeLab/scripts/image-for.sh alloy)" \ run --server.http.listen-addr=0.0.0.0:12345 \ --storage.path=/var/lib/alloy/data \ /etc/alloy/config.alloy @@ -35,6 +35,14 @@ sudo docker run -d \ The two `*_URL` variables are the only difference from the monitoring host's own agent — inside the compose stack they default to service names. +The image comes from `compose.yaml` rather than being written out here, so a new +host starts on the same Alloy — tag *and* digest — that the monitoring host runs, +and keeps doing so after Dependabot bumps it. A version copied into this runbook +would be stale from the next bump onward, which is the whole argument in the +header of `scripts/image-for.sh`. If the repository is not on the new host, run +`./scripts/image-for.sh alloy` on the monitoring host and paste the reference it +prints. + `--hostname` is not optional. Alloy labels everything it produces with `constants.hostname`, which inside a container is the **container ID** unless one is set — a hex string that identifies nothing and changes every time the From d5a75a220175eff4eb5998b871b0b697297741f1 Mon Sep 17 00:00:00 2001 From: Garrett Allen <98648590+Gerrrt@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:42:33 +0000 Subject: [PATCH 3/3] feat(ci): require every docker image to come from compose.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI had three image checks and #65 walked past all three, because each is a pattern match and the defect was a pin that was absent rather than wrong: `make backup` ran a bare `alpine`, which has no `:latest` to grep for, no prom/ or grafana/ prefix, and is not an `image:` line in compose.yaml. Both of the pins fixed in the previous two commits got in the same way. scripts/check_image_pins.py parses instead. Every docker run/pull/create in the Makefile, scripts/*.sh, the workflows and the runbooks' fenced shell blocks must carry an argument that resolves through scripts/image-for.sh — either a variable traced to an image-for.sh assignment in the same file, or an inline substitution. Deliberately not "the image operand must be a variable". That would still pass `IMG=alpine; docker run "$IMG"`, which is the identical defect, and it would need a table of which docker flags take a value, maintained against Docker's CLI forever, where an unknown one becomes a false positive — already broken here, since the first positional in snmp-generate is a bash array. Tracing the name back to compose.yaml is what closes the hole; being a variable closes nothing. The reasoning and the known limits are in the script's docstring. Success prints the number of invocations examined. A parser that quietly stopped matching would otherwise report green having checked nothing, which is a failure mode this repository has been bitten by before. Also widens the duplication grep past the prom/ and grafana/ prefixes that let gitleaks through, and wires the check into scripts/validate.sh and a `make check-image-pins` target so local and CI stay in step (#68). Verified by reverting each of the three real violations in a scratch copy and confirming the check goes red, and by reconstructing the original bare `alpine`, the `IMG=alpine` variant, and a digest pinned outside compose.yaml. Closes #65 Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 20 +- Makefile | 4 + README.md | 12 +- SECURITY.md | 3 + scripts/check_image_pins.py | 503 ++++++++++++++++++++++++++++++++++++ scripts/validate.sh | 18 ++ 6 files changed, 558 insertions(+), 2 deletions(-) create mode 100755 scripts/check_image_pins.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e54f786..f9d8245 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,9 +78,14 @@ jobs: # Guard against the duplication coming back. Any image: pin outside # compose.yaml is drift waiting to happen, since Dependabot cannot see it. + # + # This matched only prom/ and grafana/ until #65, which is how + # zricethezav/gitleaks sat pinned in this file's own env: block. Any + # repo/name:version fails now. The illustrative version in the header of + # scripts/pin-digests.sh is a comment and is filtered out below. - name: Verify image versions are not duplicated outside compose.yaml run: | - if grep -rnE '(prom|grafana)/[a-z-]+:v?[0-9]+\.[0-9]+' \ + if grep -rnE '(^|[^a-zA-Z0-9._/-])[a-z0-9][a-z0-9._-]*/[a-z0-9][a-z0-9._-]*:v?[0-9]+\.[0-9]+' \ --include='*.sh' --include='*.yml' --include='Makefile' \ scripts .github Makefile 2>/dev/null \ | grep -vE '^[^:]+:[0-9]+:[[:space:]]*#' ; then @@ -117,6 +122,19 @@ jobs: done < <(awk '$1 == "image:" { print $2 }' "$STACK/compose.yaml") exit "$missing" + # The three checks above are all pattern matches, and #65 walked past all + # three: `make backup` ran a bare `alpine` — no `:latest` literal, no + # prom/ or grafana/ prefix, and not an `image:` line in compose.yaml. A + # grep cannot match a pin that is simply absent. This parses every docker + # run/pull/create in the Makefile, the scripts, this workflow and the + # runbooks, and requires each one's image to resolve through + # scripts/image-for.sh — so an image that is not in compose.yaml cannot be + # run at all, whatever it is called. That is also what makes the digest + # check above complete: it reads compose.yaml alone, and compose.yaml is + # now the only place an image can come from. + - name: Verify every docker image comes from compose.yaml + run: python3 scripts/check_image_pins.py + - name: promtool check config run: | docker run --rm --entrypoint promtool \ diff --git a/Makefile b/Makefile index 3fb7a7c..afeba8a 100644 --- a/Makefile +++ b/Makefile @@ -130,6 +130,10 @@ check-compose-health: ## Verify compose health dependencies can be satisfied check-loki-rules: ## Validate Loki (LogQL) alerting rules ./scripts/check_loki_rules.sh +.PHONY: check-image-pins +check-image-pins: ## Verify every docker image comes from compose.yaml + python3 scripts/check_image_pins.py + .PHONY: pin-digests pin-digests: ## Re-resolve image digests in compose.yaml (--write applies) ./scripts/pin-digests.sh --write diff --git a/README.md b/README.md index e785dc7..8c370fc 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,10 @@ incident. the build. - **Supply chain pinned by digest.** Every image carries both a tag and a `sha256:` digest, so a moved tag cannot change what deploys. CI enforces it; - `make pin-digests` re-resolves them from the registry. + `make pin-digests` re-resolves them from the registry. Every `docker run` in + the Makefile, the scripts, the workflow and the runbooks resolves its image + from `compose.yaml` too, so an image that is not pinned there cannot be run + at all. - **Documented decisions and runbooks.** Eleven ADRs covering what was chosen and what was rejected — including the costs accepted knowingly; twelve runbooks for the operations that are easy to get wrong at 1am. @@ -251,6 +254,13 @@ Container images are pinned by **tag and digest**. A tag is a mutable pointer; a digest is the content hash, so a moved tag cannot change what gets deployed. CI enforces it, and `make pin-digests` re-resolves them. +`compose.yaml` is the only place an image may be named, including images no +service runs — the tar that takes backups and the scanner CI runs are both +profile-gated entries there. CI parses every `docker run`, `pull` and `create` +in the repository and requires each to resolve its image through +`scripts/image-for.sh`, because the pin that caused this rule was not a wrong +one but a missing one, and no amount of grepping finds those. + ## Roadmap Open work is tracked in diff --git a/SECURITY.md b/SECURITY.md index 451ac31..8ec89b5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -121,6 +121,9 @@ Every push and pull request runs: `.purge-secrets.txt` — is ever a tracked file. - Verification that every container image is pinned by **tag *and* digest**, so a moved tag cannot silently change what is deployed. +- Verification that every image any script, recipe, workflow step or runbook + runs is resolved from `compose.yaml`, so a container cannot be started from an + image the digest check never saw. See [`docs/security.md`](docs/security.md) for the threat model and segmentation rationale. diff --git a/scripts/check_image_pins.py b/scripts/check_image_pins.py new file mode 100755 index 0000000..f959a9e --- /dev/null +++ b/scripts/check_image_pins.py @@ -0,0 +1,503 @@ +#!/usr/bin/env python3 +"""Assert every container image this repository runs comes from compose.yaml. + +`make backup` used to run a bare `alpine` — no tag, no digest — and it survived +every image check CI had, because all three are pattern matches and the defect +was the *absence* of a pattern: + + * the floating-tag check greps for a literal `:latest`, and a bare `alpine` + has no tag to match; + * the duplication check greps for `prom/` and `grafana/` prefixes, and + `alpine` has neither; + * the digest check reads `image:` lines out of compose.yaml, and a shell + recipe is not compose.yaml. + +That is #65. A grep cannot see a missing pin, so this parses instead. + +The rule +-------- +Every `docker run`, `docker pull` and `docker create` must carry, among its +arguments, at least one standalone token that resolves to scripts/image-for.sh: +either `$NAME`/`${NAME}` for a NAME assigned on a statement that mentions +image-for.sh in the same file, or an inline `$(... image-for.sh ...)`. + +Scope is the Makefile, scripts/*.sh, the workflows, and fenced shell blocks in +the documents. A runbook is a shell recipe that a human executes, and +docs/runbooks/add-monitored-device.md pinned `grafana/alloy` by hand for exactly +as long as nothing checked it — the same argument check_docs.py makes for +extending CI into prose (#73). + +Why traceability rather than "the image operand must be a variable" +------------------------------------------------------------------- +Two reasons, and the first is the important one. + +`IMG=alpine` followed by `docker run "$IMG"` passes "must be a variable" and is +the identical defect. So is `IMG=alpine:3.22@sha256:...` — a digest written +anywhere but compose.yaml is one Dependabot cannot bump and `make pin-digests` +cannot re-resolve, which is the whole argument in the header of image-for.sh. +Tracing the name back to image-for.sh is what closes those; being a variable +closes nothing. + +Second, identifying "the image operand" positionally requires a table of which +`docker run` flags consume a value (-v, --entrypoint, -w, --user, --network, +--security-opt, --mount, ...), maintained against Docker's CLI forever, where an +unknown value-taking flag silently becomes a false positive. It is already +broken here: the first positional in the snmp-generate recipe is `$${flags[@]}`, +a bash array. Requiring a traced token *somewhere* in the command needs no such +table. + +The cost is that `docker run --label "from=$PROM_IMAGE" alpine tar` would pass. +That takes deliberate effort; the job of this check is to make the accident +impossible. + +Known limits, stated rather than engineered away +------------------------------------------------ +File scope is coarser than shell scope: a name traced in one workflow job +satisfies a use in another, which would fail loudly at run time with an empty +image argument. The trace does not verify image-for.sh was asked for a real +service; image-for.sh already exits 1 on that. restore-volumes.sh takes its +archiver from the backup manifest and only falls back to image-for.sh — that is +correct, because the manifest value was itself written from image-for.sh, and it +is the one place "same file" is doing real work. A heredoc body is parsed as +shell and would be flagged; a quoted string is a single token and would not. +podman and nerdctl are out of scope. + +There is deliberately no ignore mechanism. If a case needs one, the rule is +wrong — see the .gitleaksignore argument in the docstring of check_docs.py. + +Usage: scripts/check_image_pins.py +""" +from __future__ import annotations + +import pathlib +import re +import subprocess +import sys +from typing import Iterator, NamedTuple + +# PyYAML is not guaranteed on a clean runner, and this script gates CI. Same +# install-rather-than-fail as check_docs.py and check_compose_health.py. +try: + import yaml +except ModuleNotFoundError: + print("installing PyYAML", file=sys.stderr) + if subprocess.run( + [sys.executable, "-m", "pip", "install", "--quiet", + "--disable-pip-version-check", "pyyaml"], + check=False, + ).returncode: + sys.exit("PyYAML is required and could not be installed") + import yaml + +REPO = pathlib.Path(__file__).resolve().parent.parent + +RESOLVER = "image-for.sh" +# Stands in for a $( ... image-for.sh ... ) that has been lifted out of a line. +# A bare word, so the tokenizer yields it as its own argument. +RESOLVED = "\x00IMAGE_FOR\x00" +SUBST = "\x00SUBST\x00" + +SHELL_GLOB = "scripts/*.sh" +WORKFLOW_GLOBS = (".github/workflows/*.yml", ".github/workflows/*.yaml") +MARKDOWN_GLOBS = ("*.md", "docs/*.md", "docs/**/*.md") +SHELL_FENCES = {"bash", "sh", "shell", "console"} + +# Only run/pull/create: they are the verbs that fetch and execute an image. +# `docker compose` reads compose.yaml and needs no check of its own, and +# `docker image inspect` neither pulls nor runs. +SUBCOMMANDS = {"run", "pull", "create"} +# `docker container run`, `docker image pull` — the long forms of the same verbs. +MANAGEMENT = {"container", "image"} +# The handful of *global* flags that take a value. Unlike `docker run`'s flags +# this set is small and stable, and getting it wrong only skips an invocation. +GLOBAL_VALUE_FLAGS = {"--context", "--config", "--host", "-H", "--log-level", + "-l", "--tlscacert", "--tlscert", "--tlskey"} + +VAR = re.compile(r"^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$") +ASSIGN = re.compile( + r"""(?:^|[\s;&|("']) + (?:export\s+|local\s+|declare\s+(?:-\w+\s+)*)? + ([A-Za-z_][A-Za-z0-9_]*)=""", + re.VERBOSE, +) +# A Make expansion, but never the `$$(` that escapes a shell substitution. +MAKE_REF = re.compile(r"(? list[Line]: + """Join backslash-continued lines, keeping the first one's number.""" + out: list[Line] = [] + buf, start = "", 0 + for lineno, text in numbered: + if not buf: + start = lineno + if text.endswith("\\"): + buf += text[:-1] + " " + continue + out.append(Line(start, buf + text, context)) + buf = "" + if buf: + out.append(Line(start, buf, context)) + return out + + +def shell_lines(text: str) -> list[Line]: + return join_continuations(list(enumerate(text.splitlines(), 1))) + + +def makefile_lines(text: str) -> list[Line]: + """Recipe lines only, with Make expansions resolved before shell parsing. + + A variable definition is not a command, so only tab-indented recipe lines + are in scope. Order matters twice over: Make's `$(NAME)` is expanded before + `$$` is collapsed to `$`, or a `$$(cmd ...)` substitution is mangled into a + Make reference and lost. + """ + variables: dict[str, str] = {} + for raw in text.splitlines(): + match = MAKE_ASSIGN.match(raw) + if match: + variables.setdefault(match.group(1), match.group(2).strip()) + for _ in range(5): # resolve references between variables, bounded + for name, value in list(variables.items()): + variables[name] = MAKE_REF.sub( + lambda m: variables.get(m.group(1), m.group(0)), value + ) + + numbered = [] + for lineno, raw in enumerate(text.splitlines(), 1): + if not raw.startswith("\t"): + continue + line = raw[1:].lstrip().lstrip("@-+") + # Expanded rather than blanked, so a future `$(DOCKER) run` is not a + # blind spot. An unknown reference becomes an opaque word. + line = MAKE_REF.sub(lambda m: variables.get(m.group(1), "MAKEVAR"), line) + numbered.append((lineno, line.replace("$$", "$"))) + return join_continuations(numbered) + + +def _walk_runs(node, name: str = "") -> Iterator[tuple[object, str]]: + """Yield every `run:` scalar node in a workflow, with its step name.""" + if isinstance(node, yaml.MappingNode): + label = name + for key, value in node.value: + if getattr(key, "value", None) == "name" and isinstance(value, yaml.ScalarNode): + label = value.value + for key, value in node.value: + if getattr(key, "value", None) == "run" and isinstance(value, yaml.ScalarNode): + yield value, label + else: + yield from _walk_runs(value, label) + elif isinstance(node, yaml.SequenceNode): + for item in node.value: + yield from _walk_runs(item, name) + + +def workflow_lines(path: pathlib.Path) -> list[Line]: + """Let PyYAML own block-scalar parsing — indentation and style are its job.""" + out: list[Line] = [] + root = yaml.compose(path.read_text(encoding="utf-8")) + if root is None: + return out + for node, name in _walk_runs(root): + # start_mark is the `run:` line itself. A block scalar's content starts + # on the line after it; an inline scalar starts on it. + base = node.start_mark.line + (2 if node.style in ("|", ">") else 1) + numbered = [(base + i, t) for i, t in enumerate(node.value.splitlines())] + out.extend(join_continuations(numbered, context=name)) + return out + + +def markdown_lines(text: str) -> list[Line]: + """Fenced shell blocks only. Prose that merely names a command is not one.""" + out: list[Line] = [] + fence, numbered = "", [] + for lineno, raw in enumerate(text.splitlines(), 1): + match = FENCE.match(raw) + if fence: + if match and match.group(1).startswith(fence): + out.extend(join_continuations(numbered)) + fence, numbered = "", [] + else: + numbered.append((lineno, raw)) + elif match and match.group(2).lower() in SHELL_FENCES: + fence = match.group(1) + out.extend(join_continuations(numbered)) + return out + + +# --------------------------------------------------------------------------- +# Shell parsing +# --------------------------------------------------------------------------- +def statements(text: str) -> list[str]: + """Split on `;`, `&&` and `||` at paren depth zero, outside quotes. + + Statement granularity is load-bearing for the trace. The snmp-generate + recipe is a single forty-line continued logical line; tracing it whole would + mark every variable it assigns as coming from image-for.sh, which would let + an unpinned image in through the same recipe. + """ + out, cur, quote, depth, i = [], "", "", 0, 0 + while i < len(text): + ch = text[i] + if quote: + cur += ch + if ch == quote: + quote = "" + i += 1 + continue + if ch in "\"'": + quote, cur = ch, cur + ch + elif ch == "(": + depth, cur = depth + 1, cur + ch + elif ch == ")": + depth, cur = max(0, depth - 1), cur + ch + elif depth == 0 and ch == ";": + out.append(cur) + cur = "" + elif depth == 0 and text[i:i + 2] in ("&&", "||"): + out.append(cur) + cur, i = "", i + 2 + continue + else: + cur += ch + i += 1 + out.append(cur) + return [s for s in out if s.strip()] + + +def extract_substs(text: str) -> tuple[list[str], str]: + """Lift every `$( )` body out as its own line to be parsed in its own right. + + A body that mentions image-for.sh collapses to RESOLVED, so the inline form + `docker run ... "$(./scripts/image-for.sh archiver)" ...` — which + docs/runbooks/restore-the-stack.md already uses — satisfies the rule + directly rather than needing a variable to hold it. + """ + inner: list[str] = [] + out, i = "", 0 + while i < len(text): + if text.startswith("$(", i): + depth, j = 1, i + 2 + while j < len(text) and depth: + if text[j] == "(": + depth += 1 + elif text[j] == ")": + depth -= 1 + j += 1 + body = text[i + 2:j - 1] if depth == 0 else text[i + 2:] + nested, flat = extract_substs(body) + inner.extend(nested) + inner.append(flat) + out += RESOLVED if RESOLVER in body else SUBST + i = j + continue + out += text[i] + i += 1 + return inner, out + + +def segments(text: str) -> list[list[str]]: + """Tokenize into pipeline/list segments, quote-aware. + + `{` and `}` are deliberately not separators. Treating them as such splits + `${#vars[@]}` into `$` and a bare `#`, a naive comment rule then swallows + the rest of the line, and the snmp-generate invocation disappears from the + check entirely — a green run that examined nothing. + """ + segs: list[list[str]] = [] + toks: list[str] = [] + cur, quote, started, i = "", "", False, 0 + + def flush_token() -> None: + nonlocal cur, started + if cur or started: + toks.append(cur) + cur, started = "", False + + def flush_segment() -> None: + nonlocal toks + flush_token() + if toks: + segs.append(toks) + toks = [] + + while i < len(text): + ch = text[i] + if quote: + if ch == quote: + quote = "" + elif ch == "\\" and quote == '"' and i + 1 < len(text): + cur += text[i + 1] + i += 2 + continue + else: + cur += ch + i += 1 + continue + if ch in "\"'": + quote, started = ch, True + elif ch == "\\" and i + 1 < len(text): + cur, started = cur + text[i + 1], True + i += 2 + continue + elif ch == "#" and not cur and not started: + break # a comment, only ever at a word boundary + elif ch.isspace(): + flush_token() + elif ch in "|;&": + flush_segment() + while i < len(text) and text[i] in "|;&": + i += 1 + continue + elif ch in "()": + flush_segment() + else: + cur, started = cur + ch, True + i += 1 + flush_segment() + return segs + + +def invocation(tokens: list[str]) -> list[str] | None: + """Return the arguments of a `docker run|pull|create`, or None. + + The subcommand must follow `docker` adjacently, past global flags and an + optional container/image management word. That is what keeps + `docker compose run` out — compose reads compose.yaml, so it needs no check. + """ + for i, tok in enumerate(tokens): + if tok.rsplit("/", 1)[-1] != "docker": + continue + j = i + 1 + while j < len(tokens): + tok = tokens[j] + if tok.startswith("-"): + j += 2 if tok in GLOBAL_VALUE_FLAGS else 1 + elif tok in MANAGEMENT: + j += 1 + else: + break + if j < len(tokens) and tokens[j] in SUBCOMMANDS: + return tokens[j + 1:] + return None + return None + + +# --------------------------------------------------------------------------- +def traced_names(lines: list[Line]) -> set[str]: + """Names assigned on a statement that mentions image-for.sh.""" + traced: set[str] = set() + for line in lines: + for statement in statements(line.text): + if RESOLVER in statement: + traced.update(m.group(1) for m in ASSIGN.finditer(statement)) + # One fixpoint pass, so renaming through an intermediate stays traced. + for _ in range(3): + for line in lines: + for statement in statements(line.text): + match = ASSIGN.search(statement) + if not match or match.group(1) in traced: + continue + rest = statement[match.end():].strip().strip("\"'") + ref = VAR.match(rest) + if ref and ref.group(1) in traced: + traced.add(match.group(1)) + return traced + + +def check_file(rel: str, lines: list[Line]) -> tuple[list[str], int]: + traced = traced_names(lines) + problems, seen = [], 0 + for line in lines: + pending = [line.text] + while pending: + inner, flat = extract_substs(pending.pop()) + pending.extend(inner) + for tokens in segments(flat): + args = invocation(tokens) + if args is None: + continue + seen += 1 + if any( + a == RESOLVED or (VAR.match(a) and VAR.match(a).group(1) in traced) + for a in args + ): + continue + where = f"{rel}:{line.lineno}" + if line.context: + where += f" ({line.context})" + problems.append( + f"{where}: this docker command runs an image that does not " + f"come from compose.yaml — resolve it with " + f"scripts/image-for.sh, adding a profile-gated stub service " + f"to stacks/observability/compose.yaml if the image is not " + f"a running one (see the `archiver` service, and #65)" + ) + return problems, seen + + +def sources() -> list[tuple[str, list[Line]]]: + out: list[tuple[str, list[Line]]] = [] + makefile = REPO / "Makefile" + out.append(("Makefile", makefile_lines(makefile.read_text(encoding="utf-8")))) + for path in sorted(REPO.glob(SHELL_GLOB)): + out.append((str(path.relative_to(REPO)), + shell_lines(path.read_text(encoding="utf-8")))) + for glob in WORKFLOW_GLOBS: + for path in sorted(REPO.glob(glob)): + out.append((str(path.relative_to(REPO)), workflow_lines(path))) + seen: set[pathlib.Path] = set() + for glob in MARKDOWN_GLOBS: + for path in sorted(REPO.glob(glob)): + if path in seen: + continue + seen.add(path) + out.append((str(path.relative_to(REPO)), + markdown_lines(path.read_text(encoding="utf-8")))) + return out + + +def main() -> int: + problems, sites, files = [], 0, 0 + for rel, lines in sources(): + found, seen = check_file(rel, lines) + problems.extend(found) + sites += seen + files += 1 if seen else 0 + + for problem in problems: + print(f" {problem}", file=sys.stderr) + if problems: + print( + f"\n{len(problems)} docker command(s) running an image from outside " + f"compose.yaml", + file=sys.stderr, + ) + return 1 + + # The count is the point. A parser that quietly stops matching would + # otherwise pass this check while examining nothing at all. + print( + f"image pins OK — {sites} docker run/pull/create invocation(s) across " + f"{files} file(s), every image resolved from compose.yaml via " + f"scripts/image-for.sh" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validate.sh b/scripts/validate.sh index e540387..dc343b7 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -226,6 +226,24 @@ else skip "python3 not installed" fi +# --------------------------------------------------------------------------- +head_ "Image pins" +# --------------------------------------------------------------------------- +# Every docker run/pull/create in this repository must take its image from +# compose.yaml via scripts/image-for.sh. `make backup` ran a bare `alpine` past +# all three of CI's pattern-based image checks (#65), because the defect was a +# pin that was absent rather than wrong. Kept in step with ci.yml — this pair +# has drifted before (#68). +if have python3; then + if python3 scripts/check_image_pins.py; then + pass "every docker image comes from compose.yaml" + else + fail "a docker image does not come from compose.yaml" + fi +else + skip "python3 not installed" +fi + # --------------------------------------------------------------------------- head_ "Loki rules" # ---------------------------------------------------------------------------