diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a0c6e68..490c6fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,13 +12,15 @@ # `test/factory-tier.bats` stubs `gh` and runs the real jq filter — a case per # clause, each written so that deleting the clause fails it. # -# `factory-watchdog` is the same argument one layer down, and its silence is -# harder still: it fires only when the foreman is gone, so nobody ever sees it -# work on a good night. `test/factory-watchdog.bats` writes lease files and ages -# logs directly to make every stall shape observable on a runner. +# `factory-watchdog` is the runner, and its silence is the loudest of all: it +# passes every twenty minutes while nobody is watching, decides which red +# branch gets a fixer lane, and revokes the lease when its passes stop landing. +# `test/factory-watchdog.bats` stubs `factory-shift` with a scripted event +# stream, writes lease files and ages logs directly, so every gate and every +# stall shape is observable on a runner in seconds. # # `factory-lease` was only ever reached as somebody else's fixture — a grant to -# start the poller above, a lease file written directly, a `chmod -x` to make +# start the runner above, a lease file written directly, a `chmod -x` to make # `doctor` report it missing — so nothing had handed `grant` a duration nobody # would type. `test/factory-lease.bats` is the verb's own, and its subject is # the same shape as the three above: a grant that prints ✓ and is already diff --git a/AGENTS.md b/AGENTS.md index 7900023..ec014b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,8 +21,8 @@ that comes from the user's config, always. | `libexec/factory-lease` | the standing merge grant | | `libexec/factory-tier` | one PR's verdict. **The filter is the definition of tier 1** | | `libexec/factory-shift` | one pass. Deterministic: no judgement lives here | -| `libexec/factory-watchdog` | notices the foreman died | -| `ai/SKILL.md`, `ai/nightshift/SKILL.md` | the agent surface — verbs, and the loop that drives them | +| `libexec/factory-watchdog` | the runner. Passes `factory shift` every `runner.interval` while the lease is live, puts every `ci-red` through the four fixer gates, revokes a lease whose passes stopped landing | +| `ai/SKILL.md` | the agent surface — the verbs. There is no loop skill any more: the loop is the runner | ## Rules @@ -43,12 +43,20 @@ that comes from the user's config, always. variable that raised the line cap would be authority anything in the shift's environment could grant itself. The three the suites use to make a 45-minute threshold reachable in seconds — `FACTORY_STALE`, `FACTORY_DEAD`, - `FACTORY_WATCHDOG_INTERVAL` — may only *shorten* the policy's number and are - refused when they would not, because a poller inherits the environment of - whoever ran `lease grant`, and on a night shift that is the foreman. - `FACTORY_NO_WATCHDOG=1` stops `grant` spawning a poller, for a suite that - must not leak one; it hides nothing, since `watchdog once` then reports NO - POLLER at exit 4 and `doctor` carries that line. + `FACTORY_WATCHDOG_INTERVAL`, and `FACTORY_RUNNER_INTERVAL` for the pass + cadence — may only *shorten* the policy's number and are refused when they + would not, because the runner inherits the environment of whoever ran + `lease grant`. `FACTORY_NO_WATCHDOG=1` stops `grant` spawning a runner, for + a suite that must not leak one; it hides nothing, since `watchdog once` then + reports NO RUNNER at exit 4 and `doctor` carries that line. +- **The fixer gates are four, in code, and each refusal is a line.** A `ci-red` + event gets a lane only when `fixer.command` is configured, no shift log holds + a `fixer-spawned` for the same head SHA, today's log holds fewer than + `fixer.cap` for the repo, and the pass's `budget` event said `fixer: true`. + Every other outcome is `fixer-skipped` with the gate named, or + `fixer-failed` with the command's stderr. These were a skill's prose once, + which is a threshold no test can reach; `test/factory-watchdog.bats` has a + case per gate, written so that deleting the gate fails it. - **Every deny clause needs a case that fails when it is deleted.** A clause that stops matching has no symptom until a PR someone meant to see merges at 3 a.m. `test/factory-tier.bats` is the shape; the README's floor table and diff --git a/README.md b/README.md index 40d89c2..28bad38 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,11 @@ sitting overnight because the human who would have merged it was asleep. `factory` merges the fraction a filter can vouch for, watches the default branch's CI, and leaves everything with taste in it for the morning. It is four bash scripts, a JSON policy file and a log. Nothing stays resident past the -lease but the poller that watches it, and there is no webhook, no service to -sign up for, and nothing that phones anywhere. +lease but the runner that passes under it, and there is no webhook, no service +to sign up for, and nothing that phones anywhere. **Its failure mode is the status quo.** No lease, an expired lease, a pass that -could not see, a foreman that died — every one of them leaves your PRs exactly +could not see, a runner that died — every one of them leaves your PRs exactly where they are today: open, waiting for you. ```sh @@ -30,8 +30,9 @@ factory shift --dry-run # sense everything, merge nothing Then, when you trust what the dry run said: ```sh -factory lease grant 12h # authority to merge tier 1, until then -factory shift # one pass — loop it, or drive it from an agent +factory lease grant 12h # authority to merge tier 1, until then. + # A runner starts with it and passes every 20 minutes +factory shift # or one pass by hand, any time ``` --- @@ -40,10 +41,10 @@ factory shift # one pass — loop it, or drive it from an agent | | | |---|---| -| `factory lease` | the standing merge grant. `grant 12h` / `status` / `revoke`. One line in a machine-local state file — so no pull request can ever grant itself authority | +| `factory lease` | the standing merge grant. `grant 12h` / `grant indefinitely` / `status` / `revoke`. One line in a machine-local state file, so no pull request can ever grant itself authority. A grant starts the runner | | `factory tier` | is one PR **tier 1**, i.e. mergeable by code alone? Decided by the policy you typed, never by a model's read of the diff | | `factory shift` | one pass: read the budget, judge every open PR, merge tier 1 under a live lease, run your after-merge hook, report a red default branch. `--dry-run` senses and merges nothing | -| `factory watchdog` | notice that the *foreman* died, which no pass can report. Started automatically by `lease grant` | +| `factory watchdog` | the runner. While the lease is live it passes `factory shift` every 20 minutes, puts every red default branch through the four fixer gates, and revokes a lease whose passes stopped landing. Started by `lease grant`; `once` asks whether it is up | Plus the surface around them: `factory config print`, `factory doctor`, `factory skill`, `factory --help`. @@ -52,11 +53,11 @@ Plus the surface around them: `factory config print`, `factory doctor`, | | | |---|---| -| **0** | ok · tier 1 · foreman healthy · `doctor` ready with nothing to note | +| **0** | ok · tier 1 · passes landing under the lease · `doctor` ready with nothing to note | | **1** | nothing (no live lease) · a pass that aborted having sensed nothing · `doctor` ready **with notes** | | **2** | usage, or a config that cannot be used · `doctor` blocking | -| **3** | refused (not tier 1) · foreman stalled · a `skill install` only partly honoured | -| **4** | a live lease with no poller watching it | +| **3** | refused (not tier 1) · shift stalled · a `skill install` only partly honoured | +| **4** | a live lease with no runner under it | **`doctor` is the one verb whose 1 is not a refusal**, and it is the row to read twice. A ready machine with something worth mentioning exits 1, and there @@ -102,6 +103,7 @@ floor below them that no config can lower — so what it prints is what "commands": ["make lockfiles", "git push"] }, "budget": { "mode": "metered", "feed": "~/.cache/usage.tsv" }, + "fixer": { "command": ["my-spawn-a-lane"] }, "notify": { "mode": "auto", "source": "factory" } } ``` @@ -156,10 +158,11 @@ write `["*"]` to drop the author test for a repo whose PRs you do not open yourself — the widest policy has to be one somebody typed, so an empty list is refused rather than read as anyone. -An agent's judgement enters exactly twice, both bounded: writing the PRs in the -first place, and deciding whether a red CI run is worth a fixer lane. Everything -`factory shift` refuses is **queued**, never closed — the verdict and its reason -land in the log, and the PR waits where it always has. +An agent's judgement enters exactly once, and it is bounded: writing the PRs +in the first place. Whether a red CI run gets a fixer lane is four checks in +code, not a judgement (see *The runner*). Everything `factory shift` refuses is +**queued**, never closed — the verdict and its reason land in the log, and the +PR waits where it always has. ### The floor `tier1.deny` sits on top of @@ -262,7 +265,8 @@ hours ago, which means the shift has been over since then. ## The budget governor Merging and sensing are `gh` calls and cost no tokens. Exactly one thing is -throttled: **can the account afford an agent lane right now.** +throttled: **can the account afford an agent lane right now.** The runner reads +the answer off this line, as the last of its four fixer gates. Point `budget.feed` at a TSV whose first four columns are `5-hour %`, `weekly %`, `5-hour reset epoch`, `weekly reset epoch`, and every pass ends its @@ -312,59 +316,136 @@ No quota to count? `"budget": {"mode": "unmetered"}` says so out loud, and the log says it too — so a feed that merely went missing can never be mistaken for a decision you made. -## When the foreman dies - -The unknown lines above keep a pass that could not *see* from reading as a quiet -night. The watchdog is the layer under them, and it exists because every one of -those lines has to be written by a pass that RAN. - -A foreman is usually an agent session driving a loop, and that loop continues -only if a turn completes and schedules the next wakeup. A turn that ends in an -error schedules nothing. Nothing is then left running, so nothing is left to -report it: the log's last line is an ordinary `pass done: 0 merged`, and the -lease goes on standing for hours with nobody exercising it. - -The heartbeat is the shift log's mtime, read as the **later** of that and the -lease's own grant stamp. Two thresholds, because a blip and a death want -different answers: at **45 minutes** quiet (`watchdog.stale`, 2700 seconds) the -watchdog writes `foreman-stalled` and cards it once, and the lease stands; at -**90** (`watchdog.dead`, 5400) it writes `foreman-gone` and **revokes the -lease**, so the morning finds the ordinary human-in-the-loop workflow rather -than a standing grant nobody is exercising. - -The poll runs every `watchdog.interval` (300 seconds), and `dead` has to be -greater than `stale`, which validation enforces at startup. The other way round -is a watchdog that revokes a lease before it has warned anybody, and a policy -saying so must never reach the loop and find out there. Those three are a -`watchdog` block in the policy file, absent from the starter config for the -reason `scope`'s two keys are: `factory config print`'s watchdog row is what is -in force. - -They are whole numbers of seconds, checked with the budget dials and for the -same reason. A fractional `dead` makes `[ quiet -ge dead ]` read false at every -poll, so the death this whole layer exists to notice is never noticed — the -quietest failure in the tool, and the only one of these that fails **open**. -`tier1.maxLines` is checked the same way, where the equivalent slip fails closed -and refuses every PR with a nonsense cap printed in the reason. - -Both thresholds count time the poller was **awake** for. A machine that -suspended has a stale log through nobody's fault — the watchdog was not running -either — so the loop measures how long its own `sleep` actually took and -subtracts the excess, writing `machine-slept` for the record. Subtracted rather -than forgiven with a grace window: a laptop that suspends and wakes all night -renews a grace window faster than it expires, and a genuinely dead foreman would -keep its lease until morning. - -The watchdog deliberately **does not run `factory shift` itself.** It could; the -script is deterministic and the lease is the authority it would run under. But -merging with no foreman means a red CI nobody reads and a `merge-failed` nobody -retries — a factory that keeps its hands moving after its eyes have closed. +## The runner + +`factory shift` is one pass. `factory lease grant` starts the thing that calls +it again: `factory watchdog run`, one process per machine, alive for as long +as the lease is. It does three things and nothing else. + +**It passes.** Every `runner.interval` (1200 seconds, 20 minutes) it runs +`factory shift --json`, reads the events, and lets the human lines land in the +shift log as they always have. A pass that could not see, whether +`prs-unknown`, `tier-unknown`, `ci-unknown`, `after-merge-failed` or a +`pass ABORTED`, gets one more pass at the next tick, five minutes later, and +writes `pass-retry` to say so. Once. A second unknown is a line for the +morning, not a loop. A shift that exits before it can write anything, which is +what a config gone invalid at 2 a.m. looks like, is `pass-failed` with its +stderr quoted. + +**It spawns fixer lanes, through four gates.** Each `CI-RED` line the pass +printed is followed by a line saying what the runner did about it: + +| gate | the line when it refuses | +|---|---| +| `fixer.command` is configured | `fixer-skipped: — no fixer.command configured` | +| no shift log holds `fixer-spawned: ` | `… a lane was already spawned for ` | +| today's log holds fewer than `fixer.cap` (2) `fixer-spawned: ` lines | `… N lane(s) already today, fixer.cap is 2` | +| the pass's budget line ended `fixer: yes` | `… budget: ` | + +All four hold, and the runner runs `fixer.command` with three words appended, +` `, and writes `fixer-spawned: `. The command is yours: on a haus machine it opens an agent lane with the +run URL in its prompt, and on a machine with none configured a red branch is +reported, carded, and left alone. It has to return once the lane is started. +It runs in the runner's turn, so a command that waits for the lane to finish +holds every pass after it. A command that exits non-zero is `fixer-failed` +with its stderr, and a card, because a hook you configured that cannot work is +the same shape as `after-merge-failed`. A failed spawn counts toward neither +the cap nor the novelty check. + +The novelty check reads every shift log there is, not tonight's. A head SHA is +unique, so a fix that broke CI again does not get a third machine, and a red +branch that stood across midnight does not get a lane a day. The cap is per +calendar day because the log is. The fifth rule you might expect, that the +failure be on the default branch, is answered before the runner asks: +`factory shift` only ever queries the base branch's runs, so every `CI-RED` is +on it by construction, and the event carries `branch` so the lane is handed a +fact. + +These four gates used to be a skill: prose an agent session read on each +wakeup, beside a retry counter. The session was the *foreman*, and this +watchdog measured whether it was still alive. The foreman's judgement turned +out to be four string checks and a retry counter, and a rule that is four +string checks is code. Written here it is deterministic, `test/factory-watchdog.bats` +has a case per gate, and no agent pane has to survive the night for a docs PR +to merge at 3 a.m. + +**It notices when passes stop landing.** The heartbeat is the shift log's +mtime, read as the later of that and the lease's own grant stamp. The runner +writes its own lines to the same log and restores the mtime after each, so only +a pass counts. What can make the log go quiet under a live runner is a shift +that dies before its first line, every twenty minutes, with the lease standing, +or one pass hanging inside a `gh` call. Two thresholds, because a blip and a +breakdown want different answers. At **45 minutes** quiet, which is +`watchdog.stale`, 2700 seconds, the runner writes `shift-stalled`, cards it +once, and the lease stands. At **90**, `watchdog.dead`, 5400, it writes +`shift-dead` and **revokes the lease**, so the morning finds the ordinary +human-in-the-loop workflow rather than a standing grant nobody is exercising. +A pass landing after a stall writes `shift-resumed`, and re-arms the card. + +The tick is `watchdog.interval` (300 seconds). Validation holds `dead` above +`stale` and `stale` above `runner.interval`, at startup. The other way round on +the first is a runner that revokes before it has warned anybody; on the second +it is a runner that calls its own gap between two passes a stall, all night. +All four are whole numbers of seconds, checked with the budget dials and for +the same reason. A fractional `dead` makes `[ quiet -ge dead ]` read false at +every tick, so the breakdown this layer exists to notice is never noticed. That +is the quietest failure in the tool and the only one of these that fails +**open**. `tier1.maxLines` is checked the same way, where the equivalent slip +fails closed and refuses every PR with a nonsense cap printed in the reason. +`factory config print` has a row for each block. None of them is in the starter +config, for the reason `scope`'s two keys are not. + +Both thresholds count time the runner was **awake** for. A machine that +suspended has a stale log through nobody's fault, so the loop measures how long +its own `sleep` took and subtracts the excess, writing `machine-slept` for the +record. Subtracted rather than forgiven with a grace window: a laptop that +suspends and wakes all night renews a grace window faster than it expires, and +a shift that genuinely could not run would keep its lease until morning. + +**What keeps the runner itself alive is not the runner.** A process can be +lost to a reboot, a panic or an out-of-memory kill, and there is deliberately +no second process watching for that. On a machine whose launchd owns the +runner, `KeepAlive` restarts it and it passes again within seconds. Anywhere +else, `factory lease grant` and `factory watchdog ensure` start one, and +`factory watchdog once`, which `doctor` carries, says `NO RUNNER` at exit 4 +for as long as a live lease has none. A dead runner restarts instead of being +reported, and a lease it left standing is the status quo. The lease is what +you switch: `grant` and it runs, `revoke` and it stops, and `shift-over` is +the log's last line when a timed lease ran out. + +``` +22:00 lease: tier 1 until Mon 10:00 +22:00 policy: 3f9a1c2e · factory 0.1.0 +22:00 budget: 5h 13% · week 16% · reserve 58 pts · headroom 21 pts · fixer: yes +22:01 merged: you/docs#212 typo in the install page +22:01 after-merge: 2 command(s) ok after 1 merge(s) +22:01 CI-RED: you/app https://github.com/you/app/actions/runs/1 +22:01 fixer-spawned: you/app 9c2e1f0 — lane on main for https://github.com/you/app/actions/runs/1 +22:01 pass done: 1 merged +22:21 CI-RED: you/app https://github.com/you/app/actions/runs/1 +22:21 fixer-skipped: you/app — a lane was already spawned for 9c2e1f0 +22:21 pass done: 0 merged +``` + +### An indefinite lease + +`factory lease grant indefinitely` writes a lease with no expiry. `status` says +`indefinite · until revoked`, and `--json` carries `indefinite: true` with +`expires` and `secondsLeft` null, so a countdown drawn off it draws "until +revoked" rather than a number of centuries. It is allowed because the bound on +what merges was never the clock: it is tier 1, and a policy you typed. What the +clock bounded was how long a standing grant could outlive whoever was +exercising it, and the runner's `shift-dead` now bounds that on its own. The +state file spells it `never` where the epoch goes, so a reader that only knows +epochs reports the lease unreadable and refuses to merge, rather than reading a +sentinel as live for a century. ## Driving it from an agent -`factory shift` is one pass. Something has to call it on a cadence, decide -whether a red branch is worth a fixer, and write the handover — that is the -**foreman**, and it is the only part with judgement in it. +Nothing has to. A live lease runs the shift, and the log is the handover. What +an agent still does is the verbs around it: grant the lease when asked, read +the log in the morning, explain a refusal. ```sh factory skill # the routing document for a coding agent @@ -388,18 +469,20 @@ exits **0**. A non-zero there would have every agent on such a machine report a broken command and try again with more force. An agent that has the skill knows the verbs, the log vocabulary, the four -unknown lines, and the two rules that matter: never merge outside -`factory shift`, and never spawn a lane the budget line has not said -`fixer: yes` to. +unknown lines, and the rules that matter: never merge outside `factory shift`, +never loop it while a lease is live because the runner already is, and never +spawn a lane off a `CI-RED` line because the line after it is the runner's +verdict on that. ## Overnight on a closed lid macOS sleeps on lid-close regardless of `caffeinate`. The lever that actually crosses a lid close is `sudo pmset -a disablesleep 1` (and the Mac has to be on -power). Asleep, the loop pauses rather than stops — but "pauses" is a claim -about the scheduler, not about the network: a wakeup that fires into an -interface that has not reassociated is a turn that errors, and that turn is the -end of the shift unless the watchdog is running. Both halves are needed. +power). Asleep, the runner pauses rather than stops: its next pass lands when +the machine wakes, the gap is written as `machine-slept`, and it is not counted +against the shift. A pass that fires into an interface that has not +reassociated is a pass full of unknowns, which gets its one retry at the next +tick and is otherwise a line for the morning. ## What it deliberately does not do @@ -408,8 +491,9 @@ end of the shift unless the watchdog is running. Both halves are needed. - **It does not write PRs.** Something else opens them; this closes the ones nobody needed to read. - **It does not phone anywhere.** No telemetry, no service, no account. -- **It does not run headless.** A merge nobody is awake to notice is the thing - the watchdog exists to prevent, not a feature. +- **It does not keep its hands moving after its eyes have closed.** The runner + merges with nobody watching, and that is the point, but only while its own + passes are landing: ninety minutes without one and the lease is revoked. ## How it looks on screen @@ -428,17 +512,18 @@ turns it on for a pipe, and `dumb` beats even that. ### The card, for the report nobody is reading -A shift runs while nobody is watching, so six moments are drawn as a +A shift runs while nobody is watching, so seven moments are drawn as a notification as well as a log line: a pass that **aborted**, a **red default -branch** (with the run's URL on it), an **after-merge hook that failed**, the -**merge tally** at the end of a pass that merged something, and the watchdog's -**stalled** and **gone**. Nothing else cards, and the two that most look like -they should are deliberate. One unseeable repo does not, because whether that -is worth waking somebody for is the foreman's judgement rather than a script's. +branch** (with the run's URL on it), an **after-merge hook that failed**, a +**fixer lane that failed to start**, the **merge tally** at the end of a pass +that merged something, and the runner's **stalled** and **dead**. Nothing else +cards, and the two that most look like they should are deliberate. One +unseeable repo does not, because it gets its retry at the next tick and a +second unknown is a line for the morning rather than a reason to wake up. `merge-failed` does not either: its commonest cause is the `--match-head-commit` pin working exactly as designed, which is a line to read in the morning and not a reason to wake up. Its rarer cause, an expired token, is a shift that has been -over for hours, and the watchdog is what cards that. +over for hours, and the runner's `shift-dead` is what cards that. `notify.mode` decides how one is sent: @@ -468,11 +553,13 @@ but whether it can reach anything. A `command` that PATH cannot find blocks, because you typed it and it cannot work. `auto` with no trill installed, and `off`, are notes rather than blocks: neither is a fault, and both are worth saying out loud on a report about whether this machine can run a night. +`fixer.command` gets the same two answers for the same reasons: a program PATH +cannot find blocks, and none configured is a note. ## Development ```sh -bats test/ # 202 cases +bats test/ # 242 cases shellcheck -x bin/factory libexec/* lib/*.sh script/*.sh # The presentation cases need snug's bash half; without it they skip. diff --git a/ai/SKILL.md b/ai/SKILL.md index cb85959..9102d2d 100644 --- a/ai/SKILL.md +++ b/ai/SKILL.md @@ -4,10 +4,10 @@ description: >- Merge the pull requests that a reviewed filter can vouch for, without waking the user — and check what a night of that did. Use when the user says "merge the safe PRs", "what did the factory do last night", "grant/revoke the merge - lease", "is anything red on main", "why didn't PR N merge", "can we afford a - fixer lane", or asks to run one pass of the shift. For running the shift on a - cadence overnight, load the `nightshift` skill instead — that one is the loop, - this one is the verbs. + lease", "keep shipping while I'm asleep", "is anything red on main", "why + didn't PR N merge", "can we afford a fixer lane", or asks to run one pass of + the shift. A live lease runs the shift on its own; this skill is the verbs + around it and how to read what it wrote. --- # factory — merge what code alone can vouch for @@ -20,21 +20,25 @@ decides with a model, never writes PRs, and never merges without a live lease. **Its failure mode is the status quo**: no lease, an expired one, or a pass that could not see leaves every PR open, exactly where it is today. -Nothing runs on a schedule. One `factory shift` is one pass; something has to -call it — usually a person, or an agent driving the `nightshift` loop. +**The lease is the on/off switch.** `factory lease grant 12h` starts a runner +(`factory watchdog run`) that passes `factory shift` every 20 minutes until the +lease ends, spawns a fixer lane on a red default branch when four gates in +code allow it, and revokes the lease if its passes stop landing. No agent +session drives it. The shift log is the handover. ## Verbs | do this | run this | |---|---| | take merge authority for a while | `factory lease grant 12h` | +| take it until told otherwise | `factory lease grant indefinitely` | | check / drop that authority | `factory lease status` · `factory lease revoke` | | sense everything, merge nothing | `factory shift --dry-run` | -| one real pass | `factory shift` | +| one real pass, by hand | `factory shift` | | ask why one PR is not mergeable | `factory tier ` | | read the effective policy | `factory config print` | | is this machine able to run a shift | `factory doctor` | -| is the foreman alive | `factory watchdog once` | +| are passes landing under the lease | `factory watchdog once` | | last night's report | `cat ~/.cache/factory/shift-$(date +%Y%m%d).log` | Every read verb takes `--json` — `doctor` included, where it returns one @@ -47,6 +51,10 @@ run always means the run failed, never that the verb had no JSON to give. - "merge the docs PRs" / "clear the safe ones" → `factory shift --dry-run` first, then `factory shift` under a lease +- "keep shipping while I'm away" → `factory doctor`, then + `factory lease grant `. That is the whole start: the runner does + the rest. Confirm with `factory watchdog once` (exit 0), and tell the user + in one line what authority stands and until when - "why is #212 still open?" → `factory tier 212` — the refusal names its own reason - "what happened last night?" → read today's (or yesterday's) shift log @@ -54,6 +62,25 @@ run always means the run failed, never that the verb had no JSON to give. - "can it merge X too?" → that is a policy edit at `factory config path`, and it is the user's call, never yours +## Reading the log + +- `merged` / `queued` / `would-merge` — verdicts. `queued` waits for a person by + design. +- `CI-RED ` — the default branch is red. The lines right after it + say what the runner did about it: `fixer-spawned: `, or + `fixer-skipped: — ` naming which of the four gates refused + (no `fixer.command`, same head SHA already had a lane, `fixer.cap` reached + today, or budget), or `fixer-failed` with the command's stderr. +- `pass-retry: — one more pass at the next tick` — the runner saw an + unknown or an abort and is running once more. Two of the same unknown in a + row is a story for the user, not something to fix. +- `shift-stalled` / `shift-resumed` / `shift-dead` — passes stopped landing + under a live lease, resumed, or stopped long enough that the runner revoked + the lease. `machine-slept` is a gap that was the machine's. `shift-over` is + the lease ending the ordinary way. +- `pass-failed` — `factory shift` exited before it could write anything. The + line quotes why; it is usually the config. + ## When NOT to - **Never merge a PR the shift queued.** A `queued:` line is a verdict: the PR @@ -64,6 +91,12 @@ run always means the run failed, never that the verb had no JSON to give. it against the new head. - **Never widen `tier1` to get something through.** The filter is the whole definition of what may merge unattended. +- **Never loop `factory shift` yourself while a lease is live.** The runner is + already doing it, on the cadence the policy names. A pass by hand is fine; + a second loop is two things merging under one grant. +- **Never spawn a fixer lane yourself off a `CI-RED` line.** The four gates are + in code and their verdict is the line after it. A lane the gates refused is + a lane the budget or the cap said no to. - Opening PRs, reviewing code, releasing — none of that is here. ## Traps @@ -74,18 +107,18 @@ run always means the run failed, never that the verb had no JSON to give. judged and refused; `tier-unknown` means nothing judged it. The second one is a PR nobody has looked at. - **`ci-unknown` is not a green branch**, and `prs-unknown` is not a repo with - no PRs. Both mean the pass was blind there. Run it again once; if it repeats, - say so. + no PRs. Both mean the pass was blind there. The runner retries once on its + own; if the line repeats, say so. - **A `pass ABORTED` exits non-zero and merged nothing.** Nothing was sensed — do not report it as a quiet night. - **The policy file is machine-local** (`factory config path`), deliberately: a copy inside a repo would be a file a PR could edit to widen the filter judging it. Do not add one to a repo. - **The lease is the user's grant.** Never grant one to get past a refusal, and - never re-grant one the watchdog revoked — that revocation is the watchdog - reporting the shift stopped being run. + never re-grant one the runner revoked — `shift-dead` is the runner reporting + its passes stopped landing, and the reason is in the lines above it. - Exit codes: `0` ok/tier 1 · `1` no lease, or an aborted pass · `2` usage or - bad config · `3` refused / foreman stalled · `4` live lease, no poller. + bad config · `3` refused / shift stalled · `4` live lease, no runner. **`doctor` is the exception**: `0` ready with nothing to note, `1` ready **with notes**, `2` blocking. A ready machine usually exits 1, so read `.ready` rather than the code. diff --git a/ai/nightshift/SKILL.md b/ai/nightshift/SKILL.md deleted file mode 100644 index dfcee50..0000000 --- a/ai/nightshift/SKILL.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -name: nightshift -description: >- - Take the factory's shift: grant the merge lease, loop `factory shift` on a - cadence until it expires, spawn capped fixer lanes on red CI, and write the - handover. Use when the user says /nightshift, "run the night shift", "take - the shift", "keep shipping while I'm away/asleep", usually with a duration - ("/nightshift 12h"). This skill is the foreman — the judgement half the - deterministic scripts refuse to carry. For one-off verbs, load `factory`. ---- - -# nightshift — the foreman loop - -You are taking the shift. The user is away; everything below runs without them, -and the shift log is your handover. Read `factory skill` once first. - -## Start - -1. `factory doctor` — before any authority is granted. It is the only step - that asks whether this machine can *reach* you: a `notify.command` PATH - cannot find, or no trill on a machine set to `auto`, is a night that merges - correctly and cards its red CI nowhere. `.blocking > 0` stops here; notes - are worth one line in your start message. -2. `factory lease grant ` — the duration from the invocation; no - duration given means **1h**. Tell the user in one line what authority you now - hold and until when. -3. `factory watchdog once` — confirm the poller `grant` just started is watching - you. It is what turns your own death into something the user finds in the - morning instead of a lease that stood all night with nobody exercising it. - Exit **0** is what you want. **4** is `NO POLLER` — a live lease nothing is - watching; say so in your start line. **1** means the grant did not take and - there is no shift to run. **2** is the watchdog refusing its own - configuration — the line names it, and it is usually a `FACTORY_*` override - in your environment that would lengthen a threshold; unset it and run - `factory watchdog ensure`, because until then nothing is watching you. -4. `factory shift --dry-run` — one sensing pass so your first real pass holds no - surprises. If it shows `would-merge` rows the user can still see, name them. - -## The loop - -Cadence **~20 min**, using whatever timer your client has. Each wakeup: - -1. `factory watchdog ensure` — the lease check and the liveness check in one, - and it restarts a poller lost to a reboot or an OOM kill. Exit **1 → jump to - Shift end** (the lease is gone). **3** means the watchdog thinks *you* have - been quiet too long, which on a wakeup you are running means the last pass - failed to log — read the shift log before doing anything else. -2. `factory shift`. The script merges, runs the after-merge hook and logs on its - own; your job is only what it printed: - - **`CI-RED `** → maybe spawn a fixer (rules below). - - **`merge-failed` / `after-merge-failed`** → **the line carries the reason; - read it before doing anything.** A merge refused on `--match-head-commit` - is the pin working — the branch moved after the verdict — and needs nothing - at all. **A `merge-failed` is never re-driven by hand, whatever the reason - says**: that is merging outside `factory shift`, against a head no verdict - covers. An `after-merge-failed` you may retry once yourself; the line names - which command stopped, and re-running the others is not the fix. - - **`queued` rows need nothing** — they are the morning's, by design. Never - merge one yourself, whatever the reason column says: the lease covers tier - 1 as the policy decides it, not as you would. - - **`prs-unknown` / `tier-unknown` / `ci-unknown`** → the pass could not SEE - that thing; it is not a verdict and not a quiet result. Run the pass again - once. If the same line comes back, say so in your next message and, for a - `ci-unknown`, check that repo yourself (`gh run list -R -b main - -L1`) rather than carrying an unknown through the night. Never spawn a - fixer off an unknown: you have not seen a failure, only a gap. - - **`scope-truncated: listed N repos`** → the org filled `scope.limit`, - so anything past the cap was never walked and its PRs were never judged. - Not a retry: the next pass lists the same N. Say so in your next message; - raising the cap is a policy edit and the user's. - - **`pass ABORTED`** (non-zero exit) → nothing sensed, nothing merged. Retry - once; if it aborts again, stop retrying, keep the loop alive at the normal - cadence, and report it. - - **`foreman-stalled` / `foreman-resumed`** in the log → the watchdog saw you - go quiet and you are back. Say so with the gap it names. - **`machine-slept`** is the same line for a gap that was the machine's, not - yours. **`foreman-gone`** you will never read — it is written as your lease - is revoked. -3. Mark the wakeup quiet only when the pass merely sensed. **An `unknown` line - or an abort is not a quiet night** — collapsing it into a run of quiet ticks - is exactly the mistake those lines exist to prevent. - -## Fixer lanes - -On `CI-RED `, all four must hold: - -- **budget**: the pass's `budget:` line ends **`fixer: yes`** — else append - `fixer-skipped: budget — ` to today's shift log and move on, where - `` is the text after `fixer: no`. Do **not** redo the arithmetic or - reason around it: a threshold re-derived in prose is one nothing can test and - nobody can see is stuck. `fixer: no (budget unknown)` is a refusal like any - other — an unreadable quota is not permission; -- **cap**: fewer than **2** fixers for this repo tonight (count your own - `fixer-spawned: ` lines in today's log); -- **novelty**: no earlier fixer tonight was spawned for this same head SHA — a - fix that broke CI again does not get a third machine; -- the failure is on the **default branch**, not a PR branch. - -Spawn it as a real background agent session — never a headless one-shot that -stalls on its first permission prompt with nobody watching. Give it: the run -URL, then "diagnose from the run log, fix it, verify, commit, push, open a PR -titled `fix(ci): …`. Stop at PR open." - -Then append `fixer-spawned: ` to today's shift log -(`~/.cache/factory/shift-*.log`). Both lines are yours to write: nothing in -`factory shift` knows a lane was considered, so a decision you only put in a -message is one the morning cannot read and the cap cannot count. - -A fixer's PR is not special: if it is docs-only the next pass merges it; a code -fix waits for the morning like every other PR. - -## Shift end - -Lease expired, or the user says "end the shift": - -1. Final `factory shift --dry-run`, so the log's last lines are the open state - of the world. -2. Write the handover from today's log: merged (count + list), queued (with - reasons), CI reds and what each fixer did, budget at close. Post it as your - final message. -3. Stop the loop. Do not renew your own lease — only the user grants one. - `factory lease revoke` stops the watchdog with it; a lease left to expire - takes the watchdog down at its next poll, so neither needs stopping by hand. - -## What this skill never does - -Merge outside `factory shift`, widen the policy, activate or deploy anything, -touch releases, or spawn anything the budget line has not said `fixer: yes` to. -Quiet nights are good nights. - -It also never stops the watchdog to quiet a `foreman-stalled` line, and never -re-grants a lease the watchdog revoked. Both are the shift reporting that it -stopped being able to do its job, and a foreman that silences either is the -exact failure those lines were added to make visible. diff --git a/bin/factory b/bin/factory index 11fbeaf..82b3f0e 100755 --- a/bin/factory +++ b/bin/factory @@ -5,7 +5,8 @@ # lease the standing merge grant this machine's owner types # tier is one PR mergeable unattended, by policy rather than judgement # shift one pass: sense, merge tier 1, report the rest -# watchdog notice that the foreman died, which no pass can report +# watchdog the runner: pass on a cadence while the lease is live, spawn the +# fixer lanes the gates let through, notice when passes stop # # Everything else here is the surface around them: `config`, `doctor`, `skill`. # shellcheck source-path=SCRIPTDIR/.. @@ -25,15 +26,16 @@ usage() { factory — merge the PRs code alone can vouch for, while nobody is watching. usage: - factory lease grant <30m|12h|2d> take tier-1 merge authority until then + factory lease grant <30m|12h|2d> take tier-1 merge authority until then; + factory lease grant indefinitely the runner passes under it until it ends factory lease status [--json] 0 live · 1 none/expired factory lease revoke factory shift [--dry-run] [--json] one pass: sense, merge, report factory tier [--json] is one PR tier 1? 0 yes · 3 no - factory watchdog once|ensure [--json] is the foreman alive? - factory watchdog run|stop the poller itself + factory watchdog once|ensure [--json] are passes landing under the lease? + factory watchdog run|stop the runner itself factory config print [--json] the EFFECTIVE policy, defaults included factory config path where that file is read from @@ -46,9 +48,9 @@ usage: factory --help · --version exit codes - 0 ok / tier 1 / healthy 3 refused (not tier 1) · stalled foreman · + 0 ok / tier 1 / alive 3 refused (not tier 1) · stalled shift · 1 nothing (no lease) · pass a skill install only partly honoured - aborted 4 live lease with no poller watching it + aborted 4 live lease with no runner under it 2 usage, or a config that cannot be used @@ -270,7 +272,9 @@ cmd_config() { "budget\t\(.budget.mode), feed \(.budget.feed // "-"), ceiling \(.budget.ceiling) reserve \(.budget.reserve) fixer \(.budget.fixer) 5h-max \(.budget.window5hMax)", "after\t\(.afterMerge.commands | length) command(s) in \(.afterMerge.workdir // "the current directory")", "notify\t\(.notify.mode), source \(.notify.source), command \(if (.notify.command | length) == 0 then "-" else (.notify.command | join(" ")) end)", - "watchdog\tstale \(.watchdog.stale)s dead \(.watchdog.dead)s interval \(.watchdog.interval)s"' + "watchdog\tstale \(.watchdog.stale)s dead \(.watchdog.dead)s interval \(.watchdog.interval)s", + "runner\ta pass every \(.runner.interval)s while the lease is live", + "fixer\tcommand \(if (.fixer.command | length) == 0 then "-" else (.fixer.command | join(" ")) end), cap \(.fixer.cap) per repo per day"' )" while IFS=$'\t' read -r k v; do ui_trow "$k" "$v"; done <<<"$rows" ui_table_data @@ -394,7 +398,7 @@ cmd_doctor() { # Whether anything will actually REACH you is a readiness question, the same # shape as the budget feed above: a shift that merges correctly and cards - # nowhere is a shift whose red CI and dead foreman go unread until morning. + # nowhere is a shift whose red CI and stalled passes go unread until morning. # Nothing here blocks a merge, but a `command` the user typed and PATH cannot # find is a configured thing that cannot work — `afterMerge.workdir`'s # footing, so `afterMerge.workdir`'s severity. @@ -402,7 +406,7 @@ cmd_doctor() { local nmode nsource nmode=$(cfg .notify.mode); nsource=$(cfg .notify.source) case "$nmode" in - off) note notify "notify.mode is off — a red CI and a dead foreman card nowhere" ;; + off) note notify "notify.mode is off — a red CI and a stalled shift card nowhere" ;; command) local ncmd; ncmd=$(cfg '.notify.command[0]') # No source on this arm: `--source` is the trill call's flag, and a command @@ -417,6 +421,21 @@ cmd_doctor() { ;; esac + # The lane spawner is the second configured hook, and it is checked the way + # `notify.command` is: a program you typed that PATH cannot find blocks, + # because every lane would fail to start; none configured is a note, since + # the default IS none and a red branch is still reported and carded. + sec fixer + local fcmd fcap + fcmd=$(cfg '.fixer.command[0] // ""'); fcap=$(cfg .fixer.cap) + if [ -z "$fcmd" ]; then + note fixer "no fixer.command — a red default branch is reported and carded, and no lane is spawned for it" + elif have "$fcmd"; then + ok fixer "runs $fcmd, at most $fcap lane(s) per repo per day" + else + bad fixer "fixer.command names $fcmd, which is not on PATH — every lane would fail to start" + fi + sec state if mkdir -p "$FACTORY_STATE_DIR" 2>/dev/null && [ -w "$FACTORY_STATE_DIR" ]; then ok state-dir "$FACTORY_STATE_DIR" diff --git a/lib/common.sh b/lib/common.sh index f865ac1..bbd5b45 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -61,9 +61,16 @@ FACTORY_FLOOR_DENY='[ # ── defaults ────────────────────────────────────────────────────────────────── # Every default fails CLOSED: no repos in scope is a config error rather than a -# quiet pass, no budget feed is `fixer: no`, and the tier filter starts at -# docs-only. Widening any of them is the user's typed decision, in one file -# `factory config print` reads back to them. +# quiet pass, no budget feed is `fixer: no`, no `fixer.command` spawns nothing +# however red a branch is, and the tier filter starts at docs-only. Widening +# any of them is the user's typed decision, in one file `factory config print` +# reads back to them. +# +# `runner.interval` is the pass cadence the runner (`factory watchdog run`) +# keeps while a lease is live; `watchdog.stale` has to be longer than it, which +# the validator holds. `fixer.command` is handed ` ` when a red default branch clears the four fixer gates, and `fixer.cap` +# is the lanes-per-repo-per-day one of those gates counts against. factory_defaults() { cat <<'JSON' { @@ -84,7 +91,9 @@ factory_defaults() { "fixer": 5, "window5hMax": 80 }, "notify": { "mode": "auto", "command": [], "source": "factory" }, - "watchdog": { "stale": 2700, "dead": 5400, "interval": 300 } + "watchdog": { "stale": 2700, "dead": 5400, "interval": 300 }, + "runner": { "interval": 1200 }, + "fixer": { "command": [], "cap": 2 } } JSON } @@ -137,7 +146,7 @@ cfgj() { factory_load; printf '%s' "$FACTORY_CFG" | jq -c "$1"; } factory_validate() { local err err="$(printf '%s' "$FACTORY_CFG" | jq -r ' - [ ("scope", "tier1", "afterMerge", "budget", "notify", "watchdog") as $k + [ ("scope", "tier1", "afterMerge", "budget", "notify", "watchdog", "runner", "fixer") as $k | if (.[$k] | type) != "object" then "\($k) must be an object" else empty end ] | join("; ")')" || die "the policy could not be validated — jq failed reading $FACTORY_CONFIG_PATH" [ -z "$err" ] || die "$err (in $FACTORY_CONFIG_PATH)" @@ -207,12 +216,32 @@ factory_validate() { # Whole for the reason the budget dials are, and this is where it costs # most: a # fractional `dead` makes `[ "$quiet" -ge "$DEAD" ]` read false at every - # poll, so the foreman death this entire layer exists to notice is never + # tick, so the stalled shift this entire layer exists to notice is never # noticed and the lease stands until morning. That one fails OPEN, which # `tier1.maxLines` above does not — `[ "$churn" -le "$max" ]` refuses # every PR instead, with the nonsense cap printed in the reason. ([.watchdog.stale, .watchdog.dead, .watchdog.interval] | map(select(type != "number" or . != floor or . < 1)) | if length > 0 then "watchdog thresholds must be whole numbers of seconds, 1 or more" else empty end), - (if .watchdog.dead <= .watchdog.stale then "watchdog.dead must be greater than watchdog.stale" else empty end) + (if .watchdog.dead <= .watchdog.stale then "watchdog.dead must be greater than watchdog.stale" else empty end), + # The pass cadence, read by the same shell arithmetic the watchdog + # thresholds are, so whole for the same reason: a fractional interval makes + # `[ $((now - last)) -ge "$RUNNER_INTERVAL" ]` read false at every tick, + # and a runner that never finds a pass due is a live lease nobody is + # exercising — the standing grant this whole layer exists to take away. + (if (.runner.interval | type) != "number" or .runner.interval != (.runner.interval | floor) or .runner.interval < 1 then "runner.interval must be a whole number of seconds, 1 or more" else empty end), + # The stall threshold has to sit OUTSIDE the cadence, or every gap + # between two passes is a stall: a `shift-stalled` line and a card + # before each pass, then `shift-resumed` after it, all night. And with + # `dead` inside the cadence too, the runner revokes its own lease + # between two passes it was about to run. + (if (.runner.interval | type) == "number" and (.watchdog.stale | type) == "number" and .watchdog.stale <= .runner.interval then "watchdog.stale must be greater than runner.interval — a stall threshold inside the pass cadence is a stall between every two passes" else empty end), + # Same argv contract as notify.command, and the same two refusals: a + # string where the list goes, and a first word with no program in it. + (if (.fixer.command | type) != "array" then "fixer.command must be an array of argv words, not a string" else empty end), + (if (.fixer.command | type) == "array" and (.fixer.command | length) > 0 and ((.fixer.command[0] | type) != "string" or (.fixer.command[0] | length) == 0) then "fixer.command starts with an empty word — there is no program there to run" else empty end), + # Read back with `-ge` against a count of log lines, so whole; 0 is a + # legal way to spell "report the red, spawn nothing", and it is logged + # as the reason on every skip. + (if (.fixer.cap | type) != "number" or .fixer.cap != (.fixer.cap | floor) or .fixer.cap < 0 then "fixer.cap must be a whole number of lanes per repo per day, 0 or more" else empty end) ] | join("; ")')" || die "the policy could not be validated — jq failed reading $FACTORY_CONFIG_PATH" [ -z "$err" ] || die "$err (in $FACTORY_CONFIG_PATH)" } diff --git a/lib/ui.sh b/lib/ui.sh index 3ae9f85..278cd8d 100644 --- a/lib/ui.sh +++ b/lib/ui.sh @@ -119,8 +119,8 @@ out() { # # For a line whose payload is one unbreakable token: snug's `ui_fold` hard-cuts # a word longer than the budget, so a CI failure's URL comes out split across a -# hanging indent, unclickable and no longer the one-line shape the foreman's -# skill reads. A terminal's own wrap keeps the token contiguous in the buffer, +# hanging indent, unclickable and no longer the one-line shape the log is +# read by. A terminal's own wrap keeps the token contiguous in the buffer, # which is the better failure at 60 columns. out_line() { local painted diff --git a/libexec/factory-lease b/libexec/factory-lease index 57f5864..996ba7f 100755 --- a/libexec/factory-lease +++ b/libexec/factory-lease @@ -1,11 +1,13 @@ #!/usr/bin/env bash # factory-lease — the standing merge grant the shift runs under. # -# factory lease grant 12h # tier-1 authority for 12 hours -# factory lease status # exit 0 live · 1 none/expired +# factory lease grant 12h # tier-1 authority for 12 hours +# factory lease grant indefinitely # the same, until revoked +# factory lease status # exit 0 live · 1 none/expired # factory lease revoke # -# A lease is (tier ceiling, expiry). While one is live, `factory shift` may +# A lease is (tier ceiling, expiry). While one is live, the runner `grant` +# starts (`factory watchdog run`) passes `factory shift` on a cadence and may # MERGE tier-1 PRs on its own; expired or revoked, everything queues at "PR # open" — i.e. the failure mode of the whole factory is the ordinary # human-in-the-loop workflow. Only tier 1 exists today; the column is there so @@ -14,6 +16,14 @@ # State is one TSV line — \t\t — under the # state dir, machine-local on purpose: authority to merge unattended belongs to # this machine's owner having typed `grant`, never to a file a PR could edit. +# +# An INDEFINITE lease writes the word `never` where the epoch goes. A word and +# not a number on purpose: every reader that only knows epochs — this file's +# own `status` before it learned the word, a tool reading the file directly — +# fails `num` on it and reports the lease unreadable, which is exit 1 and "may +# not merge". A sentinel epoch (0, or 2^62) would read as expired in one reader +# and as live for a century in another. The bound on an indefinite lease is +# tier 1 itself, which is why it is allowed at all. # Every jq program in this file is single-quoted on purpose: `$d`, `$repo` and # the rest are jq's OWN variables, bound by --arg/--argjson, and expanding them # in the shell first is exactly the bug the quoting prevents. @@ -43,6 +53,7 @@ usage() { cat >&2 <<'EOF' usage: factory lease grant # 30m / 12h / 2d — tier-1 authority until then + factory lease grant indefinitely # tier-1 authority until `revoke` factory lease status [--json] # exit 0 live · 1 none/expired factory lease revoke EOF @@ -102,12 +113,20 @@ parse_duration() { # 30m / 12h / 2d → seconds echo "$secs" } -emit() { # emit +# `expires` is an epoch, or `never` for an indefinite lease. In the JSON that +# is `expires: null`, `secondsLeft: null` and `indefinite: true` — null rather +# than a large number, because a caller drawing a countdown off `secondsLeft` +# should draw "until revoked" and not "1,051 years". +emit() { # emit if [ "$json" = 1 ]; then - jq -nc --argjson live "$1" --argjson tier "$2" --argjson expires "$3" \ - --argjson granted "$4" --arg line "$5" \ - '{live: $live, tier: $tier, expires: $expires, granted: $granted, - secondsLeft: (if $live then ($expires - (now | floor)) else 0 end), line: $line}' + local expires="$3" indefinite=false + if [ "$3" = never ]; then expires=null; indefinite=true; fi + jq -nc --argjson live "$1" --argjson tier "$2" --argjson expires "$expires" \ + --argjson granted "$4" --argjson indefinite "$indefinite" --arg line "$5" \ + '{live: $live, tier: $tier, expires: $expires, granted: $granted, indefinite: $indefinite, + secondsLeft: (if $live and $indefinite then null + elif $live then ($expires - (now | floor)) else 0 end), + line: $line}' elif [ "$1" = true ]; then out_ok "$5" else @@ -122,45 +141,51 @@ now=$(date +%s) case "${1:-}" in grant) if [ -z "${2:-}" ] || [ $# -gt 3 ]; then usage; fi - secs=$(parse_duration "$2") tier="${3:-1}" [ "$tier" = 1 ] || die "only tier 1 exists — widen tier1.allow instead; a wider grant would be a new tier value here" - # The EXPIRY is what gets written, so it is checked here, where the clock it - # is added to is in scope. Both ways of not being one are the same failure to - # whoever reads the line: `now + secs` past 2^63 wraps negative, and `status` - # then calls the state file this verb has just written unreadable; an expiry - # `date` will not format leaves the ✓ ending on a bare "until", because `at` - # answers with nothing. A duration can clear parse_duration and still land - # on either — 106751991167300d multiplies without wrapping. - expires=$((now + secs)) - # `|| when=""` because a `date` that refuses the stamp exits non-zero, and - # under `set -e` that status inside a command substitution ends the script - # before the check below can turn it into a sentence. The empty answer IS - # the signal here, so it is caught rather than propagated. - when=$(at "$expires") || when="" - if [ "$expires" -le "$now" ] || [ -z "$when" ]; then - die "duration '$2' puts the expiry past what this machine can represent" + if [ "$2" = indefinitely ]; then + expires=never + line="lease: tier $tier indefinitely — until revoked" + else + secs=$(parse_duration "$2") + # The EXPIRY is what gets written, so it is checked here, where the clock + # it is added to is in scope. Both ways of not being one are the same + # failure to whoever reads the line: `now + secs` past 2^63 wraps negative, + # and `status` then calls the state file this verb has just written + # unreadable; an expiry `date` will not format leaves the ✓ ending on a + # bare "until", because `at` answers with nothing. A duration can clear + # parse_duration and still land on either — 106751991167300d multiplies + # without wrapping. + expires=$((now + secs)) + # `|| when=""` because a `date` that refuses the stamp exits non-zero, and + # under `set -e` that status inside a command substitution ends the script + # before the check below can turn it into a sentence. The empty answer IS + # the signal here, so it is caught rather than propagated. + when=$(at "$expires") || when="" + if [ "$expires" -le "$now" ] || [ -z "$when" ]; then + die "duration '$2' puts the expiry past what this machine can represent" + fi + line="lease: tier $tier until $when" fi mkdir -p "$FACTORY_STATE_DIR" printf '%s\t%s\t%s\n' "$expires" "$tier" "$now" >"$LEASE" - emit true "$tier" "$expires" "$now" "lease: tier $tier until $when" - # A live lease always has a watchdog, and it is started HERE rather than by - # the foreman so the invariant is structural: the one thing that must not - # depend on the agent still working is the thing that notices it stopped. - # `ensure` is idempotent (one pidfile per machine, claimed by hardlink), so a - # second grant inside a live shift extends the lease without racing — and it - # is the same verb the foreman re-runs each wakeup, so there is one - # implementation of "make sure something is watching". + emit true "$tier" "$expires" "$now" "$line" + # A live lease always has a runner, and it is started HERE so the invariant + # is structural: the lease is the on/off switch, and nothing else has to be + # remembered. `ensure` is idempotent (one pidfile per machine, claimed by + # hardlink), so a second grant inside a live shift extends the lease without + # starting a second runner — and on a machine whose launchd keeps the runner + # alive, `ensure` finds it already running and does nothing. if [ "${FACTORY_NO_WATCHDOG:-0}" != 1 ] && [ -x "$WATCHDOG" ]; then # Its report is not this verb's, so stdout goes. Its REFUSAL is: exit 2 is - # the watchdog declining its own configuration — an environment override - # it may not honour — and a poller that never started is the one thing a - # ✓ grant may not be quiet about. The lease stands without it, and the - # line says so, until `watchdog once` says NO POLLER. + # the runner declining its own configuration — an environment override it + # may not honour — and a runner that never started is the one thing a ✓ + # grant may not be quiet about. The lease stands without it, and the line + # says so, until `watchdog once` says NO RUNNER. rc=0 err=$("$WATCHDOG" ensure 2>&1 >/dev/null) || rc=$? [ "$rc" -ne 2 ] || - out_warn "watchdog did not start — $(oneline "$err") — the lease stands with nothing watching it" + out_warn "runner did not start — $(oneline "$err") — the lease stands and no pass will run under it" fi ;; status) @@ -168,9 +193,15 @@ status) if [ ! -s "$LEASE" ]; then emit false 0 0 0 "lease: none"; exit 1; fi # The granted stamp is the third field; `_` consumes it where nothing reads it. IFS=$'\t' read -r expires tier granted <"$LEASE" - num "${expires:-}" || { emit false 0 0 0 "lease: unreadable"; exit 1; } + if [ "${expires:-}" != never ] && ! num "${expires:-}"; then + emit false 0 0 0 "lease: unreadable"; exit 1 + fi num "${granted:-}" || granted=0 num "${tier:-}" || tier=0 + if [ "$expires" = never ]; then + emit true "$tier" never "$granted" "lease: tier $tier · indefinite · until revoked" + exit 0 + fi if [ "$now" -ge "$expires" ]; then emit false "$tier" "$expires" "$granted" "lease: expired $(at "$expires")"; exit 1 fi @@ -181,9 +212,11 @@ status) revoke) [ $# -eq 1 ] || usage rm -f "$LEASE" - # The watchdog exits on its own the next time it reads a dead lease, but that - # is up to one poll interval of a poller outliving the thing it watches. - # Stopping it here keeps "no lease" and "nothing running" the same moment. + # The runner exits on its own the next time it reads a dead lease, but that + # is up to one tick of a runner outliving its authority — and a pass it had + # already started finishes under `factory shift`'s own lease check, which + # now says no. Stopping it here keeps "no lease" and "nothing running" the + # same moment. if [ -x "$WATCHDOG" ]; then "$WATCHDOG" stop >/dev/null 2>&1 || true; fi emit false 0 0 0 "lease: revoked" ;; diff --git a/libexec/factory-shift b/libexec/factory-shift index 303c253..2608fc2 100755 --- a/libexec/factory-shift +++ b/libexec/factory-shift @@ -8,15 +8,16 @@ # # One pass does four things, in order: # budget read the configured usage feed and answer the one question the -# foreman asks of it: can the account afford an agent lane right +# runner asks of it: can the account afford an agent lane right # now? The line ends `fixer: yes` or `fixer: no ()`, and the -# foreman spawns on nothing else. +# runner (`factory watchdog run`) spawns on nothing else. # merge every open PR in scope through factory-tier; tier 1 + live lease # → `gh pr merge`. Anything else is printed with its reason and # left for the morning. # after if anything merged: the configured afterMerge commands, in order. # ci the latest completed run on each repo's default-branch, red → a -# CI-RED line (the foreman's cue) and a fault card. +# CI-RED line (the runner's cue to consider a fixer lane) and a +# fault card. # # Four lines say the pass could not SEE, and none of them means "nothing to # report": `prs-unknown` (a repo's open PRs unlistable), `tier-unknown` (one PR @@ -76,7 +77,7 @@ say() { # say [ok|warn|err|url|muted] err) out_bad "$2" ;; # A line whose payload is a URL. Same mark as `err`, no folding: the run URL # is one unbreakable token and a narrow window would otherwise cut it in - # half, which costs the foreman the only thing that line is carrying. + # half, which costs the reader the only thing that line is carrying. url) out_bad_url "$2" ;; *) out_info "$2" ;; esac @@ -105,9 +106,10 @@ say "$(J --arg d "$(factory_policy_digest)" --arg v "$FACTORY_VERSION" \ # because a threshold written out in English, in another file, over numbers only # this one can see, is a threshold no test can reach — and its refusal is the # same word as a correct refusal, so a gate stuck shut looks exactly like a gate -# doing its job. What stays the foreman's is whether a red main is WORTH a lane; -# whether the account can afford one is answered here, once, and logged where -# the morning reads it. +# doing its job. The runner reads the verdict off the `budget` event's `fixer` +# and `reason` fields and applies its other three gates on top; whether the +# account can afford a lane is answered here, once, and logged where the +# morning reads it. # # The question is forward-looking on purpose. "Is the week spent no faster than # the clock so far" is a question nobody has, and it cannot be answered yes by @@ -189,16 +191,21 @@ else headroom=$((CEILING - pw - reserve)) # 5h first, and it is not about the week at all: a factory that saturates # the rolling window at 4 a.m. rate-limits the person who sits down at 9. + # + # The reason is its own field in the event, not only the tail of the + # human line: the runner quotes it on its `fixer-skipped: … budget — …` + # line, and one verb never parses another's human line. + budget_why="" if [ "$p5" -ge "$MAX5H" ]; then - budget_verdict="no (5h window at ${p5}%)" + budget_why="5h window at ${p5}%" elif [ "$headroom" -lt "$FIXER" ]; then - budget_verdict="no (headroom ${headroom} pts, one fixer needs ${FIXER})" - else - budget_verdict=yes + budget_why="headroom ${headroom} pts, one fixer needs ${FIXER}" fi + if [ -n "$budget_why" ]; then budget_verdict="no ($budget_why)"; else budget_verdict=yes; fi say "$(J --argjson p5 "$p5" --argjson pw "$pw" --argjson r "$reserve" --argjson h "$headroom" \ - --argjson ok "$([ "$budget_verdict" = yes ] && echo true || echo false)" \ - '{event:"budget", mode:"metered", window5h:$p5, week:$pw, reserve:$r, headroom:$h, fixer:$ok}')" \ + --argjson ok "$([ "$budget_verdict" = yes ] && echo true || echo false)" --arg why "$budget_why" \ + '{event:"budget", mode:"metered", window5h:$p5, week:$pw, reserve:$r, headroom:$h, fixer:$ok, + reason:(if $ok then null else $why end)}')" \ "budget: 5h ${p5}% · week ${pw}% · reserve ${reserve} pts · headroom ${headroom} pts · fixer: $budget_verdict" fi fi @@ -225,7 +232,7 @@ fi # merged" — character-for-character what a genuinely quiet night prints. Same # rule the budget block states about itself: degrade to a stated unknown, never # to an answer that happens to parse. A pass that sensed nothing exits non-zero -# so the foreman cannot read it as a clean one. +# so the runner cannot read it as a clean one. abort() { # abort say "$(J --arg r "$1" '{event:"aborted", reason:$r}')" "pass ABORTED: $1" err # This one DOES card, where ci-unknown does not, and the asymmetry is blast @@ -295,7 +302,7 @@ for repo in $repos; do # and anything else is `set -e` aborting inside it — most often on a `gh` # call. Those are three answers, not two. "Refused" and "could not be # judged" collapsing into one `queued:` line is the expensive one: a - # foreman is told that queued rows need nothing because they are the + # skill says that queued rows need nothing because they are the # morning's by design, so a PR nobody managed to sense gets filed as one # nobody needs to look at. # Asked for JSON, and that is not a style choice. One verb parsing @@ -367,13 +374,20 @@ for repo in $repos; do # 404 rather than returning an empty list, so that repo would be `ci-unknown` # on every pass forever. If one is, the answer is `scope.exclude` or an arm # that recognises the 404, never a wider `||`. + # + # The head SHA and the branch ride in the event beside the URL, for the + # runner's fixer gates: `head` is what "never a second lane for the same + # failure" is counted on, and `branch` is what the lane is handed. Both are + # facts this pass already had — the query is `-b "$base"`, so every red + # reported here is on the default branch by construction, and that is the + # fourth gate answered before the runner asks it. if run_row=$(gh run list -R "$repo" -b "$base" -L1 \ - --json conclusion,url -q '.[] | [.conclusion, .url] | @tsv' 2>"$ERRF"); then - read -r conclusion url <<<"$run_row" + --json conclusion,url,headSha -q '.[] | [.conclusion, .url, .headSha] | @tsv' 2>"$ERRF"); then + read -r conclusion url run_head <<<"$run_row" case "${conclusion:-}" in failure | timed_out | startup_failure) - say "$(J --arg r "$repo" --arg u "$url" --arg c "$conclusion" \ - '{event:"ci-red", repo:$r, url:$u, conclusion:$c}')" "CI-RED: $repo $url" url + say "$(J --arg r "$repo" --arg u "$url" --arg c "$conclusion" --arg h "${run_head:-}" --arg b "$base" \ + '{event:"ci-red", repo:$r, url:$u, conclusion:$c, head:$h, branch:$b}')" "CI-RED: $repo $url" url notify fault "CI red on $repo $base" "$url" ;; esac @@ -382,7 +396,7 @@ for repo in $repos; do '{event:"ci-unknown", repo:$r, stderr:$e}')" \ "ci-unknown: $repo — $base's latest run not read: $(oneline "$(cat "$ERRF")")" warn fi - conclusion=""; url="" + conclusion=""; url=""; run_head="" done # ── after merge ─────────────────────────────────────────────────────────────── diff --git a/libexec/factory-watchdog b/libexec/factory-watchdog index 5f38af1..326b53d 100755 --- a/libexec/factory-watchdog +++ b/libexec/factory-watchdog @@ -1,70 +1,113 @@ #!/usr/bin/env bash -# factory-watchdog — notice that the FOREMAN died, which no pass can report. +# factory-watchdog — the RUNNER: while the lease is live, pass `factory shift` +# on a cadence, spawn the fixer lanes the four gates let through, and notice +# when the passes stop landing. # -# factory watchdog once # 0 healthy · 3 stalled · 4 no poller · 1 no lease -# factory watchdog ensure # the same, and start a poller if one is missing -# factory watchdog run # poll until the lease ends +# factory watchdog once # 0 alive · 3 stalled · 4 no runner · 1 no lease +# factory watchdog ensure # the same, and start a runner if one is missing +# factory watchdog run # the runner itself: pass until the lease ends # factory watchdog stop # -# `factory shift`'s four unknown lines keep a pass that could not SEE from -# reading as a quiet night. This is the layer under them, and it exists because -# every one of those lines has to be written by a pass that RAN: the foreman is -# an agent session driving a loop, and that loop continues only if a turn -# completes and schedules the next wakeup. A turn that ends in an error -# schedules nothing, so nothing is left running and nothing is left to say so — -# the log's last line is an ordinary `pass done` while the lease goes on -# standing. +# This file used to be a poller that measured the liveness of a FOREMAN — an +# agent session driving `factory shift` in a loop and applying four rules to +# every CI-RED line it printed — and revoked the lease when that session died. +# The foreman is gone. Its judgement was four string checks and a retry +# counter, and a rule that is four string checks is code, not a skill: written +# here it is deterministic, tested, and needs no agent pane to have stayed +# alive. `factory lease grant 12h` now starts this, and this runs the night. # -# THE HEARTBEAT IS THE SHIFT LOG'S MTIME. Every `say` in `factory-shift` -# appends to $FACTORY_STATE_DIR/shift-YYYYMMDD.log, so that file's modification -# time is "the last moment the shift was demonstrably alive" — during a pass as -# well as at the end of one. It costs `factory-shift` nothing and adds no second -# artifact that could disagree with the first. +# ONE PASS EVERY runner.interval (20m). `factory shift --json` is run with its +# events captured and its human lines going to the shift log as they always +# have. Off the events, three things: +# +# retry a pass that could not SEE — `prs-unknown`, `tier-unknown`, +# `ci-unknown`, `after-merge-failed`, or a `pass ABORTED` — is run +# once more at the next tick (watchdog.interval, 5m) rather than +# the next cadence. Once: a second unknown is a story for the +# morning, not a loop, and both lines are in the log. +# fixers every `ci-red` event is put through the four gates below, and +# the verdict is a line in the log whichever way it went. +# failed a shift that exited before writing anything — a config gone +# invalid mid-night, `jq` missing — is `pass-failed` with its +# stderr, and NOT a heartbeat (see `note`). +# +# THE FOUR FIXER GATES, in the order their reason is quoted. A red default +# branch gets a lane only when all four hold, and each refusal names itself: +# +# command `fixer.command` is configured. The default is none, so a machine +# that has not said what a lane IS spawns nothing however red the +# branch — `fixer-skipped: … no fixer.command configured`. +# novelty no `fixer-spawned: ` line in ANY shift log. A +# head SHA is globally unique, so the search is every log and not +# tonight's: a fix that broke CI again does not get a third machine, +# and a red that stood across midnight does not get a lane a day. +# cap fewer than `fixer.cap` (2) `fixer-spawned: ` lines in +# TODAY's log. Per calendar day, because the log is. +# budget the pass's `budget` event says `fixer: true`. Its `reason` is +# quoted otherwise, and `budget unknown` is a refusal like any +# other — an unreadable quota is not permission. +# +# The fourth rule the foreman carried — default branch only — is answered +# before this file asks it: `factory shift` queries `-b "$base"`, so every +# `ci-red` it reports is on the default branch by construction. The event +# carries `branch` so the lane is handed a fact rather than a re-derivation. +# +# A lane is `fixer.command` plus three words, ` `, run in this process's turn with stdin closed. It has to RETURN once +# the lane is started — a spawner, not the lane — because a command that waits +# for the lane to finish holds every pass after it. Exit 0 is `fixer-spawned: +# `; anything else is `fixer-failed` with its stderr, and a +# card, since a configured thing that cannot work is `after-merge-failed`'s +# shape. A failed spawn counts toward neither cap nor novelty: nothing was +# spawned, and the next pass tries again. +# +# THE HEARTBEAT IS STILL THE SHIFT LOG'S MTIME, and it still matters, because +# a runner that is alive is not the same as passes that are landing. Every +# `say` in `factory-shift` appends to $FACTORY_STATE_DIR/shift-YYYYMMDD.log, +# so that file's mtime is "the last moment a pass was demonstrably running". +# What can make it go quiet under a live runner is a shift that dies before +# its first `say` — every twenty minutes, with the lease standing — or one +# pass hanging inside a `gh` call. Two thresholds, as before: +# +# watchdog.stale (45m) quiet for longer than the cadence plus slack. Says so +# in the log (`shift-stalled`), draws ONE card, and the +# lease stands. +# watchdog.dead (90m) quiet for twice that. The lease is REVOKED and this +# exits (`shift-dead`). Authority to merge unattended +# belongs to passes that are running; the morning +# should find the ordinary human-in-the-loop workflow +# rather than a standing grant nobody is exercising. # # Which forces two things this file would be wrong without: # # 1. This script's own lines go in that same log — one morning report, not -# two — so `note` RESTORES the mtime after appending. A watchdog whose -# `foreman-stalled` line reset the clock it reads would find itself healthy -# on the next poll, write `foreman-resumed`, and alternate forever without -# the quiet time ever reaching $DEAD. The revoke below would then be -# unreachable code. +# two — so `note` RESTORES the mtime after appending. A runner whose +# `pass-failed` line reset the clock it reads would keep a lease standing +# under a shift that cannot start, forever, with a named line every +# twenty minutes reading as a heartbeat. Only a pass is a heartbeat. # 2. `last_seen` is the LATER of the newest log's mtime and the lease's own # grant stamp, never just the log. Logs are per-day and never swept, so on # every night after the first a fresh grant would otherwise be judged # against yesterday's file and revoked within milliseconds of being made. # -# TWO THRESHOLDS, because a blip and a death want different answers: +# AND A SLEEPING MACHINE IS NOT A STALLED SHIFT. Both thresholds count time +# this process was AWAKE for, not wall clock: a machine that suspended for two +# hours wakes with the log two hours old through nobody's fault. The loop +# notices its own `sleep` overrunning and subtracts the excess. SUBTRACTED, +# rather than forgiven with a grace window, which is what this did first and +# got wrong: a laptop that suspends and wakes all night renews a window faster +# than it expires, so a shift that genuinely could not run kept its lease +# until morning. Awake time accumulates across any number of naps; only the +# naps themselves are free. # -# watchdog.stale (45m) quiet for longer than the cadence plus slack. Says so -# in the log and draws ONE card. The lease stands — a -# foreman whose network dropped for one turn can still -# be mid-retry, and revoking under it would turn a -# recoverable blip into a shift that needs a person. -# watchdog.dead (90m) quiet for twice that. The lease is REVOKED and this -# exits. Authority to merge unattended belongs to a -# foreman that is running; the morning should find the -# ordinary human-in-the-loop workflow rather than a -# standing grant nobody is exercising. -# -# A recovery re-arms the card, so a night that stalls twice reports twice. -# -# AND A SLEEPING MACHINE IS NOT A DEAD FOREMAN. Both thresholds count time this -# process was AWAKE for, not wall clock: a machine that suspended for two hours -# wakes with the log two hours old through nobody's fault, the foreman included. -# The loop notices its own `sleep` overrunning and subtracts the excess. -# SUBTRACTED, rather than forgiven with a grace window, which is what this did -# first and got wrong: a laptop that suspends and wakes all night renews a -# window faster than it expires, so a genuinely dead foreman kept its lease -# until morning and never even drew a stall card. Awake time accumulates across -# any number of naps; only the naps themselves are free. -# -# WHAT IT DELIBERATELY DOES NOT DO: run `factory shift` itself. It could — the -# script is deterministic and the lease is the authority it would run under — -# but merging with no foreman means a `CI-RED` nobody reads and a `merge-failed` -# nobody retries, which is a factory that keeps its hands moving after its eyes -# have closed. Widening this into a headless foreman is a design change first, -# not a flag here. +# WHAT KEEPS THE RUNNER ITSELF ALIVE is not this file. A runner can be lost to +# a reboot, a panic or an OOM kill, and there is deliberately no second process +# watching for that: on a machine whose launchd owns it, KeepAlive restarts it +# and it passes again; anywhere else, `lease grant` and `watchdog ensure` start +# one, and `watchdog once` — which `doctor` carries — says NO RUNNER at exit 4 +# for as long as a live lease has none. A dead runner restarts instead of +# being reported, and a lease it left standing is the status quo: PRs open, +# waiting for you. # Every jq program in this file is single-quoted on purpose: `$d`, `$repo` and # the rest are jq's OWN variables, bound by --arg/--argjson, and expanding them # in the shell first is exactly the bug the quoting prevents. @@ -77,19 +120,21 @@ factory_load SELF="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" LEASECMD="$(dirname "$SELF")/factory-lease" +SHIFTCMD="$(dirname "$SELF")/factory-shift" LEASE="$FACTORY_STATE_DIR/lease" PIDFILE="$FACTORY_STATE_DIR/watchdog.pid" # Env beats config so a suite can make a 45-minute threshold reachable in # seconds without writing a config file for every case — and only DOWNWARD. A -# poller inherits the environment of whoever ran `lease grant`, and on a night -# shift that is the foreman: a variable that could lengthen `dead` is a foreman -# able to keep its lease after it dies, which is the one thing this script -# exists to take away. Refused rather than clamped, because a number that was -# set and silently not used is the swallow every verb here refuses — and -# `config print`'s watchdog row is then still what is in force. -threshold() { # threshold +# runner inherits the environment of whoever ran `lease grant`: a variable that +# could lengthen `dead` is a lease able to stand after its passes stopped, +# which is the one thing this script exists to take away, and one that could +# lengthen `runner.interval` is a night of fewer passes than the policy says. +# Refused rather than clamped, because a number that was set and silently not +# used is the swallow every verb here refuses — and `config print`'s rows are +# then still what is in force. +threshold() { # threshold local from_cfg from_env - from_cfg=$(cfg ".watchdog.$2") + from_cfg=$(cfg ".$2") from_env="${!1:-}" if [ -z "$from_env" ]; then printf '%s\n' "$from_cfg"; return 0; fi # Digits, at most 18 of them, then read in base 10 — the lease's own three @@ -102,23 +147,32 @@ threshold() { # threshold printf '%s\n' "$from_env"; return 0 fi fi - die "$1=$from_env — the environment may only shorten watchdog.$2 (${from_cfg}s), never lengthen it" + die "$1=$from_env — the environment may only shorten $2 (${from_cfg}s), never lengthen it" } -STALE=$(threshold FACTORY_STALE stale) || exit 2 -DEAD=$(threshold FACTORY_DEAD dead) || exit 2 -INTERVAL=$(threshold FACTORY_WATCHDOG_INTERVAL interval) || exit 2 -# The config's own invariant, re-asked over the overrides: the validator saw -# only the file, and a DEAD shortened under an unshortened STALE is a watchdog -# that revokes before it has warned anybody. +STALE=$(threshold FACTORY_STALE watchdog.stale) || exit 2 +DEAD=$(threshold FACTORY_DEAD watchdog.dead) || exit 2 +INTERVAL=$(threshold FACTORY_WATCHDOG_INTERVAL watchdog.interval) || exit 2 +RUNNER_INTERVAL=$(threshold FACTORY_RUNNER_INTERVAL runner.interval) || exit 2 +# The config's own invariants, re-asked over the overrides: the validator saw +# only the file. A DEAD shortened under an unshortened STALE is a runner that +# revokes before it has warned anybody; a STALE shortened under an unshortened +# cadence is a runner that calls its own gap between two passes a stall — and, +# with DEAD inside the cadence too, revokes its own lease between two passes it +# was about to run. [ "$DEAD" -gt "$STALE" ] || die "FACTORY_DEAD=$DEAD is not greater than the stale threshold ($STALE)" +[ "$STALE" -gt "$RUNNER_INTERVAL" ] || die "FACTORY_STALE=$STALE is not greater than the pass cadence ($RUNNER_INTERVAL)" +CAP=$(cfg .fixer.cap) +# The lane spawner's argv, read once. Empty is the default and the first gate. +FIXER_ARGV=() +while IFS= read -r a; do FIXER_ARGV+=("$a"); done < <(cfg '.fixer.command[]?') mkdir -p "$FACTORY_STATE_DIR" usage() { cat >&2 <<'EOF' usage: - factory watchdog once # 0 healthy · 3 stalled · 4 live lease, no poller · 1 no lease - factory watchdog ensure # the same check, and start a poller if one is missing - factory watchdog run # poll until the lease ends; revoke a dead foreman's + factory watchdog once # 0 alive · 3 stalled · 4 live lease, no runner · 1 no lease + factory watchdog ensure # the same check, and start a runner if one is missing + factory watchdog run # the runner: pass every runner.interval until the lease ends factory watchdog stop EOF exit 2 @@ -136,14 +190,14 @@ for a in "$@"; do done set -- ${args[@]+"${args[@]}"} # Every verb here takes no argument, so a second word is a call this cannot -# have understood — `stop now` stops the poller no differently from `stop`, and +# have understood — `stop now` stops the runner no differently from `stop`, and # a caller who typed the word meant something by it. [ $# -le 1 ] || usage -# The last moment the shift was demonstrably alive: the newest shift log's mtime -# or the moment the lease was granted, whichever is LATER. Prints nothing and -# fails when there is no lease — the caller's cue that there is no shift to -# watch, which is not the same as a shift that is quiet. +# The last moment a pass was demonstrably running: the newest shift log's +# mtime or the moment the lease was granted, whichever is LATER. Prints nothing +# and fails when there is no lease — the caller's cue that there is no shift +# to watch, which is not the same as a shift that is quiet. last_seen() { local newest="" f stamp m seen=0 for f in "$FACTORY_STATE_DIR"/shift-*.log; do @@ -162,25 +216,31 @@ last_seen() { printf '%s\n' "$seen" } +today_log() { printf '%s\n' "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log"; } + # Append to today's shift log WITHOUT counting as a heartbeat. One morning -# report is worth the bookkeeping; a watchdog that touched the mtime it reads -# would be reporting itself alive. See note 1 in the header. +# report is worth the bookkeeping; a runner that touched the mtime it reads +# would be reporting a pass that never ran. See note 1 in the header. # # The restore is skipped when the file grew by more than our own line while we -# were writing — that is `factory-shift` landing a real pass in the same -# instant, and rewinding the clock over it would erase the recovery this script -# is waiting for. -note() { - local f before size_before size_after line grew - f="$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" +# were writing — that is a pass landing in the same instant, and rewinding the +# clock over it would erase the heartbeat this script is waiting for. +note() { # note + local mark="$1" f before size_before size_after line grew + shift + f=$(today_log) before=$(last_seen) || before="" line="$(date '+%H:%M') $*" size_before=0 if [ -e "$f" ]; then size_before=$(wc -c <"$f" | tr -d ' '); fi # Marked on screen, plain in the log: the log is grepped for its event names - # by the morning's handover and by this repo's own tests, and a mark in front - # of one is a mark in front of every pattern that reads it. - out_warn "$*" + # by this repo's own tests and by whoever reads the morning, and a mark in + # front of one is a mark in front of every pattern that reads it. + case "$mark" in + ok) out_ok "$*" ;; + warn) out_warn "$*" ;; + *) out_info "$*" ;; + esac printf '%s\n' "$line" >>"$f" [ -n "$before" ] || return 0 # Measured in BYTES both sides. `${#line}` counts CHARACTERS, and these lines @@ -206,7 +266,7 @@ is_watchdog() { cmd=$(ps -p "$1" -o command= 2>/dev/null || ps -p "$1" -o args= 2>/dev/null) || return 1 case "$cmd" in *factory-watchdog\ run) return 0 ;; *) return 1 ;; esac } -poller_pid() { # prints a live watchdog's pid, or fails +runner_pid() { # prints a live runner's pid, or fails local pid [ -s "$PIDFILE" ] || return 1 pid=$(cat "$PIDFILE" 2>/dev/null) @@ -215,7 +275,7 @@ poller_pid() { # prints a live watchdog's pid, or fails printf '%s\n' "$pid" } -# 0 healthy · 3 stalled · 1 nothing to watch. The quiet duration goes to stdout +# 0 alive · 3 stalled · 1 nothing to watch. The quiet duration goes to stdout # in both of the first two cases so callers need not re-derive it. $1 is the # suspended-time discount; `once` has none to pass. check() { @@ -236,10 +296,11 @@ check() { return 0 } -# Start a detached poller unless a live one is already there. Shared by `ensure` -# and by `factory lease grant`, so the invariant has one implementation. +# Start a detached runner unless a live one is already there. Shared by +# `ensure` and by `factory lease grant`, so the invariant has one +# implementation. spawn() { - poller_pid >/dev/null && return 0 + runner_pid >/dev/null && return 0 nohup "$SELF" run >/dev/null 2>&1 & disown 2>/dev/null || true } @@ -247,8 +308,8 @@ spawn() { report() { # report if [ "$json" = 1 ]; then jq -nc --arg state "$1" --argjson quiet "${2:-0}" --arg line "$3" \ - --argjson pid "$(poller_pid 2>/dev/null || echo null)" \ - '{state: $state, quietSeconds: $quiet, pollerPid: $pid, line: $line}' + --argjson pid "$(runner_pid 2>/dev/null || echo null)" \ + '{state: $state, quietSeconds: $quiet, runnerPid: $pid, line: $line}' else # The state names the mark, so the two can never disagree: `alive` is the # only ✓ here, `no-lease` is the ordinary quiet of a machine nobody granted @@ -256,33 +317,123 @@ report() { # report # happening" are the ones worth a ⚠. case "$1" in alive) out_ok "$3" ;; - no-poller | stalled) out_warn "$3" ;; + no-runner | stalled) out_warn "$3" ;; *) out_info "$3" ;; esac fi } +# ── one pass, and what the runner does with it ──────────────────────────────── +# Where the shift's stderr and a fixer command's output are parked long enough +# to be quoted into a log line. Under the state dir rather than mktemp so a +# killed runner leaves it beside the log it belongs to. +ERRF="$FACTORY_STATE_DIR/.runner-stderr.$$" + +# The fixer gates, for one `ci-red` event. Every arm ends in a line. +fixer() { # fixer + local repo="$1" branch="$2" head="$3" url="$4" ok="$5" why="$6" n rc + if [ "${#FIXER_ARGV[@]}" -eq 0 ]; then + note info "fixer-skipped: $repo — no fixer.command configured" + return 0 + fi + # No head, no novelty judgement — and a lane spawned on a failure this + # cannot later recognise is a lane it could spawn again every pass. + if [ -z "$head" ]; then + note warn "fixer-skipped: $repo — the run's head SHA was not reported, so this failure could not be told from the last" + return 0 + fi + # `-F`: a repo name has a `/` in it and a SHA is hex, neither of which is a + # pattern. `-s`: a state dir with no logs yet is a machine on its first + # night, not an error. + if grep -qsF -- "fixer-spawned: $repo $head" "$FACTORY_STATE_DIR"/shift-*.log; then + note info "fixer-skipped: $repo — a lane was already spawned for $head" + return 0 + fi + # `grep -c` prints the count and exits 1 when it is zero, and prints nothing + # at all for a file that is not there — so the answer is read as a number + # and not as an exit status. + n=$(grep -csF -- "fixer-spawned: $repo " "$(today_log)" 2>/dev/null) || true + num "${n:-}" || n=0 + if [ "$n" -ge "$CAP" ]; then + note info "fixer-skipped: $repo — $n lane(s) already today, fixer.cap is $CAP" + return 0 + fi + if [ "$ok" != true ]; then + note info "fixer-skipped: $repo — budget: ${why:-budget unknown}" + return 0 + fi + if "${FIXER_ARGV[@]}" "$repo" "$branch" "$url" "$ERRF" 2>&1; then + note ok "fixer-spawned: $repo $head — lane on $branch for $url" + else + rc=$? + note warn "fixer-failed: $repo $head — fixer.command exited $rc: $(oneline "$(cat "$ERRF")")" + notify fault "factory: fixer lane for $repo did not start" + fi +} + +# Run `factory shift --json`, act on its events, and set `retry` if this pass +# earned one more. `$1` says whether THIS pass was already the retry. +pass() { # pass + local was_retry="$1" out rc events ok why kinds repo branch head url + out=$("$SHIFTCMD" --json "$ERRF"); rc=$? + # 0 is a pass; 1 is `pass ABORTED`, which the shift wrote itself and whose + # `aborted` event is below. Anything else is the shift not running at all — + # `die` at exit 2 before its first `say`, or a signal — and that is not a + # heartbeat: `note` restores the mtime, so a shift that cannot start reaches + # `dead` on the ordinary clock. + case "$rc" in + 0 | 1) ;; + *) + note warn "pass-failed: factory shift exited $rc before a pass could run — $(oneline "$(cat "$ERRF")")" + return 0 + ;; + esac + # Slurped into one array rather than read line by line, so an output that is + # not the event stream this file was promised is one refusal and not a + # silent zero of every count below. + if ! events=$(printf '%s\n' "$out" | jq -sc . 2>/dev/null); then + note warn "pass-failed: factory shift's --json output could not be read — $(oneline "$out")" + return 0 + fi + ok=$(jq -r '[.[] | select(.event == "budget")] | last | .fixer // false' <<<"$events") + why=$(jq -r '[.[] | select(.event == "budget")] | last | .reason // "budget unknown"' <<<"$events") + # The head SHA is read LAST, because it is the one field that can be empty + # and a tab is whitespace to `read`: two tabs around an empty field collapse + # into one, every column after it shifts left, and the URL would arrive as + # the head. A trailing empty field is the one place `read` leaves alone. + while IFS=$'\t' read -r repo branch url head; do + [ -n "$repo" ] || continue + fixer "$repo" "$branch" "${head:-}" "$url" "$ok" "$why" + done < <(jq -r '.[] | select(.event == "ci-red") | [.repo, .branch, .url, (.head // "")] | @tsv' <<<"$events") + kinds=$(jq -r '[.[] | select(.event | IN("aborted", "prs-unknown", "tier-unknown", "ci-unknown", "after-merge-failed")) | .event] | unique | join(", ")' <<<"$events") + if [ -n "$kinds" ] && [ "$was_retry" = 0 ]; then + retry=1 + note info "pass-retry: $kinds — one more pass at the next tick" + fi +} + case "${1:-}" in once | ensure) quiet=$(check); rc=$? - # `ensure` is the loop's verb: `grant` establishes the invariant once, and a - # poller can still be lost to a reboot, a panic or an OOM kill, which is the - # likeliest overnight foreman-killer after an API error precisely because it - # takes both of them at once. + # `ensure` is the verb that re-establishes the invariant: `grant` starts a + # runner once, and one can still be lost to a reboot, a panic or an OOM + # kill. On a launchd machine that is launchd's job and this finds nothing + # to do; anywhere else, this is what `doctor`'s NO RUNNER line tells you to + # run. if [ "$1" = ensure ] && [ "$rc" != 1 ]; then spawn sleep 1 fi case "$rc" in 0) - # A live lease with no poller is its own fault, and a distinct one: the - # shift is running and nothing is watching it. `grant` starts the poller + # A live lease with no runner is its own fault, and a distinct one: the + # lease stands and nothing is passing under it. `grant` starts the runner # with all output discarded, so a lost exec bit or a failed spawn is # otherwise silent — and silent is the one thing this script may not be. - if poller_pid >/dev/null; then - report alive "$quiet" "watchdog: shift alive — last pass $((quiet / 60))m ago, poller watching" + if runner_pid >/dev/null; then + report alive "$quiet" "watchdog: shift alive — last pass $((quiet / 60))m ago, runner up" else - report no-poller "$quiet" "watchdog: NO POLLER — the lease is live and nothing is watching the foreman" + report no-runner "$quiet" "watchdog: NO RUNNER — the lease is live and nothing is passing under it" rc=4 fi ;; @@ -292,7 +443,15 @@ once | ensure) exit "$rc" ;; run) - # One watchdog per machine, claimed by writing the pid to a private name and + # A runner with no shift to run is not a runner. Fatal for the reason an + # unreachable lease command is in `check`: a live lease under a process + # that could pass nothing is exactly the standing grant this exists to + # refuse, and it would otherwise sit here reporting itself alive. + if [ ! -x "$SHIFTCMD" ]; then + fail "cannot run $SHIFTCMD — no pass can be run, so there is nothing to be the runner of" + exit 2 + fi + # One runner per machine, claimed by writing the pid to a private name and # HARD-LINKING that into place: `ln` fails when the name is taken, which is # the same exclusion an O_EXCL create gives, but the file holds the pid # before it has the shared name. Neither a read-then-write nor an O_EXCL @@ -301,8 +460,8 @@ run) # not see — still holding a trap that would delete its successor's pidfile. # O_EXCL closed that and left a narrower door open: exclusive is not atomic, # so the pidfile exists EMPTY between the create and the write, and an empty - # pidfile is exactly what the recovery below reads as a crashed poller's. A - # racer landing in that window deleted a live poller's claim and started a + # pidfile is exactly what the recovery below reads as a crashed runner's. A + # racer landing in that window deleted a live runner's claim and started a # duplicate on top of it. # # Which makes the recovery's reading true rather than merely usual: nothing @@ -320,7 +479,7 @@ run) if (printf '%s\n' "$$" >"$mine") 2>/dev/null && [ -s "$mine" ]; then if ln "$mine" "$PIDFILE" 2>/dev/null; then claimed=1 - elif pid=$(poller_pid); then + elif pid=$(runner_pid); then out_info "watchdog: already running (pid $pid)" rm -f "$mine" exit 0 @@ -332,19 +491,36 @@ run) fi rm -f "$mine" [ "$claimed" = 1 ] || { fail "could not claim $PIDFILE"; exit 1; } - # Only ever remove a pidfile that is still OURS, so a poller exiting late + # Only ever remove a pidfile that is still OURS, so a runner exiting late # cannot delete the successor's. - trap 'if [ "$(cat "$PIDFILE" 2>/dev/null)" = "$$" ]; then rm -f "$PIDFILE"; fi' EXIT + trap 'if [ "$(cat "$PIDFILE" 2>/dev/null)" = "$$" ]; then rm -f "$PIDFILE"; fi; rm -f "$ERRF"' EXIT carded=0 slept=0 prev_seen="" + # The first pass is due at once: a grant should be followed by a pass within + # seconds, and a runner launchd restarted after a crash has a gap to close. + last_pass=0 + passes=0 + retry=0 while :; do + now=$(date +%s) + if [ "$retry" = 1 ] || [ $((now - last_pass)) -ge "$RUNNER_INTERVAL" ]; then + # Under a lease, or not at all — and the lease is asked here, not + # trusted from the last tick: a `revoke` lands between ticks. + if "$LEASECMD" status >/dev/null 2>&1; then + was_retry=$retry + retry=0 + last_pass=$now + passes=$((passes + 1)) + pass "$was_retry" + fi + fi quiet=$(check "$slept"); rc=$? - # Did the SHIFT move, as opposed to the clock? `note` restores the mtime - # after its own writes, so a heartbeat that advanced is always a real pass. - # This is what resets the discount and clears the stall — not `quiet` falling - # back under $STALE, which a large discount alone can do while the foreman is - # still just as dead. + # Did a PASS land, as opposed to the clock moving? `note` restores the + # mtime after its own writes, so a heartbeat that advanced is always a + # real pass. This is what resets the discount and clears the stall — not + # `quiet` falling back under $STALE, which a large discount alone can do + # while the shift is still just as stuck. seen=$(last_seen) || seen="" moved=0 if [ -n "$seen" ] && [ "$seen" != "$prev_seen" ]; then @@ -354,22 +530,27 @@ run) if [ "$moved" = 1 ]; then slept=0 if [ "$carded" = 1 ]; then - note "foreman-resumed: a pass landed after the stall" + note info "shift-resumed: a pass landed after the stall" carded=0 fi fi case "$rc" in 1) - # The lease ended. If the foreman was alive to the end it wrote its own - # handover; this exits quietly rather than adding a line to it. + # The lease ended. A runner that passed under it says so, once, so the + # log's last line is the end of the shift and not merely its last pass; + # one that never saw a lease — started after the expiry, or racing a + # revoke — has nothing to add. No lease, so `note` restores nothing. + [ "$passes" -eq 0 ] || note info "shift-over: lease ended — $passes pass(es) this run" exit 0 ;; 3) if [ "$quiet" -ge "$DEAD" ]; then IFS=$'\t' read -r expires _ _ <"$LEASE" - note "foreman-gone: no pass in $((quiet / 60))m awake — lease revoked (it stood until $(at "$expires"))" + stood="it stood until $(at "$expires")" + [ "$expires" != never ] || stood="it was indefinite" + note warn "shift-dead: no pass in $((quiet / 60))m awake — lease revoked ($stood)" # Drop the pidfile BEFORE revoking. `factory lease revoke` stops the - # watchdog, and the watchdog is us: left in place, that line would have + # runner, and the runner is us: left in place, that line would have # us kill ourselves mid-sentence, before the card saying why. rm -f "$PIDFILE" "$LEASECMD" revoke >/dev/null 2>&1 || true @@ -377,12 +558,12 @@ run) exit 3 fi if [ "$carded" = 0 ]; then - note "foreman-stalled: no pass in $((quiet / 60))m — lease still standing" + note warn "shift-stalled: no pass in $((quiet / 60))m — lease still standing" notify fault "shift stalled — $((quiet / 60))m with no pass" carded=1 fi ;; - *) : ;; # healthy — the recovery, if there was one, is handled above + *) : ;; # alive — the recovery, if there was one, is handled above esac before=$(date +%s) sleep "$INTERVAL" @@ -392,17 +573,17 @@ run) over=$(( $(date +%s) - before - INTERVAL )) if [ "$over" -gt $((INTERVAL * 2)) ]; then slept=$((slept + over)) - # Re-read the lease first: one that expired during the suspend belongs to a - # shift that already ended, and a line landing after the foreman's handover + # Re-read the lease first: one that expired during the suspend belongs to + # a shift that already ended, and a line landing after its `shift-over` # is noise in the one artifact the morning is meant to read. if "$LEASECMD" status >/dev/null 2>&1; then - note "machine-slept: $((over / 60))m suspended — not counted against the foreman" + note info "machine-slept: $((over / 60))m suspended — not counted against the shift" fi fi done ;; stop) - if pid=$(poller_pid) && kill "$pid" 2>/dev/null; then + if pid=$(runner_pid) && kill "$pid" 2>/dev/null; then out_ok "watchdog: stopped" else out_info "watchdog: not running" diff --git a/nix/skill.nix b/nix/skill.nix index ee4e2f0..ae3233e 100644 --- a/nix/skill.nix +++ b/nix/skill.nix @@ -1,16 +1,15 @@ # factory's agent skills, as a derivation. # -# TWO skills, one derivation, one directory each: +# ONE skill today, one directory, and a layout that takes more without an edit: # -# ai/SKILL.md → $out/factory/SKILL.md the verbs -# ai/nightshift/SKILL.md → $out/nightshift/SKILL.md the loop that drives them +# ai/SKILL.md → $out/factory/SKILL.md the verbs +# ai//SKILL.md → $out//SKILL.md any sibling, discovered # -# The second is not a second copy of the first. `factory` teaches an agent what -# to run when the user says "merge the safe PRs" or "why didn't #212 merge". -# `nightshift` teaches it the one thing that has no verb: the cadence, the fixer -# cap, and what to do with each line a pass printed. The tool is deliberately -# one pass at a time — something has to decide to call it again, and that -# something is judgement rather than a flag. +# There used to be a second, `nightshift`: the loop that called `factory shift` +# on a cadence and applied four rules to every CI-RED line. Its judgement was +# four string checks and a retry counter, so it is code now — `factory watchdog +# run` is the runner, started by `lease grant` — and the skill is gone. The +# loop below still walks `ai/*/` so a real second skill needs no edit here. # # `$out//SKILL.md` is the family standard's compliant-tool layout: one # nesting level, named for the SKILL rather than the tool, so a consumer links a diff --git a/share/config.example.json b/share/config.example.json index f893218..04c5748 100644 --- a/share/config.example.json +++ b/share/config.example.json @@ -26,5 +26,9 @@ "feed": null }, + "fixer": { + "command": [] + }, + "notify": { "mode": "auto", "source": "factory" } } diff --git a/test/agent-surface.bats b/test/agent-surface.bats index 400666a..d212ab2 100644 --- a/test/agent-surface.bats +++ b/test/agent-surface.bats @@ -45,7 +45,13 @@ setup() { cp "$BATS_TEST_DIRNAME/../VERSION" "$ROOT/" # `skill` reads `ai/` off FACTORY_HOME, and it reads the DIRECTORY rather # than a list — a $ROOT without it turns every case below into "no such - # skill" against a tool that ships two. + # skill". + # + # The tool ships ONE skill now, and the install cases below want two: they + # are about a run that lands SOME of what it was asked for, which needs a + # second file to land. So the fixture plants a sibling under `ai/second/`, + # which also keeps the discovery loop — `ai/*/SKILL.md`, the reason a third + # skill needs no edit in nix/skill.nix — exercised by something. # # Read-only, because that is the SHIPPED shape: `flake.nix` copies `ai/` into # the store and store files are 444, so `cp` inheriting the source's mode is @@ -53,6 +59,8 @@ setup() { # checkout is 644 and would leave the "install twice" case green forever. The # files only — the directories stay writable so bats can clean its tmpdir. cp -R "$BATS_TEST_DIRNAME/../ai" "$ROOT/ai" + mkdir -p "$ROOT/ai/second" + printf -- '---\nname: second\ndescription: a sibling skill the fixture plants so install cases have two files to land\n---\n# second\n' >"$ROOT/ai/second/SKILL.md" chmod a-w "$ROOT"/ai/SKILL.md "$ROOT"/ai/*/SKILL.md FACTORY="$ROOT/bin/factory" @@ -186,6 +194,40 @@ EOF [[ "$output" != *"not on PATH"* ]] } +# ── the lane spawner, the second configured hook ────────────────────────────── +# `fixer.command` is what the runner hands ` ` to on a red +# default branch. Same two answers as `notify.command`, for the same reasons. + +@test "doctor names a fixer.command that PATH cannot find, and blocks on it" { + printf '{"scope":{"orgs":["hausfold"]},"fixer":{"command":["spawn-a-lane","--bg"]}}\n' >"$FACTORY_CONFIG" + run "$FACTORY" doctor --json + [ "$status" -eq 2 ] + [ "$(jq -r '.checks[] | select(.check == "fixer") | .state' <<<"$output")" = bad ] + [[ "$(jq -r '.checks[] | select(.check == "fixer") | .line' <<<"$output")" == *"spawn-a-lane"*"not on PATH"* ]] +} + +@test "a fixer.command that exists is green, and names what it will run and the cap" { + printf '#!/usr/bin/env bash\n' >"$TMP/bin/spawn-a-lane" + chmod +x "$TMP/bin/spawn-a-lane" + printf '{"scope":{"orgs":["hausfold"]},"fixer":{"command":["spawn-a-lane"],"cap":3}}\n' >"$FACTORY_CONFIG" + run "$FACTORY" doctor --json + [ "$(jq -r '.checks[] | select(.check == "fixer") | .state' <<<"$output")" = ok ] + [[ "$(jq -r '.checks[] | select(.check == "fixer") | .line' <<<"$output")" == *"runs spawn-a-lane, at most 3 lane(s)"* ]] +} + +@test "no fixer.command is a note — the default is none, and a red branch is still reported" { + run "$FACTORY" doctor --json + [ "$(jq -r '.checks[] | select(.check == "fixer") | .state' <<<"$output")" = warn ] + [[ "$(jq -r '.checks[] | select(.check == "fixer") | .line' <<<"$output")" == *"no fixer.command"* ]] +} + +@test "config print has a row for the runner and one for the fixer" { + run "$FACTORY" config print + [ "$status" -eq 0 ] + [[ "$output" == *"runner"*"a pass every 1200s"* ]] + [[ "$output" == *"fixer"*"command -, cap 2 per repo per day"* ]] +} + @test "notify.mode off is a note, not a block — it is a decision, not a fault" { printf '{"scope":{"orgs":["hausfold"]},"notify":{"mode":"off"}}\n' >"$FACTORY_CONFIG" run "$FACTORY" doctor @@ -208,11 +250,12 @@ EOF @test "a quiet machine is ready, and says so in both exit code and field" { run "$FACTORY" doctor --json # No org missing, gh answering, state dir writable: nothing blocks, and the - # two notes are the budget feed and the empty afterMerge list. Counted rather - # than described: without the trill stub `setup` installs there would be a - # third, and a comment is not what keeps that stub load-bearing. + # three notes are the budget feed, the empty afterMerge list and the absent + # fixer.command. Counted rather than described: without the trill stub + # `setup` installs there would be a fourth, and a comment is not what keeps + # that stub load-bearing. [ "$status" -eq 1 ] - [ "$(jq -r .notes <<<"$output")" = 2 ] + [ "$(jq -r .notes <<<"$output")" = 3 ] [ "$(jq -r .ready <<<"$output")" = true ] [ "$(jq -r .blocking <<<"$output")" = 0 ] [ "$(jq -r .exit <<<"$output")" = 1 ] @@ -225,7 +268,7 @@ EOF # breaking change and belongs here rather than in a caller's surprise. local ids ids=$(jq -r '[.checks[].check] | join(" ")' <<<"$output") - [[ "$ids" == "jq gh config-file scope digest budget after-merge notify state-dir" ]] + [[ "$ids" == "jq gh config-file scope digest budget after-merge notify fixer state-dir" ]] [ "$(jq -r '[.checks[] | select(.section == "")] | length' <<<"$output")" = 0 ] [ "$(jq -r '[.checks[] | select(.state | test("^(ok|warn|bad)$") | not)] | length' <<<"$output")" = 0 ] } @@ -253,7 +296,7 @@ EOF [ "$(jq -r .lease.live <<<"$output")" = false ] [ "$(jq -r .lease.line <<<"$output")" = "lease: none" ] [ "$(jq -r .watchdog.state <<<"$output")" = no-lease ] - [ "$(jq -e 'has("pollerPid")' <<<"$(jq -c .watchdog <<<"$output")")" = true ] + [ "$(jq -e 'has("runnerPid")' <<<"$(jq -c .watchdog <<<"$output")")" = true ] } # Silence is a claim: a doctor that could not read the lease must not emit the @@ -361,9 +404,23 @@ EOF run "$FACTORY" skill [ "$status" -eq 0 ] [[ "$output" == *"name: factory"* ]] - run "$FACTORY" skill nightshift + run "$FACTORY" skill second [ "$status" -eq 0 ] - [[ "$output" == *"name: nightshift"* ]] + [[ "$output" == *"name: second"* ]] +} + +# The loop skill is gone: its four rules are `factory watchdog run`'s gates +# now. An agent that still asks for it by name gets the refusal a skill that +# never existed gets, not a stale copy. +@test "the nightshift skill no longer ships, and asking for it is a refusal" { + run --separate-stderr "$FACTORY" skill nightshift + [ "$status" -eq 2 ] + [ -z "$output" ] + [[ "$stderr" == *"no such skill 'nightshift'"* ]] + [ ! -e "$BATS_TEST_DIRNAME/../ai/nightshift" ] + # And neither the tool's own skill nor the docs send anyone to it. + ! grep -q nightshift "$BATS_TEST_DIRNAME/../ai/SKILL.md" + ! grep -qi "foreman" "$BATS_TEST_DIRNAME/../ai/SKILL.md" } @test "a skill this tool does not ship is a refusal on fd 2, not an empty document" { @@ -426,7 +483,7 @@ EOF run "$FACTORY" skill install --dir "$TMP/scratch" [ "$status" -eq 0 ] [ -f "$TMP/scratch/factory/SKILL.md" ] - [ -f "$TMP/scratch/nightshift/SKILL.md" ] + [ -f "$TMP/scratch/second/SKILL.md" ] [[ "$output" == *"2 written, 0 left alone"* ]] } @@ -457,7 +514,7 @@ EOF run "$FACTORY" skill install --client claude [ "$status" -eq 0 ] [[ "$output" == *"haus.ai.skill already did"* ]] - [ -f "$HOME/.claude/skills/nightshift/SKILL.md" ] + [ -f "$HOME/.claude/skills/second/SKILL.md" ] [ -L "$HOME/.claude/skills/factory" ] } @@ -466,9 +523,9 @@ EOF # non-zero here would have an agent report a broken command and retry with # more force, against a directory where force corrupts a generation. @test "a run that finds only symlinks says so, and does not read as a failure" { - mkdir -p "$TMP/scratch" "$TMP/store/factory" "$TMP/store/nightshift" + mkdir -p "$TMP/scratch" "$TMP/store/factory" "$TMP/store/second" ln -s "$TMP/store/factory" "$TMP/scratch/factory" - ln -s "$TMP/store/nightshift" "$TMP/scratch/nightshift" + ln -s "$TMP/store/second" "$TMP/scratch/second" run "$FACTORY" skill install --dir "$TMP/scratch" [ "$status" -eq 0 ] [[ "$output" == *"nothing to install"* ]] @@ -549,5 +606,5 @@ EOF [ "$status" -eq 3 ] [[ "$output" == *"cannot write it"* ]] [ -f "$HOME/.codex/skills/factory/SKILL.md" ] - [ -f "$HOME/.codex/skills/nightshift/SKILL.md" ] + [ -f "$HOME/.codex/skills/second/SKILL.md" ] } diff --git a/test/factory-lease.bats b/test/factory-lease.bats index 975debd..86b5b14 100644 --- a/test/factory-lease.bats +++ b/test/factory-lease.bats @@ -243,6 +243,72 @@ expires_in() { [ "$left" -le 90 ] || fail "90s granted ${left}s" } +# ── indefinitely ────────────────────────────────────────────────────────────── +# A lease with no expiry. Allowed because the bound on what merges was never +# the clock — it is tier 1 — and what the clock bounded, a standing grant +# outliving whoever exercised it, the runner's `shift-dead` now bounds on its +# own. The state file spells it `never`, a word rather than a sentinel epoch, +# so a reader that only knows epochs fails closed on it. + +@test "grant indefinitely is accepted, and status says so" { + run "$LEASECMD" grant indefinitely + [ "$status" -eq 0 ] + [[ "$output" == *"lease: tier 1 indefinitely"* ]] + [[ "$output" == *"until revoked"* ]] + [ "$(cut -f1 "$FACTORY_STATE_DIR/lease")" = never ] + run "$LEASECMD" status + [ "$status" -eq 0 ] + [[ "$output" == *"lease: tier 1 · indefinite · until revoked"* ]] +} + +@test "status --json on an indefinite lease carries nulls, not a century" { + # A caller drawing a countdown off `secondsLeft` should draw "until revoked" + # and never a number of years — so both the expiry and the countdown are + # null, and `indefinite` is the field to branch on. + "$LEASECMD" grant indefinitely >/dev/null + run "$LEASECMD" status --json + [ "$status" -eq 0 ] + [ "$(jq -r .live <<<"$output")" = true ] + [ "$(jq -r .indefinite <<<"$output")" = true ] + [ "$(jq -r .expires <<<"$output")" = null ] + [ "$(jq -r .secondsLeft <<<"$output")" = null ] + [ "$(jq -r .granted <<<"$output")" -gt 0 ] +} + +@test "a timed lease says indefinite: false, so the field is always there to branch on" { + lease_expiring 3600 + run "$LEASECMD" status --json + [ "$(jq -r .indefinite <<<"$output")" = false ] + [ "$(jq -r .secondsLeft <<<"$output")" -gt 0 ] +} + +@test "revoke ends an indefinite lease the way it ends a timed one" { + "$LEASECMD" grant indefinitely >/dev/null + run "$LEASECMD" revoke + [ "$status" -eq 0 ] + [ ! -f "$FACTORY_STATE_DIR/lease" ] + run "$LEASECMD" status + [ "$status" -eq 1 ] +} + +@test "only the one word is the indefinite grant — a near miss is a bad duration, not a lease" { + for w in indefinite forever never always; do + run "$LEASECMD" grant "$w" + [ "$status" -eq 2 ] || fail "grant $w exited $status, not 2: $output" + [[ "$output" == *"bad duration"* ]] || fail "grant $w: $output" + [ ! -s "$FACTORY_STATE_DIR/lease" ] || fail "grant $w wrote a lease" + done +} + +@test "a state file spelling anything but an epoch or never is unreadable, and unreadable is not live" { + # The fail-closed half of choosing a word: a reader that does not know it + # reports the lease unreadable, which is exit 1 and "may not merge". + printf 'indefinitely\t1\t%s\n' "$(date +%s)" >"$FACTORY_STATE_DIR/lease" + run "$LEASECMD" status + [ "$status" -eq 1 ] + [[ "$output" == *"lease: unreadable"* ]] +} + # ── tier ────────────────────────────────────────────────────────────────────── @test "a grant naming a tier other than 1 is refused" { @@ -250,6 +316,9 @@ expires_in() { [ "$status" -eq 2 ] [[ "$output" == *"only tier 1 exists"* ]] [ ! -s "$FACTORY_STATE_DIR/lease" ] + run "$LEASECMD" grant indefinitely 2 + [ "$status" -eq 2 ] + [ ! -s "$FACTORY_STATE_DIR/lease" ] } # ── status ──────────────────────────────────────────────────────────────────── diff --git a/test/factory-shift.bats b/test/factory-shift.bats index f7bda88..9f91fa0 100644 --- a/test/factory-shift.bats +++ b/test/factory-shift.bats @@ -15,6 +15,8 @@ # running a test suite is never a reason to take it. The stub also records its # calls, which makes the notify POLICY testable — see the pair of cases on it. +bats_require_minimum_version 1.5.0 # `run --separate-stderr`, for the event cases + setup() { TMP="$BATS_TEST_TMPDIR" # A throwaway tree. `factory-shift` resolves its siblings by path @@ -123,8 +125,8 @@ case "\$1 \$2" in "run list") case "$2" in fail) echo "http2: client conn could not be established" >&2; exit 1 ;; - red) printf 'failure\thttps://example.invalid/run/1\n' ;; - *) printf 'success\thttps://example.invalid/run/1\n' ;; + red) printf 'failure\thttps://example.invalid/run/1\t9c2e1f0\n' ;; + *) printf 'success\thttps://example.invalid/run/1\t9c2e1f0\n' ;; esac ;; esac @@ -171,6 +173,22 @@ stub_tier() { [[ "$output" == *"CI-RED: hausfold/perch"* ]] } +# The runner's fixer gates read the `ci-red` EVENT: `head` is what "never a +# second lane for the same failure" is counted on, and `branch` is what the +# lane is handed. Both ride in the event and neither is in the human line, +# which stays the one unbreakable URL it always was. +@test "a ci-red event carries the run's head SHA and the branch, for the runner's gates" { + stub_gh ok red none + run --separate-stderr "$SHIFT" --dry-run --json + [ "$status" -eq 0 ] + local ev + ev=$(jq -c 'select(.event == "ci-red")' <<<"$output") + [ "$(jq -r .repo <<<"$ev")" = hausfold/perch ] + [ "$(jq -r .head <<<"$ev")" = 9c2e1f0 ] + [ "$(jq -r .branch <<<"$ev")" = main ] + [ "$(jq -r .url <<<"$ev")" = https://example.invalid/run/1 ] +} + @test "a PR that WAS judged and refused is still queued, with its reason" { stub_gh ok green one stub_tier 3 @@ -400,6 +418,31 @@ EOF [[ "$output" == *"fixer: yes"* ]] } +# The verdict's reason is a FIELD of the budget event, so the runner's +# `fixer-skipped: … budget — ` quotes it without parsing the human line. +# Every `fixer: false` arm carries one. +@test "a budget refusal carries its reason in the event, and an affirmative carries none" { + stub_usage 90 16 $((604800 * 84 / 100)) + run --separate-stderr "$SHIFT" --dry-run --json + local ev + ev=$(jq -c 'select(.event == "budget")' <<<"$output") + [ "$(jq -r .fixer <<<"$ev")" = false ] + [ "$(jq -r .reason <<<"$ev")" = "5h window at 90%" ] + + stub_usage 10 16 $((604800 * 84 / 100)) + run --separate-stderr "$SHIFT" --dry-run --json + ev=$(jq -c 'select(.event == "budget")' <<<"$output") + [ "$(jq -r .fixer <<<"$ev")" = true ] + [ "$(jq -r .reason <<<"$ev")" = null ] + + # The unknown arms too: an unreadable quota names itself. + rm -f "$TMP/usage.tsv" + run --separate-stderr "$SHIFT" --dry-run --json + ev=$(jq -c 'select(.event == "budget")' <<<"$output") + [ "$(jq -r .fixer <<<"$ev")" = false ] + [[ "$(jq -r .reason <<<"$ev")" == "no usage feed at "* ]] +} + @test "a saturated 5-hour window refuses however much of the week is left" { # Not the same question as the week, and it outranks it: a factory that # saturates the rolling window at 4am rate-limits whoever sits down at 9. @@ -574,8 +617,9 @@ EOF run "$SHIFT" --dry-run [ "$status" -eq 0 ] [[ "$output" == *"tier-unknown: hausfold/perch#7"* ]] - # The expensive collapse: the nightshift skill tells the foreman that queued - # rows need nothing, so an unjudged PR filed as queued is one nobody revisits. + # The expensive collapse: `queued` rows need nothing by design — the skill + # says so — so an unjudged PR filed as queued is one nobody revisits, and one + # the runner never retries either, since only the unknown lines earn a retry. [[ "$output" != *"queued: hausfold/perch#7"* ]] } diff --git a/test/factory-watchdog.bats b/test/factory-watchdog.bats index 328bd05..141158c 100644 --- a/test/factory-watchdog.bats +++ b/test/factory-watchdog.bats @@ -1,19 +1,26 @@ #!/usr/bin/env bats -# Unit tests for `libexec/factory-watchdog` — the layer that notices the FOREMAN -# stopped, which `factory-shift`'s own unknown-lines cannot, because writing one -# requires a pass that ran. +# Unit tests for `libexec/factory-watchdog` — the RUNNER: the process a live +# lease starts, which passes `factory shift` on a cadence, puts every CI-RED +# through the four fixer gates, and notices when its own passes stop landing. # -# The shape being pinned is the one the README's *When the foreman dies* -# records: a session whose loop ended in an API error, so no next wakeup was -# ever scheduled. Every case here is a variation on "the log went quiet -# while the lease stayed live", because that pair is the entire signal. +# `factory-shift` is STUBBED, with a scripted event stream, because the contract +# between the two verbs is the `--json` events and nothing else: the shift +# suite tests the real shift, and this one tests what the runner does with what +# a shift said. The stub appends a `pass done` line to the day's log the way the +# real one does, so the heartbeat moves, and it records every call. +# +# The shape half the cases pin is the one the README's *The runner* records: a +# runner that is alive is not the same as passes that are landing, so "the log +# went quiet while the lease stayed live" is still the whole liveness signal — +# and under a runner, the way that happens is a shift that exits before its +# first line, which the `die` stub is. # # Three cases below are REGRESSION tests for bugs this script shipped with in # review, and each one made the revoke unreachable rather than merely wrong — # they are marked ⚠ and are the reason the suite exists at all: -# • the watchdog's own log line resetting the mtime it reads, +# • the runner's own log line resetting the mtime it reads, # • yesterday's log outranking today's grant stamp, -# • a lease revoked out from under a foreman because the MACHINE slept. +# • a lease revoked out from under a living shift because the MACHINE slept. # # `trill` is stubbed and PATH is PREPENDED, so `notify`'s `command -v` finds the # stub rather than the real binary on a developer's Mac. Several cases reach a @@ -22,9 +29,9 @@ # stall, re-armed by a recovery — testable rather than merely unobtrusive. # # The lease file is usually written directly rather than through `factory-lease -# grant`, because `grant` now spawns a real poller and a leaked one would -# outlive the test that spawned it. The two cases that DO call `grant` are the -# ones whose subject is that spawn, and they stop it themselves. +# grant`, because `grant` now spawns a real runner and a leaked one would +# outlive the test that spawned it. The cases that DO call `grant` are the ones +# whose subject is that spawn, and they stop it themselves. bats_require_minimum_version 1.5.0 # `run --separate-stderr`, for the flag refusal @@ -40,6 +47,7 @@ setup() { cp "$BATS_TEST_DIRNAME/../VERSION" "$TMP/root/" WD="$TMP/root/libexec/factory-watchdog" LEASECMD="$TMP/root/libexec/factory-lease" + SHIFT="$TMP/root/libexec/factory-shift" export FACTORY_STATE_DIR="$TMP/state" # A config the suite owns, so nothing here reads the machine's own policy. @@ -49,6 +57,9 @@ setup() { printf '{"scope":{"orgs":["hausfold"]}}\n' >"$TMP/config.json" mkdir -p "$FACTORY_STATE_DIR" export FACTORY_WATCHDOG_INTERVAL=1 + # The cadence too, and shorter than STALE in every case that shortens + # STALE: the runner refuses a stall threshold inside its own cadence. + export FACTORY_RUNNER_INTERVAL=1 export FACTORY_NO_WATCHDOG=1 PATH="$TMP/bin:$PATH" @@ -59,16 +70,19 @@ setup() { printf '%s\n' "$*" >>"$TRILL_CALLS" EOF chmod +x "$TMP/bin/trill" + export SHIFT_CALLS="$TMP/shift-calls" + export SPAWN_CALLS="$TMP/spawn-calls" + stub_shift ok FAKE_PID="" RACERS="" } teardown() { [ -z "$FAKE_PID" ] || kill "$FAKE_PID" 2>/dev/null || true - # The race case starts pollers that are deliberately NOT the pidfile's — two + # The race case starts runners that are deliberately NOT the pidfile's — two # of the three are meant to lose it — so the pidfile alone cannot reap them. - # A regression there means a poller that outlives its test and goes on - # polling the runner underneath every case after it. + # A regression there means a runner that outlives its test and goes on + # passing underneath every case after it. local p for p in $RACERS; do case "$(ps -p "$p" -o command= 2>/dev/null)" in @@ -86,6 +100,74 @@ teardown() { fi } +# The scripted shift. Every mode but `die` appends a `pass done` line to the +# day's log, which is the heartbeat, and prints the events the runner reads: +# ok a quiet pass, budget refused (no feed), nothing red +# red a red default branch, budget refused — the budget gate's case +# red-yes a red default branch, budget says yes — the affirmative +# unknown one ci-unknown — the retry's case +# abort `pass ABORTED`, exit 1 +# after after-merge-failed — the third retry-worthy event +# die exit 2 before writing anything, like a config gone invalid +# garbage exit 0 with stdout that is not the event stream +# Written to a private name and `mv`ed into place, because one case swaps the +# stub while the runner is calling it once a second: a `cat >` truncates first, +# and bash reading a script mid-rewrite is a stub that did neither shape. +stub_shift() { # stub_shift [head sha] + # `-` and not `:-`: an EMPTY head is one case's whole subject, and `:-` + # would read it as unset and hand that case a SHA. + local head="${2-deadbeef}" + cat >"$SHIFT.tmp" <>"\$SHIFT_CALLS" +mode="$1" +if [ "\$mode" = die ]; then + echo "factory: tier1.allow is empty — no PR could ever be tier 1 (in /x/config.json)" >&2 + exit 2 +fi +log="\$FACTORY_STATE_DIR/shift-\$(date +%Y%m%d).log" +echo "\$(date '+%H:%M') policy: 00000000 · factory 0.0.0" >>"\$log" +echo '{"event":"policy","digest":"00000000","version":"0.0.0"}' +if [ "\$mode" = garbage ]; then echo "this is not an event"; fi +case "\$mode" in +red-yes) echo '{"event":"budget","mode":"unmetered","fixer":true}' ;; +*) echo '{"event":"budget","mode":"metered","fixer":false,"reason":"5h window at 90%"}' ;; +esac +case "\$mode" in +red | red-yes) + echo "\$(date '+%H:%M') CI-RED: hausfold/perch https://example.invalid/run/1" >>"\$log" + echo '{"event":"ci-red","repo":"hausfold/perch","url":"https://example.invalid/run/1","conclusion":"failure","head":"$head","branch":"main"}' + ;; +unknown) + echo '{"event":"ci-unknown","repo":"hausfold/perch","stderr":"connection reset by peer"}' + ;; +after) + echo '{"event":"after-merge-failed","command":"./bench ship","stderr":"edge did not move"}' + ;; +abort) + echo "\$(date '+%H:%M') pass ABORTED: hausfold listed zero repos" >>"\$log" + echo '{"event":"aborted","reason":"hausfold listed zero repos"}' + exit 1 + ;; +esac +echo "\$(date '+%H:%M') pass done: 0 merged" >>"\$log" +echo '{"event":"pass-done","merged":0}' +EOF + chmod +x "$SHIFT.tmp" + mv "$SHIFT.tmp" "$SHIFT" +} + +# A lane spawner that records its argv, and either returns at once or fails. +stub_spawner() { # stub_spawner ok|fail + cat >"$TMP/bin/spawner" <>"\$SPAWN_CALLS" +case "$1" in fail) echo "scruff: no such repo checkout" >&2; exit 1 ;; esac +EOF + chmod +x "$TMP/bin/spawner" + printf '{"scope":{"orgs":["hausfold"]},"fixer":{"command":["spawner"]}}\n' >"$TMP/config.json" +} + # A lease expiring $1 seconds from now, granted $2 seconds ago. lease() { printf '%s\t1\t%s\n' "$(($(date +%s) + $1))" "$(($(date +%s) - $2))" \ @@ -101,11 +183,13 @@ log_aged() { touch -t "$(stamp "$(($(date +%s) - $1))")" "$f" } +today_log() { printf '%s\n' "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log"; } + # A process that answers to the name the pidfile claims, without being a real -# poller. `is_watchdog` matches a command ENDING in `factory-watchdog run`, so +# runner. `is_watchdog` matches a command ENDING in `factory-watchdog run`, so # the stub has to be a script of that name invoked with that verb — an # `exec -a` rename cannot produce it, since the sleep duration would follow. -fake_poller() { +fake_runner() { cat >"$TMP/bin/factory-watchdog" <<'EOF' #!/usr/bin/env bash sleep 30 @@ -116,31 +200,42 @@ EOF printf '%s\n' "$FAKE_PID" >"$FACTORY_STATE_DIR/watchdog.pid" } -# Wait up to 25s for a predicate, so nothing here races a 1s poll interval — -# nor the deliberately over-long `sleep`s the suspend cases install. -until_ok() { - local i=0 - while [ $i -lt 250 ]; do +# Wait up to 25s for a predicate, so nothing here races a 1s tick — nor the +# deliberately over-long `sleep`s the suspend cases install. `until_ok_long` +# is a minute, for the one case whose subject is a string of naps. +until_ok() { until_n 250 "$@"; } +until_ok_long() { until_n 600 "$@"; } +until_n() { + local n="$1" i=0 + shift + while [ $i -lt "$n" ]; do if "$@"; then return 0; fi sleep 0.1; i=$((i + 1)) done return 1 } -# How many of the NAMED pids are live pollers — never a `pgrep -f +# The number of times the stubbed shift has been called, as a predicate for +# `until_ok`: `$(wc -l …)` written into until_ok's own arguments would be +# expanded once, before the first attempt. +shift_calls_reach() { lines_reach "$SHIFT_CALLS" "$1"; } +lines_reach() { [ -s "$1" ] && [ "$(wc -l <"$1" | tr -d ' ')" -ge "$2" ]; } +shift_calls() { if [ -s "$SHIFT_CALLS" ]; then wc -l <"$SHIFT_CALLS" | tr -d ' '; else echo 0; fi; } + +# How many of the NAMED pids are live runners — never a `pgrep -f # "factory-watchdog run"` over the process table, which is a different and -# wrong question. A poller forks a subshell for every command substitution in +# wrong question. A runner forks a subshell for every command substitution in # its loop, and a forked child inherits its parent's argv verbatim: `quiet=$( # check "$slept")` alone runs a whole `factory-lease` and a `jq` while a second # process with a byte-identical command line sits in the table. Any pattern -# match sampling that instant counts the one correct poller twice. That phantom +# match sampling that instant counts the one correct runner twice. That phantom # is what made the race case below fail on a loaded runner while the claim it # tests was doing exactly the right thing — and it fired more often the busier -# the machine, because the fork lives for as long as the poll takes. +# the machine, because the fork lives for as long as the tick takes. # # Asking `ps` about a pid we started answers the question the case actually -# has: of the processes THIS test spawned, how many are still pollers. -pollers_alive() { +# has: of the processes THIS test spawned, how many are still runners. +runners_alive() { local pid n=0 for pid in "$@"; do case "$(ps -p "$pid" -o command= 2>/dev/null)" in @@ -150,12 +245,12 @@ pollers_alive() { printf '%s\n' "$n" } -# The predicate form, because `until_ok` re-runs its argv: a `$(pollers_alive +# The predicate form, because `until_ok` re-runs its argv: a `$(runners_alive # ...)` written into until_ok's own arguments would be expanded once, before # the first attempt, and every retry would re-test the first answer. -poller_count_is() { +runner_count_is() { local want="$1"; shift - [ "$(pollers_alive "$@")" -eq "$want" ] + [ "$(runners_alive "$@")" -eq "$want" ] } # ── nothing to watch ────────────────────────────────────────────────────────── @@ -174,26 +269,326 @@ poller_count_is() { [[ "$output" == *"no live lease"* ]] } +# ── the runner: a live lease passes on its own ─────────────────────────────── +# The affirmative first, for the reason the shift suite leads with `fixer: +# yes`: until a pass can land with no agent involved, none of the refusals and +# gates below is distinguishable from a path that never runs. + +@test "a live lease and the runner produce pass done lines on the cadence, with no agent involved" { + lease 3600 5 + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok shift_calls_reach 3 + kill $pid 2>/dev/null || true + # Three passes, each of them `factory shift --json` and nothing else — the + # runner reads events, never the human line. + [ "$(shift_calls)" -ge 3 ] + [ "$(sort -u "$SHIFT_CALLS")" = "--json" ] + [ "$(grep -c "pass done" "$(today_log)")" -ge 3 ] +} + +@test "the first pass lands within seconds of the grant, not one cadence later" { + # `grant` starts the runner; a user who typed it should see a pass now, and + # a runner launchd restarted after a crash has a gap to close. + unset FACTORY_NO_WATCHDOG + FACTORY_RUNNER_INTERVAL=600 "$LEASECMD" grant 30m >/dev/null + until_ok shift_calls_reach 1 + [ "$(shift_calls)" -ge 1 ] + "$LEASECMD" revoke >/dev/null +} + +@test "a pass runs only under a live lease, and the runner leaves when the lease ends" { + # The ordinary end of a timed shift: the log's last line says the shift is + # over, not merely that a pass happened to be its last. + lease 3 5 + run "$WD" run + [ "$status" -eq 0 ] + grep -q "pass done" "$(today_log)" + grep -q "shift-over: lease ended" "$(today_log)" + # And every call the stub saw was made while the lease stood: the runner + # asks the lease at each pass, not once at start. + [ "$(shift_calls)" -ge 1 ] + [ "$(shift_calls)" -le 4 ] +} + +@test "a runner that never saw a lease adds nothing to the log" { + # Started after the expiry — launchd restarting it, or `ensure` racing a + # revoke. Nothing passed, so nothing is over. + lease 1 3600 + log_aged 60 + sleep 2 + run "$WD" run + [ "$status" -eq 0 ] + [ -z "$output" ] + ! grep -q "shift-over" "$(today_log)" + [ "$(shift_calls)" -eq 0 ] +} + +@test "a runner with no shift to run refuses to be one" { + # A live lease under a process that could pass nothing is exactly the + # standing grant the runner exists to refuse — and it would otherwise sit + # there reporting itself alive. + lease 3600 5 + chmod -x "$SHIFT" + run --separate-stderr "$WD" run + [ "$status" -eq 2 ] + [[ "$stderr" == *"cannot run"*"factory-shift"* ]] + [ ! -e "$FACTORY_STATE_DIR/watchdog.pid" ] +} + +# ── the retry: once, at the next tick ──────────────────────────────────────── +# A pass that could not see gets one more pass five minutes later, not twenty. +# Once, because a second unknown is a story for the morning and not a loop; and +# at the next TICK rather than at once, because the blip that made a `gh` call +# fail is usually still there a second later. + +@test "an unknown line earns one more pass at the next tick, and only one" { + lease 3600 5 + stub_shift unknown + # A long cadence, so the second call can only be the retry. + FACTORY_RUNNER_INTERVAL=60 "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok shift_calls_reach 2 + # Three more ticks: a third call would be a retry of the retry. + sleep 3 + kill $pid 2>/dev/null || true + [ "$(shift_calls)" -eq 2 ] + grep -q "pass-retry: ci-unknown" "$(today_log)" + [ "$(grep -c "pass-retry" "$(today_log)")" -eq 1 ] +} + +@test "an aborted pass and a failed after-merge hook are retried the same way" { + lease 3600 5 + stub_shift abort + FACTORY_RUNNER_INTERVAL=60 "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok shift_calls_reach 2 + sleep 3 + kill $pid 2>/dev/null || true + [ "$(shift_calls)" -eq 2 ] + grep -q "pass-retry: aborted" "$(today_log)" + + rm -f "$SHIFT_CALLS" + rm -f "$(today_log)" + stub_shift after + FACTORY_RUNNER_INTERVAL=60 "$WD" run >/dev/null 2>&1 & + pid=$! + until_ok shift_calls_reach 2 + sleep 3 + kill $pid 2>/dev/null || true + [ "$(shift_calls)" -eq 2 ] + grep -q "pass-retry: after-merge-failed" "$(today_log)" +} + +@test "a quiet pass is not retried" { + lease 3600 5 + FACTORY_RUNNER_INTERVAL=60 "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok shift_calls_reach 1 + sleep 3 + kill $pid 2>/dev/null || true + [ "$(shift_calls)" -eq 1 ] + ! grep -q "pass-retry" "$(today_log)" +} + +# ── the four fixer gates ────────────────────────────────────────────────────── +# Each case below is written so that deleting its gate fails it. These were a +# skill's prose once — a threshold no test could reach, whose refusal was the +# same word as a correct refusal. + +@test "a CI-RED clears all four gates and spawns through fixer.command with repo, branch and url" { + lease 3600 5 + stub_spawner ok + stub_shift red-yes 9c2e1f0 + "$WD" run >/dev/null 2>&1 & + local pid=$! + # Waited on the LOG line, which lands after the spawner returns: waiting on + # the spawner's own record races the `note` that follows it. + until_ok grep -qs "fixer-spawned" "$(today_log)" + kill $pid 2>/dev/null || true + # Three words, in this order: the lane is handed facts, not a JSON document. + [ "$(head -1 "$SPAWN_CALLS")" = "hausfold/perch main https://example.invalid/run/1" ] + grep -q "fixer-spawned: hausfold/perch 9c2e1f0" "$(today_log)" +} + +@test "gate 1 — no fixer.command configured: the red is reported and the skip says why" { + lease 3600 5 + stub_shift red-yes + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -q "fixer-skipped" "$(today_log)" + kill $pid 2>/dev/null || true + grep -q "CI-RED: hausfold/perch" "$(today_log)" + grep -q "fixer-skipped: hausfold/perch — no fixer.command configured" "$(today_log)" + ! grep -q "fixer-spawned" "$(today_log)" + [ ! -e "$SPAWN_CALLS" ] +} + +@test "gate 2 — the same head SHA never gets a second lane, however many passes report it" { + lease 3600 5 + stub_spawner ok + stub_shift red-yes 9c2e1f0 + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok shift_calls_reach 3 + kill $pid 2>/dev/null || true + [ "$(wc -l <"$SPAWN_CALLS" | tr -d ' ')" -eq 1 ] + [ "$(grep -c "fixer-spawned: hausfold/perch 9c2e1f0" "$(today_log)")" -eq 1 ] + grep -q "fixer-skipped: hausfold/perch — a lane was already spawned for 9c2e1f0" "$(today_log)" +} + +@test "gate 2 reads every shift log, not tonight's — a red that stood across midnight does not get a lane a day" { + lease 3600 5 + stub_spawner ok + stub_shift red-yes 9c2e1f0 + local old="$FACTORY_STATE_DIR/shift-20260828.log" + echo "23:50 fixer-spawned: hausfold/perch 9c2e1f0 — lane on main for https://example.invalid/run/1" >"$old" + touch -t "$(stamp "$(($(date +%s) - 86400))")" "$old" + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -q "fixer-skipped" "$(today_log)" + kill $pid 2>/dev/null || true + [ ! -e "$SPAWN_CALLS" ] + grep -q "already spawned for 9c2e1f0" "$(today_log)" +} + +@test "gate 3 — at most fixer.cap lanes per repo per day, and a new SHA past the cap says so" { + lease 3600 5 + stub_spawner ok + stub_shift red-yes cccccc3 + # Two lanes already today, for two earlier failures. + printf '01:00 fixer-spawned: hausfold/perch aaaaaa1 — lane on main for x\n02:00 fixer-spawned: hausfold/perch bbbbbb2 — lane on main for y\n' >"$(today_log)" + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -q "fixer-skipped" "$(today_log)" + kill $pid 2>/dev/null || true + [ ! -e "$SPAWN_CALLS" ] + grep -q "fixer-skipped: hausfold/perch — 2 lane(s) already today, fixer.cap is 2" "$(today_log)" +} + +@test "gate 3 counts the repo, not the night — another repo's lanes are not this one's" { + lease 3600 5 + stub_spawner ok + stub_shift red-yes cccccc3 + printf '01:00 fixer-spawned: hausfold/pounce aaaaaa1 — lane on main for x\n02:00 fixer-spawned: hausfold/pounce bbbbbb2 — lane on main for y\n' >"$(today_log)" + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -qs "fixer-spawned: hausfold/perch" "$(today_log)" + kill $pid 2>/dev/null || true + grep -q "fixer-spawned: hausfold/perch cccccc3" "$(today_log)" +} + +@test "gate 3 is the policy's number — fixer.cap 1 stops at one" { + lease 3600 5 + stub_spawner ok + printf '{"scope":{"orgs":["hausfold"]},"fixer":{"command":["spawner"],"cap":1}}\n' >"$TMP/config.json" + stub_shift red-yes cccccc3 + printf '01:00 fixer-spawned: hausfold/perch aaaaaa1 — lane on main for x\n' >"$(today_log)" + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -q "fixer-skipped" "$(today_log)" + kill $pid 2>/dev/null || true + [ ! -e "$SPAWN_CALLS" ] + grep -q "1 lane(s) already today, fixer.cap is 1" "$(today_log)" +} + +@test "gate 4 — a budget that said no is quoted, and no lane is spawned" { + lease 3600 5 + stub_spawner ok + stub_shift red + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -q "fixer-skipped" "$(today_log)" + kill $pid 2>/dev/null || true + [ ! -e "$SPAWN_CALLS" ] + # The reason comes off the budget EVENT, not the human line. + grep -q "fixer-skipped: hausfold/perch — budget: 5h window at 90%" "$(today_log)" +} + +@test "a spawner that fails is fixer-failed with its stderr, carded, and counted toward nothing" { + lease 3600 5 + stub_spawner fail + stub_shift red-yes 9c2e1f0 + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok lines_reach "$SPAWN_CALLS" 2 + kill $pid 2>/dev/null || true + grep -q "fixer-failed: hausfold/perch 9c2e1f0 — fixer.command exited 1: scruff: no such repo checkout" "$(today_log)" + ! grep -q "fixer-spawned" "$(today_log)" + grep -q "fixer lane for hausfold/perch did not start" "$TRILL_CALLS" + # Nothing was spawned, so the next pass tries again rather than reading the + # failure as a lane already there. + [ "$(wc -l <"$SPAWN_CALLS" | tr -d ' ')" -ge 2 ] +} + +@test "a red whose head SHA was not reported is skipped, not spawned every pass" { + lease 3600 5 + stub_spawner ok + stub_shift red-yes "" + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -q "fixer-skipped" "$(today_log)" + kill $pid 2>/dev/null || true + [ ! -e "$SPAWN_CALLS" ] + grep -q "head SHA was not reported" "$(today_log)" +} + +# ── a pass that could not run at all ───────────────────────────────────────── + +@test "a shift that exits before writing anything is pass-failed with its stderr" { + lease 3600 5 + stub_shift die + FACTORY_RUNNER_INTERVAL=60 "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -q "pass-failed" "$(today_log)" + sleep 2 + kill $pid 2>/dev/null || true + grep -q "pass-failed: factory shift exited 2 before a pass could run — factory: tier1.allow is empty" "$(today_log)" + # Not retried: a config that cannot be read at 02:00 cannot be read at 02:05. + ! grep -q "pass-retry" "$(today_log)" + [ "$(shift_calls)" -eq 1 ] +} + +@test "a shift whose --json output is not the event stream is pass-failed, not a quiet zero of every count" { + lease 3600 5 + stub_shift garbage + "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -q "pass-failed" "$(today_log)" + kill $pid 2>/dev/null || true + grep -q "pass-failed: factory shift's --json output could not be read" "$(today_log)" +} + # ── the heartbeat ───────────────────────────────────────────────────────────── -@test "live lease, recent pass, poller watching: healthy" { +@test "live lease, recent pass, runner up: alive" { lease 3600 3600 log_aged 300 - fake_poller + fake_runner run "$WD" once [ "$status" -eq 0 ] [[ "$output" == *"shift alive"* ]] [[ "$output" == *"5m ago"* ]] } -@test "live lease and a recent pass but NO poller is its own fault, not healthy" { - # `grant` spawns the poller with all output discarded, so a lost exec bit is - # otherwise silent — and the SKILL tells the foreman to confirm at start. +@test "live lease and a recent pass but NO runner is its own fault, not alive" { + # `grant` spawns the runner with all output discarded, so a lost exec bit is + # otherwise silent — and `doctor` carries this line. lease 3600 3600 log_aged 300 run "$WD" once [ "$status" -eq 4 ] - [[ "$output" == *"NO POLLER"* ]] + [[ "$output" == *"NO RUNNER"* ]] +} + +@test "once --json names the runner's pid under runnerPid" { + lease 3600 3600 + log_aged 300 + fake_runner + run "$WD" once --json + [ "$status" -eq 0 ] + [ "$(jq -r .state <<<"$output")" = alive ] + [ "$(jq -r .runnerPid <<<"$output")" = "$FAKE_PID" ] } @test "live lease, log quiet past the stale threshold: STALLED" { @@ -208,7 +603,7 @@ poller_count_is() { @test "a pass still short of the threshold is not a stall" { lease 21600 3600 log_aged 2400 - fake_poller + fake_runner run "$WD" once [ "$status" -eq 0 ] } @@ -219,50 +614,50 @@ poller_count_is() { echo "23:50 pass done: 0 merged" >"$old" touch -t "$(stamp "$(($(date +%s) - 86400))")" "$old" log_aged 120 - fake_poller + fake_runner run "$WD" once [ "$status" -eq 0 ] } # ── ⚠ regression: yesterday's log must not outrank today's grant ────────────── -@test "⚠ a fresh grant with only an old log is healthy, not instantly dead" { +@test "⚠ a fresh grant with only an old log is alive, not instantly dead" { # Logs are per-day and never swept. Reading the newest log ALONE meant that - # on every night after the first, `grant` spawned a poller that revoked the - # lease before the foreman's first pass could write anything — so the shift - # aborted at step 2 with "the grant did not take". + # on every night after the first, `grant` spawned a runner that revoked the + # lease before its first pass could write anything. local old="$FACTORY_STATE_DIR/shift-20260828.log" echo "23:50 pass done: 0 merged" >"$old" touch -t "$(stamp "$(($(date +%s) - 72000))")" "$old" lease 43200 30 - fake_poller + fake_runner run "$WD" once [ "$status" -eq 0 ] } -@test "no log yet, lease just granted: healthy" { +@test "no log yet, lease just granted: alive" { lease 43200 60 - fake_poller + fake_runner run "$WD" once [ "$status" -eq 0 ] } @test "no log yet, lease granted an hour ago: STALLED" { - # A foreman that died before its first pass is exactly as dead as one that - # died after ten, and leaves no log to say so. + # A runner that died before its first pass leaves no log to say so. lease 43200 3600 run "$WD" once [ "$status" -eq 3 ] [[ "$output" == *"STALLED"* ]] } -# ── ⚠ regression: the watchdog's own lines are not a heartbeat ──────────────── +# ── ⚠ regression: the runner's own lines are not a heartbeat ────────────────── -@test "⚠ a persisting stall reaches DEAD and revokes, despite the watchdog logging" { +@test "⚠ a shift that cannot start reaches DEAD and revokes, despite the runner logging every pass" { # `note` appends to the same file whose mtime IS the heartbeat. Without the - # mtime being restored, writing `foreman-stalled` reset quiet to zero, the - # next poll read healthy and wrote `foreman-resumed`, and the pair alternated - # forever — quiet could never exceed STALE, so this revoke was unreachable. + # mtime being restored, every `pass-failed` line — one per cadence, all + # night — reset quiet to zero, and a lease stood forever under a shift that + # could not run, with a named line every twenty minutes reading as a pass. + # Same bug the foreman-era `foreman-stalled` line had, one layer closer. + stub_shift die FACTORY_STALE=2 FACTORY_DEAD=6 lease 21600 60 log_aged 3 FACTORY_STALE=2 FACTORY_DEAD=6 "$WD" run >/dev/null 2>&1 & @@ -270,30 +665,61 @@ poller_count_is() { until_ok test ! -f "$FACTORY_STATE_DIR/lease" kill $pid 2>/dev/null || true [ ! -f "$FACTORY_STATE_DIR/lease" ] - grep -q "foreman-gone" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" + grep -q "pass-failed" "$(today_log)" + grep -q "shift-dead" "$(today_log)" # The alternation the bug produced: one stall line, and never a resume, since # nothing ever landed a real pass. - [ "$(grep -c "foreman-stalled" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log")" -eq 1 ] - ! grep -q "foreman-resumed" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" + [ "$(grep -c "shift-stalled" "$(today_log)")" -eq 1 ] + ! grep -q "shift-resumed" "$(today_log)" } -# ── what `run` does about it ────────────────────────────────────────────────── +@test "a pass landing after a stall is shift-resumed, and re-arms the card" { + # The stub is switched from `die` to `ok` mid-run: the runner reads the + # script fresh on every pass, so the stall the first shape caused is + # recovered from by the second. + stub_shift die + FACTORY_STALE=2 FACTORY_DEAD=30 lease 21600 60 + log_aged 1 + FACTORY_STALE=2 FACTORY_DEAD=30 FACTORY_RUNNER_INTERVAL=1 "$WD" run >/dev/null 2>&1 & + local pid=$! + until_ok grep -q "shift-stalled" "$(today_log)" + stub_shift ok + until_ok grep -q "shift-resumed" "$(today_log)" + kill $pid 2>/dev/null || true + [ -f "$FACTORY_STATE_DIR/lease" ] + [ "$(wc -l <"$TRILL_CALLS" | tr -d ' ')" -eq 1 ] +} + +# ── what `run` does about a stall ───────────────────────────────────────────── @test "a stall past DEAD revokes the lease and cards it" { + stub_shift die lease 21600 10800 log_aged 7200 run "$WD" run [ "$status" -eq 3 ] [ ! -f "$FACTORY_STATE_DIR/lease" ] - grep -q "foreman-gone" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" - grep -q "lease revoked" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" + grep -q "shift-dead" "$(today_log)" + grep -q "lease revoked (it stood until" "$(today_log)" grep -q "fault" "$TRILL_CALLS" } +@test "an indefinite lease whose passes stopped is revoked too, and the line says it was indefinite" { + # The clock never bounded an indefinite lease; this is what does. + stub_shift die + printf 'never\t1\t%s\n' "$(($(date +%s) - 10800))" >"$FACTORY_STATE_DIR/lease" + log_aged 7200 + run "$WD" run + [ "$status" -eq 3 ] + [ ! -f "$FACTORY_STATE_DIR/lease" ] + grep -q "shift-dead: .*lease revoked (it was indefinite)" "$(today_log)" +} + @test "a stall short of DEAD says so but leaves the lease standing" { - # The distinction the two thresholds exist for: a foreman whose network - # dropped for one turn may still be mid-retry, and revoking under it turns a - # recoverable blip into a shift that needs a person. + # The distinction the two thresholds exist for: one pass hanging inside a + # `gh` call may still return, and revoking under it turns a recoverable blip + # into a shift that needs a person. + stub_shift die lease 21600 3600 log_aged 3600 "$WD" run >/dev/null 2>&1 & @@ -302,12 +728,13 @@ poller_count_is() { # would race the `notify` that follows it and kill the process between them. until_ok test -s "$TRILL_CALLS" kill $pid 2>/dev/null || true - grep -q "foreman-stalled" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" + grep -q "shift-stalled" "$(today_log)" [ -f "$FACTORY_STATE_DIR/lease" ] grep -q "fault" "$TRILL_CALLS" } -@test "the stall card fires once, not once per poll" { +@test "the stall card fires once, not once per tick" { + stub_shift die lease 21600 3600 log_aged 3600 "$WD" run >/dev/null 2>&1 & @@ -318,22 +745,11 @@ poller_count_is() { [ "$(wc -l <"$TRILL_CALLS")" -eq 1 ] } -@test "run exits quietly when the lease ends under it" { - # The ordinary end of a shift: the foreman was alive to the last pass and - # wrote its own handover. The watchdog has nothing to add to it. - lease 1 3600 - log_aged 60 - sleep 2 - run "$WD" run - [ "$status" -eq 0 ] - [ -z "$output" ] -} - -# ── ⚠ regression: a sleeping Mac is not a dead foreman ──────────────────────── +# ── ⚠ regression: a sleeping Mac is not a stalled shift ─────────────────────── # One "suspended machine" per line in $FACTORY_STATE_DIR/.naps, consumed in -# order, keyed on the poller's own interval so the suite's sub-second waits -# never eat one. +# order, keyed on the runner's own tick so the suite's sub-second waits never +# eat one. stub_sleep() { printf '%s\n' "$@" >"$FACTORY_STATE_DIR/.naps" cat >"$TMP/bin/sleep" <<'EOF' @@ -349,44 +765,60 @@ EOF chmod +x "$TMP/bin/sleep" } -@test "⚠ a suspend is discounted, not counted against the foreman" { +@test "⚠ a suspend is discounted, not counted against the shift" { # Quiet time is measured in seconds this process was AWAKE for. A machine - # asleep past DEAD wakes to a stale log through nobody's fault — the poller - # was not running either — and a lease revoked out from under a living - # foreman is the failure this script would be introducing rather than fixing. - stub_sleep 8 + # asleep past DEAD wakes to a stale log through nobody's fault — the runner + # was not running either — and a lease revoked out from under a shift that + # would have passed on waking is the failure this script would be + # introducing rather than fixing. + # A 12s nap against DEAD=10: at the kill, two seconds after the nap is + # noted, the wall clock is past DEAD and awake time is under STALE. The gap + # between those two is what a loaded runner gets to be slow in — under a + # runner passing every tick, each tick costs a few forks more than the + # poller's did, and a 2s margin here failed once on a busy machine. + stub_shift die + stub_sleep 12 lease 21600 60 log_aged 1 - FACTORY_STALE=4 FACTORY_DEAD=6 "$WD" run >/dev/null 2>&1 & + FACTORY_STALE=4 FACTORY_DEAD=10 "$WD" run >/dev/null 2>&1 & local pid=$! - until_ok grep -q "machine-slept" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" + until_ok grep -q "machine-slept" "$(today_log)" sleep 2 kill $pid 2>/dev/null || true # Wall clock is past DEAD; awake time is not, so nothing was revoked. [ -f "$FACTORY_STATE_DIR/lease" ] - ! grep -q "foreman-gone" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" + ! grep -q "shift-dead" "$(today_log)" } -@test "⚠ repeated suspends still let a dead foreman reach DEAD" { +@test "⚠ repeated suspends still let a shift that cannot run reach DEAD" { # The bug the discount replaced: a grace WINDOW re-armed on every jump, so a # laptop that suspends and wakes all night — the documented default, with - # `haus.power.lidAwake` off — renewed it faster than it expired. A genuinely - # dead foreman then kept its lease until morning and never even drew a card. - stub_sleep 5 5 5 5 5 5 + # `haus.power.lidAwake` off — renewed it faster than it expired. A shift that + # genuinely could not run then kept its lease until morning and never even + # drew a card. + # + # The numbers leave room for a loaded machine: the first check comes AFTER a + # pass now, and under a nix shell that pass is a dozen forks — with the log + # a second old and DEAD at 3, one slow first tick revoked before the first + # nap was ever taken, and the case failed on the `machine-slept` line it + # never had a chance to write. Six awake seconds is the budget instead, and + # ten naps of four seconds is more than enough wall clock to get there. + stub_shift die + stub_sleep 4 4 4 4 4 4 4 4 4 4 lease 21600 60 - log_aged 1 - FACTORY_STALE=2 FACTORY_DEAD=3 "$WD" run >/dev/null 2>&1 & + log_aged 0 + FACTORY_STALE=3 FACTORY_DEAD=6 "$WD" run >/dev/null 2>&1 & local pid=$! - until_ok test ! -f "$FACTORY_STATE_DIR/lease" + until_ok_long test ! -f "$FACTORY_STATE_DIR/lease" kill $pid 2>/dev/null || true [ ! -f "$FACTORY_STATE_DIR/lease" ] - grep -q "machine-slept" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" - grep -q "foreman-gone" "$FACTORY_STATE_DIR/shift-$(date +%Y%m%d).log" + grep -q "machine-slept" "$(today_log)" + grep -q "shift-dead" "$(today_log)" } # ── the pidfile, and the wiring `factory-lease` depends on ──────────────────── -@test "a second run is a no-op while one is already polling" { +@test "a second run is a no-op while one is already running" { lease 21600 3600 log_aged 60 "$WD" run >/dev/null 2>&1 & @@ -401,7 +833,7 @@ EOF @test "⚠ a stale pidfile is not a licence to signal whatever now holds that pid" { # The trap only clears the pidfile on a clean exit; a SIGKILL, a panic or a # reboot leaves it, and PIDs restart low after one. `$$` here is bats itself - # — very much a live process, and very much not a watchdog. + # — very much a live process, and very much not a runner. lease 21600 3600 log_aged 60 printf '%s\n' "$$" >"$FACTORY_STATE_DIR/watchdog.pid" @@ -413,26 +845,26 @@ EOF kill -0 $$ } -@test "a stale pidfile does not read as a poller that is watching" { +@test "a stale pidfile does not read as a runner that is up" { lease 21600 3600 log_aged 60 printf '%s\n' "$$" >"$FACTORY_STATE_DIR/watchdog.pid" run "$WD" once [ "$status" -eq 4 ] - [[ "$output" == *"NO POLLER"* ]] + [[ "$output" == *"NO RUNNER"* ]] } -@test "ensure starts a poller when a live lease has lost one" { - # `grant` establishes the invariant once; a poller can still be lost to a - # reboot or an OOM kill, which is the likeliest overnight foreman-killer - # after an API error precisely because it takes both at once. +@test "ensure starts a runner when a live lease has lost one" { + # `grant` establishes the invariant once; a runner can still be lost to a + # reboot or an OOM kill. On a launchd machine that is launchd's job; anywhere + # else, this is the verb `doctor`'s NO RUNNER line points at. lease 21600 60 log_aged 30 run "$WD" once [ "$status" -eq 4 ] run "$WD" ensure [ "$status" -eq 0 ] - [[ "$output" == *"poller watching"* ]] + [[ "$output" == *"runner up"* ]] run "$WD" once [ "$status" -eq 0 ] } @@ -444,9 +876,9 @@ EOF } @test "⚠ a claim it could not write publishes nothing, not an empty pidfile" { - # An empty pidfile is not inert. `poller_pid` reads it as nothing-alive, so + # An empty pidfile is not inert. `runner_pid` reads it as nothing-alive, so # the next `run` deletes it and claims it — and if what it deleted was a live - # poller's claim caught mid-write, that is a duplicate poller started on top + # runner's claim caught mid-write, that is a duplicate runner started on top # of one, which `revoke` cannot stop because it only ever stops the pid the # file names. So the claim writes the pid to a private name and hardlinks it # into place: `watchdog.pid` never exists while empty. @@ -455,11 +887,11 @@ EOF # suite can schedule. A write that CANNOT land makes the same claim testable: # either the pid is published or nothing is. Under an O_EXCL create the file # is created before the write that fails, and the leftover is what the next - # `run` would evict a live poller over. + # `run` would evict a live runner over. # The lease is EXPIRED on purpose. The claim runs before any lease is read, # so nothing here is weakened by it — but a regression that wrongly claims # then finds nothing to watch and exits, where a live lease would leave it - # polling and HANG this case rather than fail it. + # running and HANG this case rather than fail it. lease -600 43200 log_aged 7200 run bash -c 'ulimit -f 0 2>/dev/null || exit 111; exec "$1" run' _ "$WD" @@ -469,7 +901,7 @@ EOF [ ! -e "$FACTORY_STATE_DIR/watchdog.pid" ] } -@test "⚠ two simultaneous grants leave exactly one poller" { +@test "⚠ two simultaneous grants leave exactly one runner" { # The pidfile is claimed by hardlinking a file that already holds the pid, # not by a read-then-write: the loser of a read-then-write became an orphan # `revoke` could not see, still holding a trap that would delete its @@ -479,7 +911,7 @@ EOF # at. The fixed second this used to take assumed the losers had already # exited, which a loaded runner need not honour; the count can only ever # FALL, since nothing here starts a fourth, so "reaches one" and "settles at - # one" are the same statement. Two pollers that stay alive — the bug this + # one" are the same statement. Two runners that stay alive — the bug this # case exists for — still fail it, after `until_ok` has given them 25s. unset FACTORY_NO_WATCHDOG lease 21600 60 @@ -489,19 +921,19 @@ EOF "$WD" run >/dev/null 2>&1 & RACERS="$RACERS $!" "$WD" run >/dev/null 2>&1 & RACERS="$RACERS $!" until_ok test -s "$FACTORY_STATE_DIR/watchdog.pid" - until_ok poller_count_is 1 $RACERS + until_ok runner_count_is 1 $RACERS # And the survivor is the one the pidfile names, with the file still there: a # loser exiting on the old read-then-write held a trap that deleted its - # successor's pidfile, which leaves exactly one poller and no claim on it. + # successor's pidfile, which leaves exactly one runner and no claim on it. [ -s "$FACTORY_STATE_DIR/watchdog.pid" ] owner=$(cat "$FACTORY_STATE_DIR/watchdog.pid") - poller_count_is 1 "$owner" + runner_count_is 1 "$owner" case " $RACERS " in *" $owner "*) ;; *) false ;; esac } -@test "grant starts a poller and revoke stops it" { - # The invariant that makes this structural rather than a step the foreman - # could forget: a live lease always has a watchdog. +@test "grant starts a runner and revoke stops it" { + # The invariant that makes this structural rather than a step anyone could + # forget: a live lease always has a runner. unset FACTORY_NO_WATCHDOG "$LEASECMD" grant 30m >/dev/null until_ok test -s "$FACTORY_STATE_DIR/watchdog.pid" @@ -538,11 +970,13 @@ EOF } # ── ⚠ an environment override may shorten a threshold, never lengthen it ───── -# `FACTORY_STALE`, `FACTORY_DEAD` and `FACTORY_WATCHDOG_INTERVAL` exist so this -# suite can reach a 45-minute threshold in seconds. A poller inherits the -# environment of whoever ran `lease grant` — on a night shift, the foreman — so -# a variable that could LENGTHEN `dead` was a foreman able to keep its lease -# after it died, and `config print`'s watchdog row was not what was in force. +# `FACTORY_STALE`, `FACTORY_DEAD`, `FACTORY_WATCHDOG_INTERVAL` and +# `FACTORY_RUNNER_INTERVAL` exist so this suite can reach a 45-minute threshold +# in seconds. The runner inherits the environment of whoever ran `lease grant`, +# so a variable that could LENGTHEN `dead` was a lease able to stand after its +# passes stopped, one that could lengthen the cadence was a night of fewer +# passes than the policy says, and `config print`'s rows were not what was in +# force. @test "⚠ an override longer than the policy's threshold is refused, not obeyed" { lease 3600 0 @@ -556,6 +990,9 @@ EOF FACTORY_WATCHDOG_INTERVAL=301 run "$WD" once [ "$status" -eq 2 ] [[ "$output" == *"may only shorten watchdog.interval"* ]] + FACTORY_RUNNER_INTERVAL=1201 run "$WD" once + [ "$status" -eq 2 ] + [[ "$output" == *"may only shorten runner.interval (1200s)"* ]] } @test "an override that is not a whole number of seconds is refused too" { @@ -573,15 +1010,15 @@ EOF [[ "$output" == *"may only shorten"* ]] } -@test "a grant whose poller refused to start says so, rather than ✓ over nothing" { +@test "a grant whose runner refused to start says so, rather than ✓ over nothing" { # `grant` discards `ensure`'s report, and used to discard its refusal with - # it: an override the watchdog may not honour left a ✓ lease with no poller + # it: an override the runner may not honour left a ✓ lease with no runner # and nothing on screen to say so until `watchdog once`. unset FACTORY_NO_WATCHDOG FACTORY_DEAD=999999 run "$LEASECMD" grant 30m [ "$status" -eq 0 ] [[ "$output" == *"lease: tier 1 until"* ]] - [[ "$output" == *"watchdog did not start"* ]] + [[ "$output" == *"runner did not start"* ]] [[ "$output" == *"may only shorten watchdog.dead"* ]] [ ! -s "$FACTORY_STATE_DIR/watchdog.pid" ] "$LEASECMD" revoke >/dev/null @@ -595,31 +1032,47 @@ EOF [[ "$output" == *"not greater than the stale threshold (2700)"* ]] } +@test "a shortened stale under an unshortened cadence is refused — it would call every gap between passes a stall" { + lease 3600 0 + log_aged 1 + run env -u FACTORY_RUNNER_INTERVAL FACTORY_STALE=2 FACTORY_DEAD=6 "$WD" once + [ "$status" -eq 2 ] + [[ "$output" == *"not greater than the pass cadence (1200)"* ]] +} + @test "an override equal to the policy's number is the policy's number" { # The boundary, so the check reads `-le` and not `-lt`: a suite that pins # the documented default through the env is not lengthening anything. lease 3600 0 log_aged 1 FACTORY_DEAD=5400 run "$WD" once - [ "$status" -eq 4 ] # a live lease, no poller — the override was accepted + [ "$status" -eq 4 ] # a live lease, no runner — the override was accepted } # The same double-pin the scope and budget defaults carry, one suite over: the -# README quotes all three thresholds, so a retune that only edits the code -# would otherwise leave the manual saying 45 and 90 with nothing red. -@test "the three watchdog defaults are still the ones the README states" { +# README quotes all five numbers, so a retune that only edits the code would +# otherwise leave the manual saying 45 and 90 with nothing red. +@test "the watchdog, runner and fixer defaults are still the ones the README states" { lib="$BATS_TEST_DIRNAME/../lib/common.sh" doc="$BATS_TEST_DIRNAME/../README.md" grep -q '"stale": 2700' "$lib" && grep -qF '`watchdog.stale`, 2700' "$doc" grep -q '"dead": 5400' "$lib" && grep -qF '`watchdog.dead`, 5400' "$doc" grep -q '"interval": 300' "$lib" && grep -qF '`watchdog.interval` (300' "$doc" + grep -q '"interval": 1200' "$lib" && grep -qF '`runner.interval` (1200' "$doc" + grep -q '"cap": 2' "$lib" && grep -qF '`fixer.cap` (2)' "$doc" + # And the starter config names none of the numbers, only the hook — the + # same claim the budget suite pins about its dials. + ex="$BATS_TEST_DIRNAME/../share/config.example.json" + ! grep -q '"cap"' "$ex" + ! grep -q '"runner"' "$ex" + grep -q '"fixer"' "$ex" } @test "a fractional watchdog threshold is refused, not a death nobody notices" { # The budget dials' hole, one block over and worse. `[ "$quiet" -ge "5400.5" ]` # complains to stderr and returns non-zero, which the `if` reads as false — at - # every poll, forever. So the foreman death this whole layer exists to notice - # is never noticed and the lease stands until morning. `type == "number"` was + # every tick, forever. So the breakdown this whole layer exists to notice is + # never noticed and the lease stands until morning. `type == "number"` was # true of it and `> 1` was true of it; only wholeness is not. printf '{"scope":{"orgs":["hausfold"]},"watchdog":{"dead":5400.5}}\n' >"$TMP/config.json" run "$WD" once @@ -627,6 +1080,41 @@ EOF [[ "$output" == *"watchdog thresholds must be whole numbers of seconds"* ]] } +@test "a fractional runner.interval is refused — a pass that is never due is a lease nobody exercises" { + printf '{"scope":{"orgs":["hausfold"]},"runner":{"interval":1200.5}}\n' >"$TMP/config.json" + run "$WD" once + [ "$status" -ne 0 ] + [[ "$output" == *"runner.interval must be a whole number of seconds"* ]] +} + +@test "a stale threshold inside the pass cadence is refused at load" { + printf '{"scope":{"orgs":["hausfold"]},"runner":{"interval":3000}}\n' >"$TMP/config.json" + run env -u FACTORY_RUNNER_INTERVAL -u FACTORY_STALE "$WD" once + [ "$status" -ne 0 ] + [[ "$output" == *"watchdog.stale must be greater than runner.interval"* ]] +} + +@test "a fixer.command written as a string is refused, not run letter by letter" { + printf '{"scope":{"orgs":["hausfold"]},"fixer":{"command":"spawner"}}\n' >"$TMP/config.json" + run "$WD" once + [ "$status" -ne 0 ] + [[ "$output" == *"fixer.command must be an array of argv words"* ]] + printf '{"scope":{"orgs":["hausfold"]},"fixer":{"command":[""]}}\n' >"$TMP/config.json" + run "$WD" once + [ "$status" -ne 0 ] + [[ "$output" == *"fixer.command starts with an empty word"* ]] +} + +@test "a fractional or negative fixer.cap is refused" { + printf '{"scope":{"orgs":["hausfold"]},"fixer":{"cap":1.5}}\n' >"$TMP/config.json" + run "$WD" once + [ "$status" -ne 0 ] + [[ "$output" == *"fixer.cap must be a whole number of lanes"* ]] + printf '{"scope":{"orgs":["hausfold"]},"fixer":{"cap":-1}}\n' >"$TMP/config.json" + run "$WD" once + [ "$status" -ne 0 ] +} + @test "a fractional tier1.maxLines is refused, and that one fails closed" { # The same slip where the consequence inverts: `[ "$churn" -le "2000.5" ]` # reads false too, so every PR is refused with a cap nobody can read printed