From 94c07e6f9ef3d40bcddfd80d11d0cbc62bb65fef Mon Sep 17 00:00:00 2001 From: mbullington Date: Sat, 22 Aug 2026 17:44:36 -0400 Subject: [PATCH] commit --- CHANGELOG.md | 10 + Cargo.lock | 237 +------------ Cargo.toml | 3 +- README.md | 45 ++- docs/index.html | 294 ---------------- docs/man/SCRIPTS.5.scd | 23 +- docs/man/SCRIPTS_WORKSPACE.toml.5.scd | 10 +- docs/man/scripts.1.scd | 44 ++- skills/scripts-runner/SKILL.md | 152 +++++++++ src/commands/clean.rs | 5 +- src/commands/env.rs | 22 +- src/commands/print_tree.rs | 12 +- src/commands/run.rs | 193 +++++++---- src/commands/run_executor.rs | 294 ++++++++++------ src/commands/run_plan.rs | 35 +- src/helpers/cache.rs | 102 +++++- src/helpers/graph.rs | 27 +- src/helpers/resolve.rs | 92 +++-- src/helpers/scripts_def.rs | 3 + src/helpers/task_list.rs | 33 +- src/main.rs | 24 +- tests/cli.rs | 467 +++++++++++++++++++++++++- 22 files changed, 1265 insertions(+), 862 deletions(-) delete mode 100644 docs/index.html create mode 100644 skills/scripts-runner/SKILL.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 267faf3..6e22ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- Run independent tasks concurrently with a bounded in-process DAG executor. +- Add `--jobs N`, defaulting to the host's logical CPU count. +- Continue independent branches after failures while skipping transitive dependents. +- Store atomic per-task cache entries so concurrent invocations cannot overwrite each other. +- Include workspace PATH configuration in task fingerprints. +- Reject malformed configuration, unknown fields, and dependency units outside the repository. +- Resolve plain task names from the nearest enclosing unit without filesystem-dependent parsing. +- Update watch registrations when the dependency graph changes. +- Add an authoritative agent skill under `skills/scripts-runner/`. + ## [0.1.0] - 2026-04-11 Initial public release. diff --git a/Cargo.lock b/Cargo.lock index 8b42d13..55112e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "aho-corasick" version = "1.1.3" @@ -82,12 +67,6 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" -[[package]] -name = "anymap2" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" - [[package]] name = "assert_cmd" version = "2.2.0" @@ -109,27 +88,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", -] - -[[package]] -name = "bimap" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" - [[package]] name = "bitflags" version = "1.3.2" @@ -286,31 +244,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "dagrs" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1392587e8929efcd58836ed2ffc9260b258dda280a9af0b0d1983210bb0f92" -dependencies = [ - "anymap2", - "bimap", - "clap", - "log", - "simplelog", - "thiserror", - "tokio", - "yaml-rust", -] - -[[package]] -name = "deranged" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" -dependencies = [ - "powerfmt", -] - [[package]] name = "difflib" version = "0.4.0" @@ -407,12 +340,6 @@ dependencies = [ "wasip3", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "globset" version = "0.4.16" @@ -507,17 +434,6 @@ dependencies = [ "libc", ] -[[package]] -name = "io-uring" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" -dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "libc", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -580,12 +496,6 @@ dependencies = [ "redox_syscall", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -604,15 +514,6 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - [[package]] name = "mio" version = "0.8.11" @@ -625,17 +526,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.59.0", -] - [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -656,7 +546,7 @@ dependencies = [ "kqueue", "libc", "log", - "mio 0.8.11", + "mio", "walkdir", "windows-sys 0.48.0", ] @@ -672,12 +562,6 @@ dependencies = [ "notify", ] -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - [[package]] name = "num-traits" version = "0.2.19" @@ -687,24 +571,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -717,24 +583,12 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - [[package]] name = "plain" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "predicates" version = "3.1.4" @@ -837,12 +691,6 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" -[[package]] -name = "rustc-demangle" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" - [[package]] name = "rustix" version = "1.1.4" @@ -880,7 +728,6 @@ dependencies = [ "clap", "clap_complete", "colored", - "dagrs", "hex", "ignore", "notify", @@ -964,23 +811,6 @@ dependencies = [ "digest", ] -[[package]] -name = "simplelog" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" -dependencies = [ - "log", - "termcolor", - "time", -] - -[[package]] -name = "slab" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" - [[package]] name = "strsim" version = "0.11.1" @@ -1011,15 +841,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "termtree" version = "0.4.1" @@ -1052,53 +873,6 @@ dependencies = [ "syn", ] -[[package]] -name = "time" -version = "0.3.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" -dependencies = [ - "deranged", - "itoa", - "libc", - "num-conv", - "num_threads", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" - -[[package]] -name = "time-macros" -version = "0.2.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tokio" -version = "1.47.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" -dependencies = [ - "backtrace", - "io-uring", - "libc", - "mio 1.0.4", - "pin-project-lite", - "slab", -] - [[package]] name = "toml" version = "0.8.23" @@ -1580,12 +1354,3 @@ dependencies = [ "unicode-xid", "wasmparser", ] - -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] diff --git a/Cargo.toml b/Cargo.toml index b7e499f..eafab95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "scripts_runner" version = "0.1.0" edition = "2021" -description = "Pragmatic monorepo task runner with dependency graphs and content-aware caching." +description = "Parallel monorepo task runner with dependency graphs and content-aware caching." license = "MIT" readme = "README.md" repository = "https://github.com/mbullington/scripts" @@ -35,7 +35,6 @@ anyhow = "1.0.86" thiserror = "1.0.61" # Misc -dagrs = "0.2.0" ignore = "0.4.23" hex = "0.4" colored = "2" diff --git a/README.md b/README.md index 018efa3..0928ffe 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # scripts -A pragmatic monorepo task runner with content-aware caching and watch mode. +A parallel monorepo task runner with content-aware caching and watch mode. - simple TOML configuration -- dependency graphs across units and languages +- bounded parallel execution of dependency graphs - content-aware caching - watch mode for development workflows - no daemon, no remote service, intentionally non-hermetic @@ -44,6 +44,7 @@ scripts run :build scripts run :test scripts run :build --force scripts run :build --watch +scripts run --jobs 4 :test ``` ## Task fields @@ -56,6 +57,8 @@ scripts run :build --watch - non-empty list: hash command text plus watched file contents - `bin`: optional list of paths added to `PATH` for the task and its dependents +Unknown task fields are errors, so misspelled keys cannot silently change task behavior. + ## Workspace configuration At the git root you can add an optional `SCRIPTS_WORKSPACE.toml` file: @@ -74,6 +77,8 @@ bin_append = [ ] ``` +Malformed workspace configuration and unknown fields are errors. + ## Commands ### `scripts run [OPTIONS] [-- ARGS...]` @@ -85,15 +90,19 @@ scripts run app:build scripts run build scripts run :build --watch scripts run dev -- echo done +scripts run --jobs 4 app:build scripts run --force tools/pkg:build scripts run --quiet app:build scripts run --verbose app:build ``` Notes: -- use `app:build` for another unit, or `build` / `:build` for the current unit +- use `app:build` for another unit, or `build` / `:build` for the nearest enclosing unit +- independent tasks run concurrently; `--jobs N` sets the limit, which defaults to the logical CPU count +- a failed task skips its dependents, while independent branches continue - anything after `--` is appended to the root task command and becomes part of the cache key - `--watch` starts after the graph finishes, then re-runs the target graph when watched inputs change +- watch mode updates its watched units when the dependency graph changes - `--quiet` suppresses routine task status lines but still streams task output - `--verbose` shows the working directory and shell command for each task - task status lines are written to stderr so stdout stays usable for task output @@ -121,7 +130,7 @@ scripts print-tree app:test --json ### `scripts clean [PATH]` -Remove the repository cache file. +Remove the repository cache directory. ```sh scripts clean @@ -145,11 +154,11 @@ Supported shells: `bash`, `elvish`, `fish`, `powershell`, `zsh`. ## Target syntax - `:` — run a specific task in another unit -- `` — run a task in the current unit -- `:` — also run a task in the current unit +- `` — run a task in the nearest enclosing unit +- `:` — also run a task in the nearest enclosing unit -If you provide a path-like target without a task name, `scripts` will ask for -`:` explicitly. +Target parsing does not inspect the filesystem. A plain name is always a task; +use the colon form to name another unit. ## Manual pages @@ -190,11 +199,12 @@ Units are directories containing a `SCRIPTS` file. Dependencies resolve by searching upward from the depending unit toward the git root: -- `(unit root)/..` -- `(unit root)/../..` -- and so on until `(git root)` +- `(unit root)/` +- `(unit root)/../` +- and so on through `(git root)/` -The first matching path that contains a `SCRIPTS` file wins. +The first matching path that contains a `SCRIPTS` file wins. Resolved units must +remain inside the git repository. ## Cache behavior @@ -203,12 +213,21 @@ For each task with `watch` present, `scripts` hashes: - a cache format version - the task command text - dependency, `bin`, and `watch` declarations +- workspace `bin_append` configuration - the contents of any watched files -The repository `.scripts_cache` file is ignored when hashing watched files, so broad patterns like `watch = ["."]` do not invalidate themselves. +The repository `.scripts_cache/` directory stores one atomic entry per task and +is ignored when hashing watched files. Separate entries let concurrent +invocations update different tasks without overwriting each other. A task is cached only when its own hash matches and none of its dependencies had to rerun. +## Agent skill + +The repository owns an agent reference at +[`skills/scripts-runner/SKILL.md`](skills/scripts-runner/SKILL.md). Keep it in +sync with CLI and configuration changes. + ## Non-goals - **Hermeticity.** `scripts` does not isolate builds from the host environment or require every dependency to be modeled inside `scripts`. diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index 16a8b8f..0000000 --- a/docs/index.html +++ /dev/null @@ -1,294 +0,0 @@ - - - - Scripts - Task Runner - - - - -
- -
-

Scripts

-

- A pragmatic monorepo task runner. Zero daemon, content-aware caching, and simple configuration. -

-
- - - - - Docs - - - - - - GitHub - -
-
- -
- -
- Configuration -
-
- [project]
- name = "web-app"

- [scripts]
- build = "npm run build"
- test = "npm test" -
-
- - Full configuration docs -
-
-
- -
- Orchestration -
-
- - Task Graph Execution -
-
- - Content-aware Caching -
-
- - Cross-language support -
-
- - Parallel Execution -
-
-
- -
- Ecosystem -
-
- - Workspaces support -
-
- - Docker integration -
-
- - Remote Caching -
-
-
- -
- Community -
-
- - Discord Chat -
-
- - Sponsors -
-
-
- -
-
- - diff --git a/docs/man/SCRIPTS.5.scd b/docs/man/SCRIPTS.5.scd index 2438166..83ac763 100644 --- a/docs/man/SCRIPTS.5.scd +++ b/docs/man/SCRIPTS.5.scd @@ -21,6 +21,8 @@ Each top-level table defines one task directly. For example: [build] ``` +Unknown task keys are errors. + # TASK KEYS ## deps @@ -41,7 +43,7 @@ Examples: Dependency units are resolved by searching upward from the depending unit toward the git repository root. The first matching directory which contains a -*SCRIPTS* file is selected. +*SCRIPTS* file is selected. Resolved units must remain inside the repository. ## command @@ -93,6 +95,13 @@ deps = [":build", "packages/api:test"] watch = [] ``` +# EXECUTION + +Tasks become runnable after all dependencies succeed or are cached. Independent +runnable tasks execute concurrently, bounded by *scripts run --jobs*. If a task +fails, its transitive dependents are skipped while independent branches +continue. + # RESOLUTION Given a dependency path declared from some unit directory, *scripts*(1) @@ -104,21 +113,25 @@ checks: - and so on until the git repository root The first candidate directory containing *SCRIPTS* is treated as the dependency -unit. +unit. Candidates which resolve outside the git repository are rejected. # CACHING -The cache key identifies a task by its canonical unit path and task name. +The cache key identifies a task by its repository-relative unit path and task +name. The cached hash for a task with *watch* present is derived from: - a cache format version - the command text - dependency, *bin*, and *watch* declarations +- workspace *bin_append* configuration - watched file contents -The repository *.scripts_cache* file is ignored when hashing watched files, so -broad patterns like *watch = ["."]* do not invalidate themselves. +The repository *.scripts_cache/* directory is ignored when hashing watched +files, so broad patterns like *watch = ["."]* do not invalidate themselves. +Each task owns one atomically replaced cache entry, allowing concurrent +*scripts*(1) processes to update different tasks safely. If a watched file is renamed, added, removed, or modified, the task hash changes. diff --git a/docs/man/SCRIPTS_WORKSPACE.toml.5.scd b/docs/man/SCRIPTS_WORKSPACE.toml.5.scd index 06a29da..2cfc315 100644 --- a/docs/man/SCRIPTS_WORKSPACE.toml.5.scd +++ b/docs/man/SCRIPTS_WORKSPACE.toml.5.scd @@ -14,7 +14,8 @@ to be appended to *PATH* for every task through *bin_append*. # FORMAT -A workspace file is regular UTF-8 TOML. +A workspace file is regular UTF-8 TOML. Unknown keys, malformed values, and +unreadable files are errors. # KEYS @@ -42,11 +43,11 @@ bin_append = [ ] ``` -### path +*path* Relative path to a directory that should be prepended to *PATH*. -### relative_to +*relative_to* How the entry should be resolved. @@ -88,6 +89,9 @@ bin_append = [ Workspace bins are appended after task-local and dependency-provided *bin* entries, so local task configuration still wins. +The workspace configuration is part of each cacheable task's fingerprint. +Changing *bin_append* therefore invalidates cached tasks. + # SEE ALSO *scripts*(1), *SCRIPTS*(5), *toml*(5) diff --git a/docs/man/scripts.1.scd b/docs/man/scripts.1.scd index 5452d64..e753f9b 100644 --- a/docs/man/scripts.1.scd +++ b/docs/man/scripts.1.scd @@ -8,7 +8,7 @@ scripts - pragmatic monorepo task runner with content-aware caching ``` scripts -scripts run [--force] [--quiet | --verbose] [--watch] target [-- args...] +scripts run [--force] [--quiet | --verbose] [--watch] [--jobs n] target [-- args...] scripts env target scripts print-tree [--json] [--flat] target scripts clean [path] @@ -36,7 +36,7 @@ from the nearest enclosing unit, if one can be found. Run a task and its dependencies. ``` -scripts run [--force] [--quiet | --verbose] [--watch] target [-- args...] +scripts run [--force] [--quiet | --verbose] [--watch] [--jobs n] target [-- args...] ``` If arguments are provided after *--*, they are appended to the root task's @@ -47,9 +47,14 @@ Use *--force* to ignore cached results and execute the task graph again. Use *--quiet* to suppress routine task status lines. Use *--verbose* to print each task's working directory and shell command before execution. +Independent tasks run concurrently. Use *--jobs n* to limit concurrent tasks. +The default is the number of logical CPUs. A failed task skips its dependents; +independent branches continue. + Use *--watch* to run the graph once, then monitor watched units and re-run the -target graph when changes are detected. Watch mode starts after the graph -finishes; it does not supervise long-running processes. +target graph when changes are detected. Watch mode updates its monitored units +when the dependency graph changes. It starts after the graph finishes and does +not supervise long-running processes. ## env @@ -78,7 +83,7 @@ Use *--json* for structured output or *--flat* for a deduplicated list. ## clean -Remove the repository cache file. +Remove the repository cache directory. ``` scripts clean [path] @@ -104,8 +109,8 @@ Supported shells include *bash*, *elvish*, *fish*, *powershell*, and *zsh*. Use one of these forms: - *:* for a task in another unit -- ** for a task in the current unit -- *:* also for a task in the current unit +- ** for a task in the nearest enclosing unit +- *:* also for a task in the nearest enclosing unit The unit portion names a directory containing a *SCRIPTS*(5) file. The task portion names one top-level task table in that file. @@ -116,8 +121,8 @@ Examples: - *test* - *:test* -If a path-like target is provided without *:*, *scripts* reports an error -and asks for the explicit *:* form. +Target parsing does not inspect the filesystem. A plain name always identifies +a task; use the colon form to identify another unit. # CACHING @@ -131,14 +136,17 @@ Caching behaves as follows: - non-empty *watch* list: hash the command text plus watched file contents A task is considered cached only when its own hash matches the stored value and -none of its dependencies had to rerun. +none of its dependencies had to rerun. Workspace *bin_append* configuration is +part of the hash. Cache entries are stored and replaced independently so +concurrent *scripts* processes cannot overwrite another task's entry. # WATCH MODE *--watch* runs the requested task graph once, then monitors the units participating in that graph and re-runs the graph when changes are detected. -Watch mode starts after the graph finishes; long-running process supervision is -left to the task command or dedicated tools. +After each run, it updates registrations for units added to or removed from the +graph. Watch mode starts after the graph finishes; long-running process +supervision is left to the task command or dedicated tools. # EXAMPLES @@ -160,6 +168,12 @@ Append an inline command fragment to the root task: scripts run :dev -- echo done ``` +Limit execution to four concurrent tasks: + +``` +scripts run --jobs 4 app:build +``` + Force a rerun even when inputs are unchanged: ``` @@ -227,15 +241,15 @@ scripts completions bash > ~/.local/share/bash-completion/completions/scripts *SCRIPTS_WORKSPACE.toml* Optional workspace configuration. See *SCRIPTS_WORKSPACE.toml*(5). -*.scripts_cache* - JSON cache file stored at the git repository root. +*.scripts_cache/* + Directory of per-task cache entries stored at the git repository root. # EXIT STATUS *0* on success. *1* if command-line parsing fails, a target cannot be resolved, a task command -fails, or a *SCRIPTS*(5) file is invalid. +fails, or a task or workspace configuration file is invalid. # NOTES diff --git a/skills/scripts-runner/SKILL.md b/skills/scripts-runner/SKILL.md new file mode 100644 index 0000000..24cf9d7 --- /dev/null +++ b/skills/scripts-runner/SKILL.md @@ -0,0 +1,152 @@ +--- +name: scripts-runner +description: Use when working with the `scripts` monorepo task runner: writing SCRIPTS files, configuring SCRIPTS_WORKSPACE.toml, running parallel task graphs, debugging caching or dependency resolution, or using watch mode. +--- + +# scripts — monorepo task runner + +A local monorepo task runner with bounded parallel DAG execution, content-aware caching, and watch mode. + +- TOML configuration in `SCRIPTS` files +- dependency graphs across units and languages +- independent tasks run concurrently +- per-task atomic cache entries +- no daemon, remote execution, hermetic sandbox, or process supervision +- Unix-like systems only; commands execute through `sh -c` + +## Core concepts + +A **unit** is a directory containing a `SCRIPTS` file. Commands invoked below a unit use the nearest enclosing unit. + +A **task** is a top-level TOML table inside a `SCRIPTS` file. + +A **target** uses one of these forms: + +- `:` — task in another unit, such as `app:build` +- `` — task in the nearest enclosing unit +- `:` — also a task in the nearest enclosing unit + +Parsing is lexical. A plain name always identifies a task, even when a file or directory has the same name. Use a colon to identify another unit. + +## Commands + +### `scripts run [OPTIONS] [-- ARGS...]` + +Run a task and its dependencies. + +- Independent ready tasks run concurrently. +- `--jobs N` sets the concurrency limit. The default is the logical CPU count. +- A failed task skips its transitive dependents; independent branches continue. +- `--force` ignores cached results and executes the graph. +- `--quiet` suppresses routine status lines but preserves task output and failures. +- `--verbose` prints each task's working directory and shell command. +- `--watch` reruns the graph when watched inputs change and updates registrations when the graph changes. +- Text after `--` is appended to the root task's shell command and included in its cache fingerprint. + +Status lines go to stderr. Task stdout remains available to pipelines. + +Examples: + +```sh +scripts run build +scripts run app:build +scripts run --jobs 4 app:test +scripts run --force :build +scripts run --watch :test +scripts run dev -- echo done +``` + +### `scripts env ` + +Start `$SHELL` with `PATH` prepared from the target, its dependencies, and workspace `bin_append` entries. The working directory does not change. + +### `scripts print-tree [--json] [--flat] ` + +Print a dependency graph. `tree` is an alias. + +### `scripts clean [PATH]` + +Remove `.scripts_cache/` from the Git root found from `PATH`, or from the current directory when omitted. + +### `scripts completions ` + +Generate completions for Bash, Elvish, Fish, PowerShell, or Zsh. + +## `SCRIPTS` format + +Each top-level table defines a task. Unknown task keys are errors. + +```toml +[build] +deps = ["tools/pkg:build", ":lint"] +command = "cargo build --release" +bin = ["target/release"] +watch = ["src/**", "Cargo.toml", "Cargo.lock"] + +[test] +deps = [":build"] +command = "cargo test" +watch = ["src/**", "tests/**"] +``` + +Task keys: + +- `deps` — optional dependency references +- `command` — optional shell command; omit it for a grouping task +- `bin` — optional unit-relative directories prepended to `PATH` for the task and its dependents +- `watch` — optional unit-relative files or glob patterns used for caching + +Caching semantics: + +- omitted `watch`: always run +- `watch = []`: hash declarations and command text without file contents +- non-empty `watch`: also hash matching file paths and contents + +A dependent reruns whenever one of its dependencies reruns. + +## Workspace configuration + +An optional `SCRIPTS_WORKSPACE.toml` at the Git root defines paths appended to every task's `PATH`: + +```toml +bin_append = [ + "tools/bin", + { path = "node_modules/.bin", relative_to = "unit" }, +] +``` + +String entries resolve from the Git root. Object entries set `relative_to` to `git_root` or `unit`. Missing directories are ignored. Malformed values and unknown fields are errors. + +Workspace configuration is part of cacheable task fingerprints. Changing `bin_append` invalidates cached tasks. + +## Dependency resolution + +For a dependency such as `tools/pkg:build`, `scripts` checks the depending unit, then each ancestor through the Git root. The first candidate containing `SCRIPTS` wins. A resolved unit may not escape the repository. + +## Cache behavior + +`.scripts_cache/` contains one opaque, atomically replaced entry per task. Repository-relative unit/task identity determines the entry name. Separate entries allow concurrent `scripts` processes to update different tasks safely. + +The task fingerprint includes: + +- cache format version +- effective root command +- dependency, `bin`, and `watch` declarations +- workspace `bin_append` configuration +- watched paths and file contents + +`.scripts_cache/` and `.git/` are excluded from watched content. + +## Watch mode + +Watch mode starts after the initial graph finishes. It watches units containing cacheable tasks plus workspace configuration. Each successful rerun rebuilds the graph and reconciles watched units, so newly added dependencies take effect without restarting. + +Watch mode reruns completed commands. It does not supervise long-running services. Do not background a server and expect `scripts` to manage its lifecycle; use a dedicated process supervisor. + +## Conventions + +- Keep `watch` patterns narrow enough to make cache hits useful and complete enough to prevent stale results. +- Use grouping tasks to name workflows. +- Model generated tools as dependency tasks with `bin` outputs. +- Use `--jobs 1` when a graph intentionally serializes access not represented by dependencies. +- Treat configuration errors as errors; do not rely on unknown keys being ignored. diff --git a/src/commands/clean.rs b/src/commands/clean.rs index a561767..78ddad8 100644 --- a/src/commands/clean.rs +++ b/src/commands/clean.rs @@ -8,7 +8,10 @@ pub fn cmd_clean_command(target: &str) -> Result<()> { let git_root = get_git_root(Path::new(target))?; let cache_path = git_root.join(".scripts_cache"); - if cache_path.exists() { + if cache_path.is_dir() { + std::fs::remove_dir_all(&cache_path)?; + println!("removed {}", cache_path.display()); + } else if cache_path.is_file() { std::fs::remove_file(&cache_path)?; println!("removed {}", cache_path.display()); } else { diff --git a/src/commands/env.rs b/src/commands/env.rs index 75e0c28..54f0941 100644 --- a/src/commands/env.rs +++ b/src/commands/env.rs @@ -1,28 +1,22 @@ -use std::path::Path; - use anyhow::Result; use crate::helpers::{ - git::get_git_root, - graph::{build_task_graph, TaskGraph}, + graph::build_target_graph, path::{build_path_var, collect_task_bins, resolve_workspace_bins}, - resolve::{parse_target, read_workspace_config}, + resolve::read_workspace_config, task_list::print_tasks_for_current_unit, }; pub fn cmd_env_command(target: &str) -> Result<()> { - let (unit, task_name) = parse_target(target)?; - let unit_path = Path::new(&unit); - - let git_root = get_git_root(unit_path)?; - let workspace_config = read_workspace_config(&git_root); - let graph: TaskGraph = match build_task_graph(unit_path, &task_name) { - Ok(graph) => graph, + let cwd = std::env::current_dir()?; + let (graph, git_root) = match build_target_graph(target, &cwd) { + Ok(result) => result, Err(error) => { print_tasks_for_current_unit(); - return Err(error.into()); + return Err(error); } }; + let workspace_config = read_workspace_config(&git_root)?; let root_unit_path = &graph.scripts[graph.root].unit_path; let mut bins = collect_task_bins(&graph, graph.root); @@ -35,7 +29,7 @@ pub fn cmd_env_command(target: &str) -> Result<()> { let shell = std::env::var("SHELL").unwrap_or_else(|_| String::from("/bin/sh")); std::process::Command::new(shell) .env("PATH", build_path_var(&bins)?) - .env("PS1", format!("({unit}) := ")) + .env("PS1", format!("({target}) := ")) .status()?; Ok(()) } diff --git a/src/commands/print_tree.rs b/src/commands/print_tree.rs index 53b42e0..c4386dd 100644 --- a/src/commands/print_tree.rs +++ b/src/commands/print_tree.rs @@ -4,11 +4,7 @@ use anyhow::Result; use serde::Serialize; use termtree::Tree; -use crate::helpers::{ - git::get_git_root, - graph::{build_task_graph, TaskGraph}, - resolve::parse_target, -}; +use crate::helpers::graph::{build_target_graph, TaskGraph}; fn get_tree(graph: &TaskGraph, handle: usize, relative_to: &Path) -> Tree { let node = &graph.scripts[handle]; @@ -63,10 +59,8 @@ fn get_flat( } pub fn cmd_print_tree_command(target: &str, json: bool, flat: bool) -> Result<()> { - let (unit, task) = parse_target(target)?; - let unit_path = Path::new(&unit); - let git_root = get_git_root(unit_path)?; - let graph: TaskGraph = build_task_graph(unit_path, &task)?; + let cwd = std::env::current_dir()?; + let (graph, git_root) = build_target_graph(target, &cwd)?; let root_handle = graph.root; if flat { let mut set = std::collections::BTreeSet::new(); diff --git a/src/commands/run.rs b/src/commands/run.rs index e0f7ea6..eb213b0 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -1,17 +1,21 @@ use std::{ - collections::HashSet, + collections::HashMap, + num::NonZeroUsize, path::{Path, PathBuf}, time::Duration, }; use anyhow::Result; -use notify_debouncer_mini::{new_debouncer, notify::RecursiveMode, DebounceEventResult}; +use notify_debouncer_mini::{ + new_debouncer, + notify::{RecursiveMode, Watcher}, + DebounceEventResult, +}; use crate::helpers::{ - cache::{load_cache, save_cache}, - git::get_git_root, - graph::{build_task_graph, TaskGraph}, - resolve::{parse_target, read_workspace_config}, + cache::TaskCache, + graph::{build_target_graph, TaskGraph}, + resolve::read_workspace_config, task_list::print_tasks_for_current_unit, }; @@ -20,8 +24,24 @@ use super::{ run_plan::RunPlan, }; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum WatchDepth { + NonRecursive, + Recursive, +} + +impl WatchDepth { + fn notify_mode(self) -> RecursiveMode { + match self { + Self::NonRecursive => RecursiveMode::NonRecursive, + Self::Recursive => RecursiveMode::Recursive, + } + } +} + struct RunOnceResult { - watch_roots: Vec, + watch_paths: HashMap, + execution_error: Option, } fn output_mode(quiet: bool, verbose: bool) -> RunOutputMode { @@ -34,66 +54,59 @@ fn output_mode(quiet: bool, verbose: bool) -> RunOutputMode { } } -fn collect_watch_roots(graph: &TaskGraph) -> Vec { - let mut roots = Vec::new(); - let mut seen = HashSet::new(); +fn collect_watch_paths(graph: &TaskGraph, git_root: &Path) -> HashMap { + let mut paths = HashMap::new(); for node in &graph.scripts { - if node.task.watch.is_some() && seen.insert(node.unit_path.clone()) { - roots.push(node.unit_path.clone()); + if node.task.watch.is_some() { + paths.insert(node.unit_path.clone(), WatchDepth::Recursive); } } - roots -} + if !paths.is_empty() { + paths + .entry(git_root.to_path_buf()) + .or_insert(WatchDepth::NonRecursive); + } -fn apply_cache_events( - cache: &mut std::collections::HashMap, - events: &[TaskEvent], -) -> bool { - let mut cache_changed = false; + paths +} +fn apply_cache_events(cache: &TaskCache, events: &[TaskEvent]) -> Result<()> { for event in events { match event { TaskEvent::Succeeded { cache_key, cache_hash: Some(cache_hash), - } => { - cache_changed |= cache.insert(cache_key.clone(), cache_hash.clone()).as_ref() - != Some(cache_hash); - } + } => cache.store(cache_key, cache_hash)?, TaskEvent::Succeeded { - cache_hash: None, .. - } => {} - TaskEvent::Failed { cache_key } => { - cache_changed |= cache.remove(cache_key).is_some(); - } + cache_key, + cache_hash: None, + } => cache.remove(cache_key)?, + TaskEvent::Failed { cache_key } => cache.remove(cache_key)?, } } - cache_changed + Ok(()) } fn run_once( target: &str, force: bool, output_mode: RunOutputMode, + jobs: usize, append_cmd: &Option, ) -> Result { - let (unit, task_name) = parse_target(target)?; - let unit_path = Path::new(&unit); - let git_root = get_git_root(unit_path)?; - let cache_path = git_root.join(".scripts_cache"); - let mut cache = load_cache(&cache_path)?; - let workspace_config = read_workspace_config(&git_root); - - let graph: TaskGraph = match build_task_graph(unit_path, &task_name) { - Ok(graph) => graph, + let cwd = std::env::current_dir()?; + let (graph, git_root) = match build_target_graph(target, &cwd) { + Ok(result) => result, Err(error) => { print_tasks_for_current_unit(); - return Err(error.into()); + return Err(error); } }; + let cache = TaskCache::open(&git_root)?; + let workspace_config = read_workspace_config(&git_root)?; let plan = RunPlan::build( &graph, @@ -103,37 +116,72 @@ fn run_once( append_cmd.as_ref(), workspace_config.as_ref(), )?; - let outcome = execute_plan(&plan, output_mode); + let outcome = execute_plan(&plan, output_mode, jobs); - if apply_cache_events(&mut cache, &outcome.events) { - save_cache(&cache_path, &cache)?; - } + apply_cache_events(&cache, &outcome.events)?; - outcome.into_result()?; Ok(RunOnceResult { - watch_roots: collect_watch_roots(&graph), + watch_paths: collect_watch_paths(&graph, &git_root), + execution_error: outcome.into_result().err(), }) } -fn event_is_relevant(path: &Path, git_root: &Path) -> bool { - path != git_root.join(".scripts_cache") && !path.starts_with(git_root.join(".git")) +fn event_is_relevant( + path: &Path, + git_root: &Path, + watched_paths: &HashMap, +) -> bool { + if path.starts_with(git_root.join(".scripts_cache")) || path.starts_with(git_root.join(".git")) + { + return false; + } + + if path.parent() == Some(git_root) + && watched_paths.get(git_root) == Some(&WatchDepth::NonRecursive) + { + return path + .file_name() + .is_some_and(|name| name == "SCRIPTS_WORKSPACE.toml"); + } + + true +} + +fn reconcile_watch_paths( + watcher: &mut W, + current: &mut HashMap, + desired: HashMap, +) -> Result<()> { + for (path, depth) in current.iter() { + if desired.get(path) != Some(depth) { + watcher.unwatch(path)?; + } + } + for (path, depth) in &desired { + if current.get(path) != Some(depth) { + watcher.watch(path, depth.notify_mode())?; + } + } + *current = desired; + Ok(()) } fn watch_target_graph( target: &str, output_mode: RunOutputMode, + jobs: usize, append_cmd: &Option, - watch_roots: Vec, + watch_paths: HashMap, ) -> Result<()> { - if watch_roots.is_empty() { + if watch_paths.is_empty() { eprintln!("watch mode requested, but no watched tasks were found in the target graph"); return Ok(()); } eprintln!("watching for changes... (Ctrl+C to exit)"); - let (unit, _) = parse_target(target)?; - let git_root = get_git_root(Path::new(&unit))?; + let cwd = std::env::current_dir()?; + let (_, git_root) = build_target_graph(target, &cwd)?; let (tx, rx) = std::sync::mpsc::channel(); let mut debouncer = new_debouncer( @@ -143,26 +191,38 @@ fn watch_target_graph( }, )?; - let mut watched_roots = HashSet::new(); - for root in watch_roots { - if watched_roots.insert(root.clone()) { - debouncer.watcher().watch(&root, RecursiveMode::Recursive)?; - } - } + let mut watched_paths = HashMap::new(); + reconcile_watch_paths(debouncer.watcher(), &mut watched_paths, watch_paths)?; loop { match rx.recv() { Ok(Ok(events)) => { let saw_relevant_change = events .iter() - .any(|event| event_is_relevant(&event.path, &git_root)); + .any(|event| event_is_relevant(&event.path, &git_root, &watched_paths)); if !saw_relevant_change { continue; } eprintln!("change detected; re-running target graph"); - if let Err(error) = run_once(target, false, output_mode, append_cmd) { - eprintln!("watch re-run failed: {error}"); + match run_once(target, false, output_mode, jobs, append_cmd) { + Ok(result) => { + reconcile_watch_paths( + debouncer.watcher(), + &mut watched_paths, + result.watch_paths, + )?; + if let Some(error) = result.execution_error { + eprintln!("watch re-run failed: {error}"); + } + if watched_paths.is_empty() { + eprintln!( + "watch mode stopped because no watched tasks remain in the target graph" + ); + return Ok(()); + } + } + Err(error) => eprintln!("watch re-run failed: {error}"), } eprintln!("watching for changes... (Ctrl+C to exit)"); } @@ -182,14 +242,23 @@ pub fn cmd_run_command( quiet: bool, verbose: bool, watch: bool, + jobs: Option, append_cmd: Option, ) -> Result<()> { let output_mode = output_mode(quiet, verbose); + let jobs = jobs.map(NonZeroUsize::get).unwrap_or_else(|| { + std::thread::available_parallelism() + .map(NonZeroUsize::get) + .unwrap_or(1) + }); - let initial = run_once(target, force, output_mode, &append_cmd)?; + let initial = run_once(target, force, output_mode, jobs, &append_cmd)?; + if let Some(error) = initial.execution_error { + return Err(error); + } if !watch { return Ok(()); } - watch_target_graph(target, output_mode, &append_cmd, initial.watch_roots) + watch_target_graph(target, output_mode, jobs, &append_cmd, initial.watch_paths) } diff --git a/src/commands/run_executor.rs b/src/commands/run_executor.rs index 130cb15..437a086 100644 --- a/src/commands/run_executor.rs +++ b/src/commands/run_executor.rs @@ -1,8 +1,7 @@ -use std::sync::{Arc, Mutex}; +use std::{collections::VecDeque, sync::mpsc, thread}; use anyhow::Result; use colored::*; -use dagrs::{log as daglog, Action, Dag, DefaultTask, EnvVar, Input, LogLevel, Output, Task}; use super::run_plan::{PlanEntry, RunPlan}; @@ -26,6 +25,7 @@ impl RunOutputMode { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum TaskState { Pending, + Running, Cached, Succeeded, Failed, @@ -47,16 +47,14 @@ pub enum TaskEvent { pub struct ExecutionOutcome { pub events: Vec, pub states: Vec, - dag_failed: bool, } impl ExecutionOutcome { pub fn into_result(self) -> Result<()> { - if self.dag_failed - || self - .states - .iter() - .any(|state| matches!(state, TaskState::Failed | TaskState::Skipped)) + if self + .states + .iter() + .any(|state| !matches!(state, TaskState::Cached | TaskState::Succeeded)) { anyhow::bail!("one or more tasks failed"); } @@ -65,134 +63,210 @@ impl ExecutionOutcome { } } -struct RunAction { - entry: PlanEntry, - output_mode: RunOutputMode, - states: Arc>>, - events: Arc>>, +struct CommandCompletion { + index: usize, + result: std::io::Result, } -pub fn execute_plan(plan: &RunPlan, output_mode: RunOutputMode) -> ExecutionOutcome { - daglog::init_logger(LogLevel::Off, None); +struct Scheduler<'a> { + plan: &'a RunPlan, + output_mode: RunOutputMode, + states: Vec, + events: Vec, + remaining_dependencies: Vec, + dependents: Vec>, + ready: VecDeque, +} - let states = Arc::new(Mutex::new(vec![TaskState::Pending; plan.entries.len()])); - let events = Arc::new(Mutex::new(Vec::new())); - let mut dag_tasks = Vec::with_capacity(plan.entries.len()); +impl<'a> Scheduler<'a> { + fn new(plan: &'a RunPlan, output_mode: RunOutputMode) -> Self { + let remaining_dependencies: Vec = plan + .entries + .iter() + .map(|entry| entry.dependencies.len()) + .collect(); + let mut dependents = vec![Vec::new(); plan.entries.len()]; + for entry in &plan.entries { + for dependency in &entry.dependencies { + dependents[*dependency].push(entry.index); + } + } + let ready = remaining_dependencies + .iter() + .enumerate() + .filter_map(|(index, remaining)| (*remaining == 0).then_some(index)) + .collect(); - for entry in &plan.entries { - let action = RunAction { - entry: entry.clone(), + Self { + plan, output_mode, - states: states.clone(), - events: events.clone(), - }; - dag_tasks.push(DefaultTask::new(action, &entry.name)); + states: vec![TaskState::Pending; plan.entries.len()], + events: Vec::new(), + remaining_dependencies, + dependents, + ready, + } } - for entry in &plan.entries { - let dep_ids: Vec = entry - .dependencies - .iter() - .map(|&handle| dag_tasks[handle].id()) - .collect(); - dag_tasks[entry.index].set_predecessors_by_id(&dep_ids); + fn next_ready(&mut self) -> Option { + while let Some(index) = self.ready.pop_front() { + if self.states[index] == TaskState::Pending { + return Some(index); + } + } + None } - let mut dag = Dag::with_tasks(dag_tasks); - let dag_failed = dag.start().is_err(); + fn complete_cached(&mut self, index: usize) { + self.states[index] = TaskState::Cached; + print_status( + self.output_mode, + "CACHED", + &self.plan.entries[index].name, + None, + ); + self.release_dependents(index); + } - let mut states = states.lock().unwrap().clone(); - for entry in &plan.entries { - if entry.should_run && states[entry.index] == TaskState::Pending { - states[entry.index] = TaskState::Skipped; + fn complete_success(&mut self, index: usize) { + let entry = &self.plan.entries[index]; + self.states[index] = TaskState::Succeeded; + self.events.push(TaskEvent::Succeeded { + cache_key: entry.cache_key.clone(), + cache_hash: entry.cache_hash.clone(), + }); + print_status(self.output_mode, "OK", &entry.name, None); + self.release_dependents(index); + } + + fn complete_failure(&mut self, index: usize, detail: &str) { + let entry = &self.plan.entries[index]; + self.states[index] = TaskState::Failed; + self.events.push(TaskEvent::Failed { + cache_key: entry.cache_key.clone(), + }); + print_status(self.output_mode, "FAIL", &entry.name, Some(detail)); + self.skip_dependents(index); + } + + fn release_dependents(&mut self, index: usize) { + for dependent in &self.dependents[index] { + if self.states[*dependent] != TaskState::Pending { + continue; + } + self.remaining_dependencies[*dependent] -= 1; + if self.remaining_dependencies[*dependent] == 0 { + self.ready.push_back(*dependent); + } + } + } + + fn skip_dependents(&mut self, index: usize) { + for dependent in self.dependents[index].clone() { + if self.states[dependent] != TaskState::Pending { + continue; + } + self.states[dependent] = TaskState::Skipped; + self.ready.retain(|queued| *queued != dependent); print_status( - output_mode, + self.output_mode, "SKIP", - &entry.name, + &self.plan.entries[dependent].name, Some("(dependency failed)"), ); + self.skip_dependents(dependent); } } - let events = events.lock().unwrap().clone(); - ExecutionOutcome { - events, - states, - dag_failed, + fn into_outcome(self) -> ExecutionOutcome { + ExecutionOutcome { + events: self.events, + states: self.states, + } } } -impl Action for RunAction { - fn run(&self, _input: Input, _env: Arc) -> Result { - if !self.entry.should_run { - self.set_state(TaskState::Cached); - print_status(self.output_mode, "CACHED", &self.entry.name, None); - return Ok(Output::empty()); - } +pub fn execute_plan(plan: &RunPlan, output_mode: RunOutputMode, jobs: usize) -> ExecutionOutcome { + assert!(jobs > 0, "executor requires at least one job"); + + let mut scheduler = Scheduler::new(plan, output_mode); + let (completion_tx, completion_rx) = mpsc::channel(); + let mut active = 0; + + thread::scope(|scope| loop { + while active < jobs { + let Some(index) = scheduler.next_ready() else { + break; + }; + let entry = &plan.entries[index]; + + if !entry.should_run { + scheduler.complete_cached(index); + continue; + } - let command = match &self.entry.command { - Some(command) => command, - None => { - print_status( - self.output_mode, - "RUN", - &self.entry.name, - Some("(no command)"), - ); - return self.finish_success(); + let Some(command) = entry.command.clone() else { + print_status(output_mode, "RUN", &entry.name, Some("(no command)")); + scheduler.complete_success(index); + continue; + }; + + print_status(output_mode, "RUN", &entry.name, None); + print_verbose_command(output_mode, entry, &command); + scheduler.states[index] = TaskState::Running; + + let dir = entry.dir.clone(); + let path_var = entry.path_var.clone(); + let completion_tx = completion_tx.clone(); + let spawn_result = thread::Builder::new().spawn_scoped(scope, move || { + use std::process::Stdio; + + let result = std::process::Command::new("sh") + .args(["-c", &command]) + .current_dir(dir) + .env("PATH", path_var) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status(); + let _ = completion_tx.send(CommandCompletion { index, result }); + }); + + match spawn_result { + Ok(_) => active += 1, + Err(error) => { + scheduler.complete_failure(index, &format!("(failed to start worker: {error})")) + } } - }; - - print_status(self.output_mode, "RUN", &self.entry.name, None); - print_verbose_command(self.output_mode, &self.entry, command); - - use std::process::Stdio; - - let status = std::process::Command::new("sh") - .args(["-c", command]) - .current_dir(&self.entry.dir) - .env("PATH", &self.entry.path_var) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .status() - .map_err(dagrs::RunningError::from_err)?; - - if status.success() { - self.finish_success() - } else { - self.finish_failure(); - let exit_code = status.code(); - let detail = format!("(exit code: {exit_code:?})"); - print_status(self.output_mode, "FAIL", &self.entry.name, Some(&detail)); - Err(dagrs::RunningError::new(format!( - "task {} failed with exit code {exit_code:?}", - self.entry.name - ))) } - } -} -impl RunAction { - fn set_state(&self, state: TaskState) { - self.states.lock().unwrap()[self.entry.index] = state; - } + if active == 0 { + if scheduler.ready.is_empty() { + break; + } + continue; + } - fn finish_success(&self) -> Result { - self.set_state(TaskState::Succeeded); - self.events.lock().unwrap().push(TaskEvent::Succeeded { - cache_key: self.entry.cache_key.clone(), - cache_hash: self.entry.cache_hash.clone(), - }); - print_status(self.output_mode, "OK", &self.entry.name, None); - Ok(Output::empty()) - } + let completion = completion_rx + .recv() + .expect("worker completion channel closed unexpectedly"); + active -= 1; + match completion.result { + Ok(status) if status.success() => scheduler.complete_success(completion.index), + Ok(status) => { + let detail = match status.code() { + Some(code) => format!("(exit code: {code})"), + None => "(terminated by signal)".to_string(), + }; + scheduler.complete_failure(completion.index, &detail); + } + Err(error) => scheduler.complete_failure( + completion.index, + &format!("(failed to start command: {error})"), + ), + } + }); - fn finish_failure(&self) { - self.set_state(TaskState::Failed); - self.events.lock().unwrap().push(TaskEvent::Failed { - cache_key: self.entry.cache_key.clone(), - }); - } + scheduler.into_outcome() } fn status_label(label: &str) -> colored::ColoredString { diff --git a/src/commands/run_plan.rs b/src/commands/run_plan.rs index 4b3dadc..0b44453 100644 --- a/src/commands/run_plan.rs +++ b/src/commands/run_plan.rs @@ -1,9 +1,9 @@ -use std::{collections::HashMap, ffi::OsString, path::PathBuf}; +use std::{ffi::OsString, path::PathBuf}; use anyhow::Result; use crate::helpers::{ - cache::compute_task_hash, + cache::{compute_task_hash, TaskCache}, graph::TaskGraph, path::{build_path_var, collect_task_bins, resolve_workspace_bins}, scripts_def::WorkspaceConfig, @@ -31,7 +31,7 @@ impl RunPlan { pub fn build( graph: &TaskGraph, git_root: &std::path::Path, - cache: &HashMap, + cache: &TaskCache, force: bool, append_cmd: Option<&String>, workspace_config: Option<&WorkspaceConfig>, @@ -44,9 +44,9 @@ impl RunPlan { } else { node.task.command.clone() }; - let cache_key = format!("{}:{}", node.unit_path.display(), node.task_name); - let cache_hash = compute_task_hash(node, command.as_deref())?; - let should_run = should_run_task(cache, force, &cache_key, cache_hash.as_ref()); + let cache_key = task_cache_key(graph, idx, git_root)?; + let cache_hash = compute_task_hash(node, command.as_deref(), workspace_config)?; + let should_run = should_run_task(cache, force, &cache_key, cache_hash.as_ref())?; let mut bins = collect_task_bins(graph, idx); bins.extend(resolve_workspace_bins( @@ -74,18 +74,20 @@ impl RunPlan { } fn should_run_task( - cache: &HashMap, + cache: &TaskCache, force: bool, cache_key: &str, cache_hash: Option<&String>, -) -> bool { +) -> Result { if force { - return true; + return Ok(true); } match cache_hash { - Some(hash) => cache.get(cache_key).is_none_or(|previous| previous != hash), - None => true, + Some(hash) => Ok(cache + .get(cache_key)? + .is_none_or(|previous| previous != *hash)), + None => Ok(true), } } @@ -102,6 +104,17 @@ fn mark_dependents_dirty(entries: &mut [PlanEntry]) { } } +fn task_cache_key(graph: &TaskGraph, idx: usize, git_root: &std::path::Path) -> Result { + let node = &graph.scripts[idx]; + let relative = node.unit_path.strip_prefix(git_root)?; + let unit = if relative.as_os_str().is_empty() { + ".".into() + } else { + relative.to_string_lossy() + }; + Ok(format!("{unit}:{}", node.task_name)) +} + fn task_display_name(graph: &TaskGraph, idx: usize, relative_to: &std::path::Path) -> String { let node = &graph.scripts[idx]; let path = node diff --git a/src/helpers/cache.rs b/src/helpers/cache.rs index a1423b6..ef383a5 100644 --- a/src/helpers/cache.rs +++ b/src/helpers/cache.rs @@ -1,13 +1,24 @@ -use std::{collections::HashMap, fs::File, io::Read, path::Path}; +use std::{ + fs::{File, OpenOptions}, + io::{ErrorKind, Read, Write}, + path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, +}; use anyhow::Result; use ignore::{overrides::OverrideBuilder, WalkBuilder}; use serde::Serialize; use sha2::{Digest, Sha256}; -use crate::helpers::graph::TaskGraphNode; +use crate::helpers::{graph::TaskGraphNode, scripts_def::WorkspaceConfig}; -const CACHE_FORMAT_VERSION: &str = "scripts-cache-v2"; +const CACHE_FORMAT_VERSION: &str = "scripts-cache-v3"; +static TEMP_FILE_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Debug)] +pub struct TaskCache { + root: PathBuf, +} #[derive(Serialize)] struct TaskFingerprint<'a> { @@ -17,22 +28,84 @@ struct TaskFingerprint<'a> { bin: Option<&'a [String]>, watch: &'a [String], watch_hashes: Vec, + workspace: Option<&'a WorkspaceConfig>, } -pub fn load_cache(path: &Path) -> Result> { - if path.exists() { - Ok(serde_json::from_str(&std::fs::read_to_string(path)?)?) - } else { - Ok(HashMap::new()) +impl TaskCache { + pub fn open(git_root: &Path) -> Result { + let root = git_root.join(".scripts_cache"); + if root.is_file() { + match std::fs::remove_file(&root) { + Ok(()) => {} + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + } + Ok(Self { root }) + } + + pub fn get(&self, key: &str) -> Result> { + let path = self.entry_path(key); + match std::fs::read_to_string(path) { + Ok(value) => Ok(Some(value)), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.into()), + } + } + + pub fn store(&self, key: &str, value: &str) -> Result<()> { + std::fs::create_dir_all(&self.root)?; + + let path = self.entry_path(key); + let sequence = TEMP_FILE_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let temp_path = self.root.join(format!( + ".{}.{}.{}.tmp", + cache_file_name(key), + std::process::id(), + sequence + )); + + let result = (|| { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp_path)?; + file.write_all(value.as_bytes())?; + file.sync_all()?; + std::fs::rename(&temp_path, path)?; + Ok(()) + })(); + + if result.is_err() { + let _ = std::fs::remove_file(temp_path); + } + result + } + + pub fn remove(&self, key: &str) -> Result<()> { + match std::fs::remove_file(self.entry_path(key)) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + } + + fn entry_path(&self, key: &str) -> PathBuf { + self.root.join(cache_file_name(key)) } } -pub fn save_cache(path: &Path, cache: &HashMap) -> Result<()> { - std::fs::write(path, serde_json::to_string(cache)?)?; - Ok(()) +fn cache_file_name(key: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(key.as_bytes()); + hex::encode(hasher.finalize()) } -pub fn compute_task_hash(node: &TaskGraphNode, command: Option<&str>) -> Result> { +pub fn compute_task_hash( + node: &TaskGraphNode, + command: Option<&str>, + workspace_config: Option<&WorkspaceConfig>, +) -> Result> { let Some(patterns) = &node.task.watch else { return Ok(None); }; @@ -48,6 +121,7 @@ pub fn compute_task_hash(node: &TaskGraphNode, command: Option<&str>) -> Result< bin: node.task.bin.as_deref(), watch: patterns, watch_hashes, + workspace: workspace_config, }; let mut hasher = Sha256::new(); @@ -72,7 +146,9 @@ fn hash_file(path: &Path) -> Result<[u8; 32]> { } fn should_skip_watch_entry(relative_path: &str) -> bool { - relative_path == ".scripts_cache" || relative_path.starts_with(".git/") + relative_path == ".scripts_cache" + || relative_path.starts_with(".scripts_cache/") + || relative_path.starts_with(".git/") } fn compute_watch_hash(root: &Path, pattern: &str) -> Result { diff --git a/src/helpers/graph.rs b/src/helpers/graph.rs index 9c5d701..38e4534 100644 --- a/src/helpers/graph.rs +++ b/src/helpers/graph.rs @@ -7,7 +7,11 @@ use std::{ use thiserror::Error; use super::{ - resolve::{parse_dependency, read_scripts, resolve_scripts_path, ResolveScriptsError}, + git::get_git_root, + resolve::{ + find_enclosing_unit, parse_dependency, parse_target, read_scripts, resolve_scripts_path, + ResolveScriptsError, + }, scripts_def::Task, }; @@ -79,6 +83,27 @@ fn format_task_ref(unit_path: &Path, task_name: &str) -> String { format!("{}:{}", unit_path.display(), task_name) } +pub fn build_target_graph( + target: &str, + working_dir: &Path, +) -> anyhow::Result<(TaskGraph, PathBuf)> { + let (unit, task) = parse_target(target)?; + let git_root = get_git_root(working_dir)?.canonicalize()?; + let initial_path = if unit == "." { + find_enclosing_unit(working_dir, &git_root).map_err(|error| TaggedResolveScriptsError { + path: working_dir.to_path_buf(), + error, + })? + } else { + resolve_scripts_path(&unit, working_dir).map_err(|error| TaggedResolveScriptsError { + path: working_dir.to_path_buf(), + error, + })? + }; + let graph = build_task_graph(&initial_path, &task)?; + Ok((graph, git_root)) +} + pub fn build_task_graph( initial_path: &Path, task: &str, diff --git a/src/helpers/resolve.rs b/src/helpers/resolve.rs index 7d057ae..3d7b67b 100644 --- a/src/helpers/resolve.rs +++ b/src/helpers/resolve.rs @@ -3,7 +3,7 @@ use std::{ path::{Path, PathBuf}, }; -use anyhow::{bail, Result as AnyhowResult}; +use anyhow::{Context, Result as AnyhowResult}; use crate::helpers::scripts_def::{ScriptsDef, WorkspaceConfig}; @@ -30,7 +30,10 @@ pub fn resolve_scripts_path( path: &str, working_dir: &Path, ) -> Result { - let git_root = get_git_root(working_dir).map_err(ResolveScriptsError::GitError)?; + let git_root = get_git_root(working_dir) + .map_err(ResolveScriptsError::GitError)? + .canonicalize() + .map_err(ResolveScriptsError::IO)?; let mut current_dir = PathBuf::from(working_dir) .canonicalize() @@ -38,7 +41,13 @@ pub fn resolve_scripts_path( loop { let candidate = current_dir.join(path); if candidate.join("SCRIPTS").is_file() { - return candidate.canonicalize().map_err(ResolveScriptsError::IO); + let candidate = candidate.canonicalize().map_err(ResolveScriptsError::IO)?; + if !candidate.starts_with(&git_root) { + return Err(ResolveScriptsError::InvalidTarget(format!( + "unit path '{path}' resolves outside the git repository" + ))); + } + return Ok(candidate); } if current_dir == git_root { @@ -54,6 +63,31 @@ pub fn resolve_scripts_path( Err(ResolveScriptsError::DoesNotExist("Unit not found")) } +pub fn find_enclosing_unit( + working_dir: &Path, + git_root: &Path, +) -> Result { + let git_root = git_root.canonicalize().map_err(ResolveScriptsError::IO)?; + let mut current = working_dir + .canonicalize() + .map_err(ResolveScriptsError::IO)?; + + while current.starts_with(&git_root) { + if current.join("SCRIPTS").is_file() { + return Ok(current); + } + if current == git_root { + break; + } + let Some(parent) = current.parent() else { + break; + }; + current = parent.to_path_buf(); + } + + Err(ResolveScriptsError::DoesNotExist("SCRIPTS file not found")) +} + pub fn read_scripts(path: &Path) -> Result { if !path.exists() { return Err(ResolveScriptsError::DoesNotExist( @@ -95,20 +129,11 @@ fn split_explicit_target(target: &str) -> std::result::Result bool { - target.contains('/') - || target == "." - || target == ".." - || target.starts_with("./") - || target.starts_with("../") - || Path::new(target).exists() -} - /// Split a CLI task target into the unit path and task name. pub fn parse_target(target: &str) -> AnyhowResult<(String, String)> { if let Some((path, task)) = split_explicit_target(target).map_err(|()| { anyhow::anyhow!( - "invalid target '{target}'. Missing task name after ':'. Use 'build' for the current unit or ':build' for another unit" + "invalid target '{target}'. Missing task name after ':'. Use 'build' for the nearest enclosing unit or ':build' for another unit" ) })? { return Ok(( @@ -117,12 +142,6 @@ pub fn parse_target(target: &str) -> AnyhowResult<(String, String)> { )); } - if looks_like_unit_path(target) { - bail!( - "invalid target '{target}'. Units must include a task name. Use ':' for another unit or '' for the current unit" - ); - } - Ok((".".to_string(), target.to_string())) } @@ -136,23 +155,27 @@ pub fn parse_dependency(dep: &str) -> Result<(String, String), ResolveScriptsErr return Ok((path.to_string(), task.to_string())); } - if looks_like_unit_path(dep) { - return Err(ResolveScriptsError::InvalidTarget(format!( - "invalid dependency '{dep}'. Use ':' for another unit or '' for the current unit" - ))); - } - Ok((String::new(), dep.to_string())) } -pub fn read_workspace_config(git_root: &Path) -> Option { +pub fn read_workspace_config(git_root: &Path) -> AnyhowResult> { let workspace_path = git_root.join("SCRIPTS_WORKSPACE.toml"); - if !workspace_path.is_file() { - return None; + if !workspace_path + .try_exists() + .with_context(|| format!("failed to inspect {}", workspace_path.display()))? + { + return Ok(None); } - let contents = read_to_string(workspace_path).ok()?; - toml::from_str(&contents).ok() + let contents = read_to_string(&workspace_path) + .with_context(|| format!("failed to read {}", workspace_path.display()))?; + let config = toml::from_str(&contents).with_context(|| { + format!( + "invalid workspace configuration in {}", + workspace_path.display() + ) + })?; + Ok(Some(config)) } #[cfg(test)] @@ -174,8 +197,11 @@ mod tests { } #[test] - fn parse_target_rejects_path_like_targets_without_task_names() { - assert!(parse_target("./tools/pkg").is_err()); - assert!(parse_target("..").is_err()); + fn parse_target_treats_path_like_names_as_current_unit_tasks() { + assert_eq!( + parse_target("./tools/pkg").unwrap(), + (".".into(), "./tools/pkg".into()) + ); + assert_eq!(parse_target("..").unwrap(), (".".into(), "..".into())); } } diff --git a/src/helpers/scripts_def.rs b/src/helpers/scripts_def.rs index 8a14427..019ef91 100644 --- a/src/helpers/scripts_def.rs +++ b/src/helpers/scripts_def.rs @@ -9,6 +9,7 @@ pub struct ScriptsDef { } #[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] pub struct Task { pub deps: Option>, pub command: Option, @@ -29,6 +30,7 @@ pub enum RelativeTo { } #[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] pub struct BinAppendObject { pub path: String, pub relative_to: RelativeTo, @@ -42,6 +44,7 @@ pub enum BinAppendEntry { } #[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] pub struct WorkspaceConfig { /// Paths that should be appended to PATH for all tasks. pub bin_append: Option>, diff --git a/src/helpers/task_list.rs b/src/helpers/task_list.rs index 9c0ad6c..76ed916 100644 --- a/src/helpers/task_list.rs +++ b/src/helpers/task_list.rs @@ -1,4 +1,7 @@ -use crate::helpers::{git::get_git_root, resolve::read_scripts}; +use crate::helpers::{ + git::get_git_root, + resolve::{find_enclosing_unit, read_scripts}, +}; pub fn print_tasks_for_current_unit() { let cwd = match std::env::current_dir() { @@ -10,24 +13,16 @@ pub fn print_tasks_for_current_unit() { Err(_) => cwd.clone(), }; - let mut current = cwd.as_path(); - while current.starts_with(&git_root) { - let scripts_path = current.join("SCRIPTS"); - if scripts_path.exists() { - if let Ok(def) = read_scripts(current) { - println!("\nTasks in {}:", current.display()); - let mut keys: Vec<_> = def.scripts.keys().collect(); - keys.sort(); - for key in keys { - println!(" :{key}"); - } - println!("\nTip: run `scripts run ` from this unit."); - } - break; - } - match current.parent() { - Some(parent) => current = parent, - None => break, + let Ok(unit) = find_enclosing_unit(&cwd, &git_root) else { + return; + }; + if let Ok(def) = read_scripts(&unit) { + println!("\nTasks in {}:", unit.display()); + let mut keys: Vec<_> = def.scripts.keys().collect(); + keys.sort(); + for key in keys { + println!(" :{key}"); } + println!("\nTip: run `scripts run ` from this unit."); } } diff --git a/src/main.rs b/src/main.rs index e57be5f..1622b9d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ #[cfg(not(unix))] compile_error!("scripts currently supports Unix-like environments (macOS and Linux) only."); -use std::io; +use std::{io, num::NonZeroUsize}; use clap::{CommandFactory, Parser}; use clap_complete::{generate, Shell}; @@ -12,9 +12,9 @@ use crate::helpers::task_list::print_tasks_for_current_unit; mod commands; mod helpers; -const ROOT_AFTER_HELP: &str = "Examples:\n scripts run app:build\n scripts run build\n scripts run :build --watch\n scripts print-tree app:test --json\n scripts env dev\n scripts completions bash > ~/.local/share/bash-completion/completions/scripts\n\nTarget syntax:\n : Run a specific task in another unit\n Run a task in the current unit\n : Also run a task in the current unit"; +const ROOT_AFTER_HELP: &str = "Examples:\n scripts run app:build\n scripts run build\n scripts run :build --watch\n scripts print-tree app:test --json\n scripts env dev\n scripts completions bash > ~/.local/share/bash-completion/completions/scripts\n\nTarget syntax:\n : Run a specific task in another unit\n Run a task in the nearest enclosing unit\n : Also run a task in the nearest enclosing unit"; -const RUN_AFTER_HELP: &str = "Examples:\n scripts run app:build\n scripts run build\n scripts run :build --watch\n scripts run dev -- echo done\n scripts run --force tools/pkg:build\n scripts run --quiet app:build\n scripts run --verbose app:build"; +const RUN_AFTER_HELP: &str = "Examples:\n scripts run app:build\n scripts run build\n scripts run :build --watch\n scripts run dev -- echo done\n scripts run --jobs 4 app:build\n scripts run --force tools/pkg:build\n scripts run --quiet app:build\n scripts run --verbose app:build"; const ENV_AFTER_HELP: &str = "Examples:\n scripts env app:dev\n scripts env dev"; @@ -27,12 +27,12 @@ const COMPLETIONS_AFTER_HELP: &str = "Examples:\n scripts completions bash > ~/ #[derive(Debug, Parser)] #[command(name = "scripts")] #[command(version)] -#[command(about = "A pragmatic monorepo task runner with content-aware caching.")] +#[command(about = "A parallel monorepo task runner with content-aware caching.")] #[command(after_help = ROOT_AFTER_HELP)] enum Cli { /// Run a task and its dependencies. Run(RunArgs), - /// Remove the repository cache file. + /// Remove the repository cache directory. Clean(CleanArgs), /// Start a shell with PATH prepared for a task. Env(EnvArgs), @@ -48,7 +48,7 @@ enum Cli { #[command(after_help = RUN_AFTER_HELP)] struct RunArgs { #[arg(value_name = "TARGET")] - /// Task target. Use : for another unit or for the current unit. + /// Task target. Use : for another unit or for the nearest enclosing unit. target: String, /// Ignore cached results and run even if inputs are unchanged. #[arg(long)] @@ -62,13 +62,16 @@ struct RunArgs { /// Watch for changes and re-run the target graph. #[arg(long)] watch: bool, + /// Maximum number of tasks to run concurrently. Defaults to the logical CPU count. + #[arg(short = 'j', long, value_name = "N")] + jobs: Option, /// Append an inline shell fragment to the root task after `--`. #[arg(trailing_var_arg = true, value_name = "ARGS")] args: Vec, } #[derive(Debug, clap::Args)] -#[command(about = "Remove the repository cache file.")] +#[command(about = "Remove the repository cache directory.")] #[command(after_help = CLEAN_AFTER_HELP)] struct CleanArgs { #[arg(default_value = ".", value_name = "PATH")] @@ -81,7 +84,7 @@ struct CleanArgs { #[command(after_help = ENV_AFTER_HELP)] struct EnvArgs { #[arg(value_name = "TARGET")] - /// Task target. Use : for another unit or for the current unit. + /// Task target. Use : for another unit or for the nearest enclosing unit. target: String, } @@ -90,7 +93,7 @@ struct EnvArgs { #[command(after_help = PRINT_TREE_AFTER_HELP)] struct PrintTreeArgs { #[arg(value_name = "TARGET")] - /// Task target. Use : for another unit or for the current unit. + /// Task target. Use : for another unit or for the nearest enclosing unit. target: String, /// Emit JSON instead of human-readable text. #[arg(long)] @@ -132,6 +135,7 @@ fn main() { args.quiet, args.verbose, args.watch, + args.jobs, appended, ) } @@ -145,7 +149,7 @@ fn main() { } }; if let Err(e) = result { - eprintln!("error: {e}"); + eprintln!("error: {e:#}"); std::process::exit(1); } } diff --git a/tests/cli.rs b/tests/cli.rs index 00cc1c1..5a3a79f 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1,4 +1,10 @@ -use std::{fs, path::Path, process::Command}; +use std::{ + fs, + path::Path, + process::{Command, Stdio}, + thread, + time::{Duration, Instant}, +}; use assert_cmd::{assert::OutputAssertExt, cargo::CommandCargoExt}; use predicates::prelude::*; @@ -39,6 +45,23 @@ fn scripts_command(repo: &TempDir) -> Command { command } +fn wait_until(description: &str, timeout: Duration, mut condition: impl FnMut() -> bool) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if condition() { + return; + } + thread::sleep(Duration::from_millis(25)); + } + panic!("timed out waiting for {description}"); +} + +fn line_count(path: &Path) -> usize { + fs::read_to_string(path) + .map(|contents| contents.lines().count()) + .unwrap_or(0) +} + #[test] fn run_caches_tasks_and_reruns_when_watched_paths_change() { let repo = init_repo(); @@ -324,7 +347,7 @@ watch = [] } #[test] -fn clean_reports_when_the_cache_file_is_removed() { +fn clean_removes_legacy_cache_file() { let repo = init_repo(); write_file(repo.path(), ".scripts_cache", "{}\n"); @@ -335,6 +358,24 @@ fn clean_reports_when_the_cache_file_is_removed() { .stdout(predicate::str::contains("removed")); } +#[test] +fn clean_removes_cache_directory() { + let repo = init_repo(); + write_file(repo.path(), "SCRIPTS", "[build]\nwatch = []\n"); + scripts_command(&repo) + .args(["run", "build"]) + .assert() + .success(); + assert!(repo.path().join(".scripts_cache").is_dir()); + + scripts_command(&repo) + .args(["clean"]) + .assert() + .success() + .stdout(predicate::str::contains("removed")); + assert!(!repo.path().join(".scripts_cache").exists()); +} + #[test] fn failed_dependencies_report_skipped_dependents() { let repo = init_repo(); @@ -422,13 +463,17 @@ fn completions_command_generates_shell_script() { } #[test] -fn path_like_dependency_targets_require_explicit_task_names() { +fn path_like_dependency_names_are_current_unit_tasks() { let repo = init_repo(); write_file( repo.path(), "app/SCRIPTS", r#" +["tools/pkg"] +command = "printf 'path-like-task\n'" +watch = [] + [build] deps = ["tools/pkg"] watch = [] @@ -436,10 +481,10 @@ watch = [] ); scripts_command(&repo) - .args(["print-tree", "app:build"]) + .args(["run", "app:build"]) .assert() - .failure() - .stderr(predicate::str::contains("invalid dependency 'tools/pkg'")); + .success() + .stdout(predicate::str::contains("path-like-task")); } #[test] @@ -482,22 +527,24 @@ watch = [] } #[test] -fn existing_unit_paths_require_explicit_task_names() { +fn task_names_do_not_change_meaning_when_a_matching_directory_exists() { let repo = init_repo(); write_file( repo.path(), - "app/SCRIPTS", + "SCRIPTS", r#" -[build] +[app] +command = "printf 'task-app\n'" watch = [] "#, ); + fs::create_dir(repo.path().join("app")).expect("create matching directory"); scripts_command(&repo) .args(["run", "app"]) .assert() - .failure() - .stderr(predicate::str::contains("Units must include a task name")); + .success() + .stdout(predicate::str::contains("task-app")); } #[test] @@ -542,3 +589,401 @@ watch = [] .success() .stdout(predicate::str::contains("workspace-helper")); } + +#[test] +fn tasks_run_from_descendants_of_their_unit() { + let repo = init_repo(); + write_file( + repo.path(), + "unit/SCRIPTS", + r#" +[build] +command = "printf 'nested-unit\n'" +watch = [] +"#, + ); + fs::create_dir_all(repo.path().join("unit/src/nested")).expect("create nested directory"); + + scripts_command(&repo) + .current_dir(repo.path().join("unit/src/nested")) + .args(["run", "build"]) + .assert() + .success() + .stdout(predicate::str::contains("nested-unit")); +} + +#[test] +fn malformed_workspace_configuration_is_an_error() { + let repo = init_repo(); + write_file(repo.path(), "SCRIPTS", "[build]\nwatch = []\n"); + write_file( + repo.path(), + "SCRIPTS_WORKSPACE.toml", + "bin_append = \"not-an-array\"\n", + ); + + scripts_command(&repo) + .args(["run", "build"]) + .assert() + .failure() + .stderr(predicate::str::contains("invalid workspace configuration")); +} + +#[test] +fn unknown_workspace_fields_are_an_error() { + let repo = init_repo(); + write_file(repo.path(), "SCRIPTS", "[build]\nwatch = []\n"); + write_file( + repo.path(), + "SCRIPTS_WORKSPACE.toml", + "bin_apend = [\"tools/bin\"]\n", + ); + + scripts_command(&repo) + .args(["run", "build"]) + .assert() + .failure() + .stderr(predicate::str::contains("unknown field `bin_apend`")); +} + +#[test] +fn unknown_task_fields_are_an_error() { + let repo = init_repo(); + write_file( + repo.path(), + "SCRIPTS", + r#" +[build] +command = "printf should-not-run" +wath = [] +"#, + ); + + scripts_command(&repo) + .args(["run", "build"]) + .assert() + .failure() + .stderr(predicate::str::contains("unknown field `wath`")); +} + +#[test] +fn workspace_configuration_changes_invalidate_cached_tasks() { + let repo = init_repo(); + for (directory, output) in [("one", "one"), ("two", "two")] { + write_file( + repo.path(), + &format!("tools/{directory}/helper"), + &format!("#!/bin/sh\nprintf '{output}\\n'\n"), + ); + make_executable(&repo.path().join(format!("tools/{directory}/helper"))); + } + write_file( + repo.path(), + "SCRIPTS", + "[build]\ncommand = \"helper\"\nwatch = []\n", + ); + write_file( + repo.path(), + "SCRIPTS_WORKSPACE.toml", + "bin_append = [\"tools/one\"]\n", + ); + + scripts_command(&repo) + .args(["run", "build"]) + .assert() + .success() + .stdout(predicate::str::contains("one")); + + write_file( + repo.path(), + "SCRIPTS_WORKSPACE.toml", + "bin_append = [\"tools/two\"]\n", + ); + scripts_command(&repo) + .args(["run", "build"]) + .assert() + .success() + .stdout(predicate::str::contains("two")); +} + +#[test] +fn dependency_resolution_stays_inside_the_git_repository() { + let repo = init_repo(); + let parent = repo.path().parent().expect("repo parent"); + let outside = tempfile::tempdir_in(parent).expect("create sibling unit"); + write_file( + outside.path(), + "SCRIPTS", + "[build]\ncommand = \"printf escaped\"\n", + ); + let outside_name = outside + .path() + .file_name() + .expect("outside unit name") + .to_string_lossy(); + write_file( + repo.path(), + "app/SCRIPTS", + &format!("[build]\ndeps = [\"../{outside_name}:build\"]\n"), + ); + + scripts_command(&repo) + .args(["run", "app:build"]) + .assert() + .failure() + .stderr(predicate::str::contains( + "resolves outside the git repository", + )); +} + +#[test] +fn jobs_two_runs_independent_tasks_concurrently() { + let repo = init_repo(); + fs::create_dir(repo.path().join("state")).expect("create state directory"); + write_file( + repo.path(), + "SCRIPTS", + r#" +[a] +command = """ +touch state/a +attempt=0 +while [ ! -e state/b ] && [ "$attempt" -lt 100 ]; do + sleep 0.01 + attempt=$((attempt + 1)) +done +test -e state/b +""" + +[b] +command = """ +touch state/b +attempt=0 +while [ ! -e state/a ] && [ "$attempt" -lt 100 ]; do + sleep 0.01 + attempt=$((attempt + 1)) +done +test -e state/a +""" + +[build] +deps = [":a", ":b"] +"#, + ); + + scripts_command(&repo) + .args(["run", "--jobs", "2", "build"]) + .assert() + .success(); +} + +#[test] +fn jobs_one_never_overlaps_tasks() { + let repo = init_repo(); + write_file( + repo.path(), + "SCRIPTS", + r#" +[a] +command = "mkdir lock; sleep 0.1; rmdir lock" + +[b] +command = "mkdir lock; sleep 0.1; rmdir lock" + +[build] +deps = [":a", ":b"] +"#, + ); + + scripts_command(&repo) + .args(["run", "--jobs", "1", "build"]) + .assert() + .success(); +} + +#[test] +fn zero_jobs_is_rejected() { + let repo = init_repo(); + write_file(repo.path(), "SCRIPTS", "[build]\n"); + + scripts_command(&repo) + .args(["run", "--jobs", "0", "build"]) + .assert() + .failure() + .stderr(predicate::str::contains("invalid value '0'")); +} + +#[test] +fn independent_branches_continue_after_a_failure() { + let repo = init_repo(); + write_file( + repo.path(), + "SCRIPTS", + r#" +[fail] +command = "exit 7" + +[independent] +command = "printf ran > independent-ran" + +[build] +deps = [":fail", ":independent"] +"#, + ); + + scripts_command(&repo) + .args(["run", "--jobs", "1", "build"]) + .assert() + .failure() + .stderr(predicate::str::contains("FAIL").and(predicate::str::contains("SKIP"))); + assert!(repo.path().join("independent-ran").is_file()); +} + +#[test] +fn concurrent_runs_keep_each_tasks_cache_entry() { + let repo = init_repo(); + write_file( + repo.path(), + "SCRIPTS", + r#" +[a] +command = "sleep 0.1; printf 'a\n'" +watch = [] + +[b] +command = "sleep 0.2; printf 'b\n'" +watch = [] +"#, + ); + + let mut a = scripts_command(&repo) + .args(["run", "a"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start task a"); + let mut b = scripts_command(&repo) + .args(["run", "b"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start task b"); + assert!(a.wait().expect("wait for task a").success()); + assert!(b.wait().expect("wait for task b").success()); + + for task in ["a", "b"] { + scripts_command(&repo) + .args(["run", task]) + .assert() + .success() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains("CACHED")); + } +} + +#[test] +fn concurrent_success_cannot_restore_a_failed_tasks_cache_entry() { + let repo = init_repo(); + write_file( + repo.path(), + "SCRIPTS", + r#" +[a] +command = "test ! -e fail" +watch = [] + +[b] +command = "sleep 0.4" +watch = [] +"#, + ); + scripts_command(&repo).args(["run", "a"]).assert().success(); + write_file(repo.path(), "fail", ""); + + let mut b = scripts_command(&repo) + .args(["run", "--force", "b"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start task b"); + thread::sleep(Duration::from_millis(50)); + scripts_command(&repo) + .args(["run", "--force", "a"]) + .assert() + .failure(); + assert!(b.wait().expect("wait for task b").success()); + + scripts_command(&repo) + .args(["run", "a"]) + .assert() + .failure() + .stderr(predicate::str::contains("FAIL")); +} + +#[test] +fn watch_mode_adds_new_dependency_roots() { + let repo = init_repo(); + write_file(repo.path(), "app/input", "first\n"); + write_file( + repo.path(), + "app/SCRIPTS", + r#" +[build] +command = "printf 'app\n' >> ../app-runs" +watch = ["input", "SCRIPTS"] +"#, + ); + write_file(repo.path(), "dep/input", "first\n"); + write_file(repo.path(), "block-dependency", ""); + write_file( + repo.path(), + "dep/SCRIPTS", + r#" +[build] +command = "printf 'attempt\n' >> ../dep-attempts; test ! -e ../block-dependency; printf 'dep\n' >> ../dep-runs" +watch = ["input"] +"#, + ); + + let mut child = scripts_command(&repo) + .args(["run", "--watch", "app:build"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("start watch mode"); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + wait_until("initial app run", Duration::from_secs(5), || { + line_count(&repo.path().join("app-runs")) == 1 + }); + thread::sleep(Duration::from_millis(750)); + + write_file( + repo.path(), + "app/SCRIPTS", + r#" +[build] +deps = ["dep:build"] +command = "printf 'app\n' >> ../app-runs" +watch = ["input", "SCRIPTS"] +"#, + ); + wait_until("failed new dependency run", Duration::from_secs(5), || { + line_count(&repo.path().join("dep-attempts")) >= 1 + }); + thread::sleep(Duration::from_millis(750)); + + fs::remove_file(repo.path().join("block-dependency")).expect("unblock dependency"); + write_file(repo.path(), "dep/input", "second\n"); + wait_until("new dependency root change", Duration::from_secs(5), || { + line_count(&repo.path().join("dep-runs")) >= 1 + && line_count(&repo.path().join("app-runs")) >= 2 + }); + })); + + child.kill().expect("stop watch mode"); + child.wait().expect("wait for watch mode"); + if let Err(payload) = result { + std::panic::resume_unwind(payload); + } +}