ci.yaml

name: CI

on:
  # CI runs on every pull request regardless of target branch. This
  # makes the default safe for stacked PRs (PR-A → PR-B → PR-C where
  # each PR's base is the previous PR's branch) and any other workflow
  # that opens a PR against a non-`develop` branch — previously such
  # PRs were silently skipped because `branches: [develop]` filtered
  # them out, and the only fix was to hand-edit ci.yaml on each new
  # feature stack. Letting every PR trigger CI is cheap (the heavy
  # M3 Ultra jobs are still gated behind the `ci:full` label below)
  # and matches what most repos default to.
  #
  # `push: develop` is intentionally absent. Every commit reaching
  # `develop` is already covered by the open Release PR (`Release:
  # develop -> main`, created by auto-release-pr.yaml) — that PR's
  # `synchronize` event runs CI on the new HEAD, and because the
  # Release PR carries the `ci:full` label the heavy gate runs too.
  # Adding `on: push: branches: [develop]` would queue a second
  # workflow instance on the same SHA, doubling self-hosted-runner
  # load on a check the Release PR's `synchronize` already provides.
  # (Under the PR-number grouping in the concurrency block below the
  # two runs would land in DIFFERENT groups — push keyed by
  # `refs/heads/develop`, PR keyed by the Release PR's number — so
  # the block would not deduplicate them.)
  #
  # `ready_for_review` is added so the workflow fires the moment a
  # draft PR is marked ready — drafts themselves skip CI via the
  # `if:` guard on each job (saves self-hosted-runner time while
  # work is still in progress).
  #
  # `labeled` / `unlabeled` are added so toggling the `ci:full` label
  # triggers (or removes) the heavy self-hosted-runner gate on demand
  # — see the `test-and-coverage` job below.
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]

concurrency:
  # Group by PR number so a new push to the same PR cancels the
  # in-flight Heavy run on the outdated commit. The self-hosted
  # M3 Ultra runner pool is shared with every other open PR —
  # letting an obsolete 60-90-min run finish wastes a slot another
  # PR could use. Grouping by SHA (the previous approach) put every
  # commit in its own group, so `cancel-in-progress: true` never
  # fired and back-to-back pushes queued sequentially.
  # Falls back to `github.ref` for push/dispatch events (where there
  # is no `pull_request.number`), so e.g. a `workflow_dispatch` on
  # the same ref serializes too.
  #
  # Label events (`labeled` / `unlabeled`) get their own isolated
  # group keyed by `run_id`, so toggling a label on a PR does NOT
  # cancel an in-flight 60-90-min Heavy run on the same PR — most
  # label toggles are unrelated (`bug`, `priority/*`, …) and killing
  # the Heavy run for them would be a footgun. Trade-off: removing
  # `ci:full` mid-run does NOT auto-stop a Heavy run that is already
  # executing; cancel it manually with `gh run cancel` if you really
  # need to free an agent.
  group: >-
    ${{
      (github.event.action == 'labeled' || github.event.action == 'unlabeled')
        && format('ci-{0}-label-{1}', github.workflow, github.run_id)
        || format('ci-{0}-{1}', github.workflow, github.event.pull_request.number || github.ref)
    }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always

# Job topology — a two-tier test-gating model:
#
# * Tier 1 — `lint-and-build` — GitHub-hosted Linux, the DEFAULT.
#   Runs on every non-draft PR and every push with no label required.
#   Catches cross-platform compile bitrot and lint regressions
#   cheaply. Runs in PARALLEL with the heavy gate below — it does not
#   gate it via `needs:`. Each job carries its own draft/label `if:`
#   guard, so a lint failure does not block the heavy gate from
#   starting (deliberate: parallel feedback. Trade-off: on a lint
#   failure the m3-ultra runner time is spent regardless).
#
# * Tier 2 — `test-and-coverage` — the authoritative test + coverage
#   gate, opt-in via the `ci:full` label. Single heavy job (~60-90 min
#   on the shared self-hosted M3 Ultra runner pool). It runs the FULL
#   node + shared nextest suite under llvm-cov instrumentation: the Postgres
#   `db_tests`, the Plonky2-heavy mint/send/receive prover flows, and
#   the 100% line + function coverage gate, all in one binary run.
#   Gated behind `ci:full` so we don't burn runner time on every
#   speculative PR — apply the label when the PR is ready for the
#   authoritative gate. Both auto-promote PRs (staging -> develop and
#   develop -> main) get the label applied automatically by
#   auto-release-pr-staging.yaml / auto-release-pr.yaml.
#
# There is no third "subset" tier: a test either runs in the default
# Lint & Build (compile/lint) or comes in with `ci:full` (the full
# suite). The previous narrow per-area subset jobs (and their
# per-area opt-in labels) were removed — the heavy gate is a strict
# superset of everything they selected, so they added a maintenance
# burden (filter drift) without extending coverage.
#
#   Why test + coverage are merged into one job: the previous topology
#   had a `node-tests` job and a separate `coverage` job, both
#   running the SAME nextest suite (`coverage` simply wrapped nextest
#   in `cargo llvm-cov nextest`). That doubled wall-clock and m3-ultra
#   agent usage on every Release PR for no signal benefit — llvm-cov
#   under nextest produces both test execution AND coverage data in a
#   single binary run. Merging them keeps the 100% lines + functions
#   gate intact (same ignore regex, same `not binary(api_remote)`
#   exclusion) while running the heavy suite once per PR.
#
#   The documented hardware target is the M3 Ultra (CONTRIBUTING.md
#   § "Working on the Plonky2 Migration"). On `ubuntu-latest` the
#   full suite repeatedly hit the 75-min timeout (issue #30); on the
#   M3 Ultra it is ~60-90 min for a Rust change. Moving the gate
#   into CI rather than the developer's laptop unblocks the developer
#   on push (issue #40).
#
# Runner ops: see scripts/ci-runner/README.md.
jobs:
  lint-and-build:
    name: Lint & Build
    # Skip on draft PRs. The heavy `test-and-coverage` gate carries
    # the same draft/push guard plus the `ci:full` label check on its
    # own `if:`, so it runs in parallel with this job rather than
    # gating behind it via `needs:`.
    if: github.event_name == 'push' || github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Rust 1.81.0
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: "1.81.0"
          components: rustfmt, clippy

      - name: Cache cargo registry and build
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-

      - name: Check formatting
        run: cargo fmt --all --check

      - name: Run clippy (node + shared, MVP feature set)
        run: cargo clippy -p node -p shared -- -D warnings

      - name: Run clippy (node, all features)
        run: cargo clippy -p node --all-features -- -D warnings

      - name: Run clippy (program + prover libs)
        run: cargo clippy -p zkcoins-program-plonky2 -p zkcoins-prover-plonky2 --lib -- -D warnings

      # Issue #84: the chain-tip wait path and the publisher's
      # commit→reveal propagation wait must be event-driven (WS /
      # ZMQ), not polled. The grep below fails the build if a
      # `tokio::time::{sleep,sleep_until,interval}` or
      # `std::thread::sleep` call sneaks back into the scanner /
      # publisher modules without the documented opt-out marker. See
      # CONTRIBUTING.md § "No polling — events only" for the per-line
      # `scanner-polling-ok:` escape hatch and the rationale for each
      # currently-grandfathered occurrence. The marker is a plain
      # comment token (not an `#[allow(...)]` attribute) so future
      # contributors cannot mistake it for a real lint suppression
      # (issue #84 round-4 MINOR 4).
      - name: Forbid polling patterns in scanner/publisher
        run: |
          set -e
          FOUND=$(grep -rEn 'tokio::time::(sleep|sleep_until|interval)|std::thread::sleep' node/src/scanner.rs node/src/scanner_runtime.rs node/src/scanner_ws.rs node/src/scanner_ws_parse.rs node/src/publisher.rs 2>/dev/null | grep -v 'scanner-polling-ok:' || true)
          if [ -n "$FOUND" ]; then
            echo "::error::Polling pattern (tokio::time::sleep|sleep_until|interval or std::thread::sleep) detected in event-driven hot paths. See issue #84."
            echo "$FOUND"
            exit 1
          fi
          echo "Scanner/publisher polling check: OK"

      - name: Build node (MVP feature set — the DEV + PRD image)
        run: cargo build -p node

      - name: Build node (all features — self-host opt-in build)
        run: cargo build -p node --all-features

  test-and-coverage:
    name: Tests + Coverage Gate (M3 Ultra, 100% lines + functions)
    # Authoritative heavy gate: runs the full nextest suite under
    # llvm-cov instrumentation, producing both test execution AND
    # coverage data in a single binary run. Replaces the previous
    # `node-tests` + `coverage` pair (the two jobs ran the same
    # nextest suite — see the file header for the merge rationale).
    #
    # Gated behind the `ci:full` label so we don't burn runner time
    # on every speculative PR. Both auto-promote PRs get the label
    # applied automatically: staging -> develop by
    # auto-release-pr-staging.yaml and develop -> main by
    # auto-release-pr.yaml.
    if: >-
      (github.event_name == 'push' || github.event.pull_request.draft == false)
      && contains(github.event.pull_request.labels.*.name, 'ci:full')
    runs-on: [self-hosted, m3-ultra]
    timeout-minutes: 120
    env:
      # All three chain-shaping env vars are required by the node
      # bootstrap (see `lib::build_network_config_from_env`). CI uses
      # `127.0.0.1:1` endpoints so any test that exercises the commit
      # pipeline / scanner WS fails fast instead of reaching a public
      # third-party host (a previous Mutinynet-flavoured silent
      # fallback used to add >60 s per test).
      IS_MAINNET: "false"
      ESPLORA_URL: http://127.0.0.1:1/api
      ESPLORA_WS_URL: ws://127.0.0.1:1/api/v1/ws
      # `USERNAME_DOMAIN` is required by the node bootstrap (no
      # default — see node/src/main.rs and issue #95). The test value
      # is irrelevant for the `info_returns_*` assertions (they only
      # check non-empty + shape).
      USERNAME_DOMAIN: test.zkcoins.local
      # `PUBLISHER_KEY` is required on every network (no default —
      # see `node/src/lib.rs`); the value is a syntactically valid
      # 32-byte hex placeholder, NOT a secret. MUST match
      # `node/src/router_tests.rs` — the test mocks derive the
      # wiremock'd publisher address from this key. The previous
      # `1234567890abcdef…` fallback was a publicly-known test key that
      # drainer bots swept within minutes of any on-chain top-up; the
      # fallback was removed network-wide. The `0000…0001` value here
      # is chosen so a future grep for the burned `1234…` key returns
      # empty across the repo + CI config; it MUST NEVER be reused on
      # any chain that holds value.
      PUBLISHER_KEY: "0000000000000000000000000000000000000000000000000000000000000001"
      # The full suite includes the `db_tests`, which use the
      # `testcontainers` crate to spin up a real Postgres 17 per test
      # against the local Docker daemon. The self-hosted runner runs
      # Colima (not Docker Desktop), whose socket lives under the
      # runner user's home directory; `DOCKER_HOST` is set in a step
      # below so the Colima socket path resolves from `$HOME` at
      # runtime.
      # `sccache` wraps `rustc` and caches compiled crates across CI
      # runs. The M3 Ultra runner agents are self-hosted, so the cache
      # lives on local disk and survives between jobs.
      RUSTC_WRAPPER: sccache
      # Bump the cache cap above sccache's 10-GiB default. The cache is
      # user-level (~/Library/Caches/Mozilla.sccache) and shared by
      # every m3-ultra agent on the host; with 3+ parallel agents the
      # 10-GiB default thrashed (writes from one agent evicted hits
      # another had not consumed yet). 50 GiB fits the current working
      # set with room to grow; the host has >600 GiB free disk. The
      # server only reads SCCACHE_CACHE_SIZE at start, so the install
      # step below restarts it when the running cap differs.
      SCCACHE_CACHE_SIZE: "50G"
      # Activate the workspace's `coverage_nightly` cfg gate so the
      # `#[cfg_attr(coverage_nightly, coverage(off))]` annotations
      # (14× repo-wide, plus the platform-detection helpers in
      # `node/src/r2_probe.rs`) actually take effect under
      # `cargo llvm-cov`. cargo-llvm-cov does NOT auto-set this cfg —
      # without it every `coverage(off)` in the workspace is inert
      # and llvm-cov counts the excluded fns / lines as uncovered,
      # which silently broke the 100%-line + 100%-function gate the
      # moment the first annotation landed in the `node` crate. Set
      # only on this job: the `lint-and-build` job runs stable
      # 1.81.0 and would reject `feature(coverage_attribute)`, so the
      # cfg has no effect there.
      RUSTFLAGS: "--cfg coverage_nightly"
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Prepend ~/.cargo/bin to PATH (use rustup proxy, not Homebrew Rust)
        run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"

      # Point `testcontainers` at the Colima socket under the runner
      # user's home (see the `DOCKER_HOST` comment in the job env block
      # above). Set in a step so the path resolves from `$HOME` at
      # runtime instead of being hard-coded.
      - name: Set DOCKER_HOST for Colima socket
        run: echo "DOCKER_HOST=unix://$HOME/.colima/default/docker.sock" >> "$GITHUB_ENV"

      # `sccache` (compile cache) and `cargo-nextest` (test runner)
      # are installed once per runner via Homebrew. Idempotent: no-op
      # on a warm runner where both tools already exist. If a server is
      # already running with a different cap than the requested
      # SCCACHE_CACHE_SIZE, stop it so the next --start-server picks up
      # the new env value; the on-disk cache files survive the restart.
      - name: Ensure sccache + cargo-nextest are installed
        run: |
          command -v sccache >/dev/null || brew install sccache
          command -v cargo-nextest >/dev/null || brew install cargo-nextest
          if ! sccache --show-stats 2>/dev/null | grep -qE "Max cache size +50 GiB"; then
            sccache --stop-server >/dev/null 2>&1 || true
          fi
          sccache --start-server >/dev/null 2>&1 || true
          sccache --show-stats

      # The full suite's `db_tests` use testcontainers to spin up a
      # real Postgres 17 per test, so Docker (via Colima) must be
      # reachable on PATH. Fail fast with a readable error if it ever
      # goes away, instead of letting the suite die minutes into the
      # run with a hard-to-read bollard error.
      - name: Verify Docker is reachable (testcontainers dependency)
        run: docker info > /dev/null

      # `cargo llvm-cov nextest` is the nextest-aware coverage
      # subcommand: collects llvm-cov data while driving the suite
      # through nextest, so the 100% line/function gate and the test
      # execution share a single binary run. This is the merge of
      # the previous `node-tests` + `coverage` pair — the previous
      # `node-tests` job ran the same nextest invocation without the
      # `cargo llvm-cov` wrapper, which produced no extra signal.
      #
      # `-p node -p shared --all-features` matches the previous
      # `node-tests` test set exactly (the previous `coverage` job
      # was scoped to `-p node` because the coverage GATE is only
      # measured against the `node` crate; the merge keeps that gate
      # scope while widening the EXECUTED set to `-p node -p shared`
      # so the shared crate's `commitment::tests::*` keep running in
      # the heavy gate — they were part of `node-tests` before).
      # The `shared/src/commitment.rs` entry in --ignore-filename-regex
      # keeps the coverage gate strictness identical to the previous
      # `-p node`-scoped gate: the shared crate's source files are
      # excluded from the 100% measurement, only the `node` crate is
      # gated. `--all-features` likewise mirrors the previous
      # `node-tests` invocation so opt-in feature-gated code paths
      # still execute.
      #
      # The `api_remote` integration test (node/tests/api_remote.rs)
      # is excluded: it targets the live DEV node and belongs in
      # the post-deploy `api-e2e` job in deploy-dev.yaml, not the
      # hermetic gate. The MVP coverage scope is measured by the
      # rest of the suite, which covers the in-process axum handlers
      # via oneshot().
      - name: Run llvm-cov nextest (MVP scope, 100% line + function gate)
        # `--test-threads=8` (issue #181 Opt A): the heavy gate is
        # the largest wall consumer on M3 Ultra (~60-90 min at
        # --test-threads=1). 8 outer threads × Rayon-pinned cores
        # exploits the runner's 24 cores without over-subscribing —
        # Plonky2 prove tests already saturate Rayon internally.
        # Per-test schema isolation (#182) + cross-process file lock
        # around the shared container (`test_db::init_shared_pg`)
        # make the suite parallel-safe under llvm-cov.
        run: |
          cargo llvm-cov nextest --release -p node -p shared --all-features --show-missing-lines \
            --ignore-filename-regex 'main\.rs|lib\.rs|publisher\.rs|runtime\.rs|scanner_runtime\.rs|scanner_ws\.rs|flow\.rs|job_dispatcher\.rs|_tests\.rs$|test_db\.rs$|bin/.*\.rs$|shared/src/.*\.rs$' \
            --fail-under-lines 100 \
            --fail-under-functions 100 \
            --test-threads 8 \
            -E 'not binary(api_remote)'

      # On gate failure, re-format the existing llvm-cov data (no
      # re-run, no new test execution — `report` reads the on-disk
      # profraw / profdata produced by the previous step) and emit
      # the per-file "Uncovered Lines" block plus a json digest of
      # files below 100% line / function. `--show-missing-lines` on
      # the gate step sometimes elides this section depending on the
      # llvm-cov build (observed empirically across this repo's
      # llvm-cov upgrades), so this step makes the detail
      # deterministic: whenever the gate fails, the operator sees
      # which file/line/function is below 100% without having to
      # reproduce locally.
      - name: Show missing coverage on gate failure
        if: failure()
        run: |
          IGNORE='main\.rs|lib\.rs|publisher\.rs|runtime\.rs|scanner_runtime\.rs|scanner_ws\.rs|flow\.rs|job_dispatcher\.rs|_tests\.rs$|test_db\.rs$|bin/.*\.rs$|shared/src/.*\.rs$'

          echo "--- llvm-cov report: --show-missing-lines (text) ---"
          cargo llvm-cov report --release --show-missing-lines \
            --ignore-filename-regex "$IGNORE" || true

          echo "--- llvm-cov report: per-file json (filter < 100%) ---"
          cargo llvm-cov report --release --json \
            --ignore-filename-regex "$IGNORE" \
            | jq -r '.data[0].files[]
                | select(.summary.lines.percent < 100 or .summary.functions.percent < 100)
                | {filename, lines: .summary.lines, functions: .summary.functions}' \
            || echo "(jq not available or json parse failed)"

          # Per-function coverage list: emits one line per uncovered
          # function with file + name + line so the operator sees the
          # exact `pub fn foo at router.rs:1234` without having to
          # cross-reference the line ranges manually.
          echo "--- llvm-cov report: uncovered functions (per-symbol) ---"
          cargo llvm-cov report --release --json \
            --ignore-filename-regex "$IGNORE" \
            | jq -r '.data[0].functions[]
                | select(.count == 0)
                | "\(.filenames[0]):\(.regions[0][0])\t\(.name)"' \
            | sort -u || echo "(per-function extraction failed)"

          # Full HTML report — uploaded as an artifact below so the
          # operator can browse the per-line coverage in a browser
          # without re-running llvm-cov locally (heavy gate is
          # ~50 min on M3 Ultra).
          echo "--- llvm-cov report: generating HTML for artifact ---"
          cargo llvm-cov report --release --html \
            --output-dir target/llvm-cov-html \
            --ignore-filename-regex "$IGNORE" \
            || echo "(HTML generation failed)"

      - name: Upload coverage HTML report on gate failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: llvm-cov-html-${{ github.run_id }}-${{ github.run_attempt }}
          path: target/llvm-cov-html
          if-no-files-found: warn
          retention-days: 14

      # Tear down the shared test container created by
      # `test_db::setup_pool` via testcontainers' `ReuseDirective::
      # Always` (see `node/src/test_db.rs`). The reuse flag tells
      # testcontainers NOT to drop the container at process exit so
      # every `cargo nextest` test process can attach to the same
      # daemon-side container — but that means nobody removes it
      # either. Always-on cleanup so a stale container from one PR run
      # cannot bleed into the next on the same self-hosted runner.
      - name: Tear down shared test Postgres container
        if: always()
        run: docker rm -f zkcoins-test-shared-pg 2>/dev/null || true

      - name: sccache stats (post-build)
        if: always()
        run: sccache --show-stats

  # Telegram alert on workflow failure for the heavy gate. Modelled
  # as a separate job (not an inline step) so job-level failures —
  # timeout, OOM, runner crash — still fire the alert. `if: failure()`
  # evaluates against the whole `needs:` group: any listed job
  # transitioning to `failure` triggers it, while skipped jobs
  # (`test-and-coverage` on a non-ci:full PR, or all jobs on a draft
  # PR) and manual cancellation stay silent.
  notify-failure:
    name: Telegram alert on failure
    needs: [lint-and-build, test-and-coverage]
    if: failure()
    runs-on: ubuntu-latest
    steps:
      - name: Send Telegram alert
        env:
          TG_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
          TG_CHAT: ${{ secrets.TELEGRAM_CHAT_ID }}
        run: |
          TEXT=$'❌ <b>'"${{ github.workflow }}"$'</b> failed\n<b>Repo:</b> '"${{ github.repository }}"$'\n<b>Branch:</b> '"${{ github.ref_name }}"$'\n<b>Run:</b> '"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          curl -sS -X POST "https://api.telegram.org/bot${TG_TOKEN}/sendMessage" \
               --data-urlencode "chat_id=${TG_CHAT}" \
               --data-urlencode "text=${TEXT}" \
               -d "parse_mode=HTML" \
               -d "disable_web_page_preview=true"