Reach a frozen source through the cron, rather than around it

.github/workflows/adapter_backfill.yml

@@ -0,0 +1,71 @@
+# Carry one adapter's frozen source into the datastore, once.
+#
+# The nightly runs what the catalog marks runnable. An adapter whose upstream
+# is archived is deliberately excluded, which leaves whatever it once published
+# stranded at whichever schema was current then, with nothing to move it
+# forward. This is the manual counterpart: `cron run --backfill` reaches
+# exactly those adapters and refuses the ones the schedule owns.
+#
+# It goes through the cron rather than calling the adapter module directly, so
+# the run records fingerprints in the raw store. That ledger is what makes a
+# second dispatch a no-op instead of a second copy of every record — filenames
+# come from uuid4, so re-publishing appends rather than overwrites.
+
+name: Adapter backfill
+
+on:
+  workflow_dispatch:
+    inputs:
+      adapter:
+        description: 'Catalog key of the adapter to backfill'
+        required: true
+        type: string
+      dry_run:
+        description: 'Convert and validate without publishing'
+        required: false
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+jobs:
+  backfill:
+    name: ${{ inputs.adapter }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    environment: cron
+    # Two runs of one adapter would read the same ledger, and the second would
+    # overwrite the first's fingerprints.
+    concurrency:
+      group: adapter-backfill-${{ inputs.adapter }}
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@v6.0.2
+      - uses: astral-sh/setup-uv@v7.6.0
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/eee-venv
+        run: uv sync --locked
+
+      - name: Backfill
+        env:
+          UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/eee-venv
+          ADAPTER: ${{ inputs.adapter }}
+          DRY_RUN: ${{ inputs.dry_run }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          ARTIFICIAL_ANALYSIS_API_KEY: ${{ secrets.ARTIFICIAL_ANALYSIS_API_KEY }}
+          LLM_STATS_API_KEY: ${{ secrets.LLM_STATS_API_KEY }}
+          MERCOR_EVAL_API_EVALEVAL_KEY: ${{ secrets.MERCOR_EVAL_API_EVALEVAL_KEY }}
+          EEE_DATASTORE_REPO_ID: ${{ vars.EEE_DATASTORE_REPO_ID }}
+          EEE_RAW_REPO_ID: ${{ vars.EEE_RAW_REPO_ID }}
+        run: |
+          set -euo pipefail
+          args=(run --adapter "${ADAPTER}" --backfill --force-full)
+          if [ "${DRY_RUN}" = "true" ]; then
+              args+=(--dry-run)
+          fi
+          uv run --locked python -m every_eval_ever.cron "${args[@]}"

every_eval_ever/cron/__main__.py

@@ -445,12 +445,33 @@ def cmd_run(args: argparse.Namespace) -> int:
     except catalog.UnknownAdapterError as exc:
         print(str(exc), file=sys.stderr)
         return 1
-    if not spec.runnable:
+    if spec.runnable and args.backfill:
+        # A backfill exists to reach what the schedule cannot. Letting it run a
+        # scheduled adapter would publish outside the cadence the catalog sets,
+        # with none of the freshness checks that decide when the source is
+        # worth refetching.
+        print(
+            f'{spec.key} is runnable, so it does not need --backfill. Let the '
+            f'schedule run it, or pass --force-full to republish.',
+            file=sys.stderr,
+        )
+        return 1
+    if not spec.runnable and not args.backfill:
         print(
             f'{spec.key} is not schedulable: {spec.unrunnable_reason}',
             file=sys.stderr,
         )
+        print(
+            'Pass --backfill to run it anyway, which is how a frozen source '
+            'is carried forward once.',
+            file=sys.stderr,
+        )
         return 1
+    if args.backfill:
+        # Say what is being overridden, so the reason is in the run log rather
+        # than only in the catalog.
+        print(f'backfilling {spec.key}, which the catalog marks unrunnable: '
+              f'{spec.unrunnable_reason}')
 
     run_date = args.date or _today()
     run_url = args.run_url or _run_url()
@@ -769,6 +790,15 @@ def build_parser() -> argparse.ArgumentParser:
             'skip and the de-duplication ledger.'
         ),
     )
+    run_parser.add_argument(
+        '--backfill',
+        action='store_true',
+        help=(
+            'Run an adapter the catalog marks unrunnable. For carrying a '
+            'frozen source the schedule will never revisit; refused for an '
+            'adapter that is runnable, which belongs on the schedule.'
+        ),
+    )
     run_parser.add_argument(
         '--datastore-repo',
         default=(

tests/test_cron_cli.py

@@ -214,7 +214,38 @@ def test_an_unknown_adapter_is_rejected(capsys) -> None:
 
 def test_an_unschedulable_adapter_is_rejected(capsys) -> None:
     assert cli.main(['run', '--adapter', 'bfcl']) == 1
-    assert 'not schedulable' in capsys.readouterr().err
+    err = capsys.readouterr().err
+    assert 'not schedulable' in err
+    # The refusal has to name the way out, or the next person edits the catalog.
+    assert '--backfill' in err
+
+
+def test_backfill_reaches_an_adapter_the_schedule_never_will(
+    monkeypatch, capsys
+) -> None:
+    """A frozen source is exactly what the runnable flag excludes."""
+    class Reached(Exception):
+        pass
+
+    def reached(*args, **kwargs):
+        raise Reached
+
+    monkeypatch.setattr(cli.runner, 'run', reached)
+
+    # Reaching the adapter is the whole assertion; what it then produces is
+    # the runner's business and is covered where the runner is.
+    with pytest.raises(Reached):
+        cli.main(['run', '--adapter', 'bfcl', '--backfill', '--dry-run'])
+
+    assert 'backfilling bfcl' in capsys.readouterr().out
+
+
+def test_backfill_is_refused_for_an_adapter_the_schedule_owns(capsys) -> None:
+    """Publishing outside the cadence skips the freshness checks that set it."""
+    assert cli.main(['run', '--adapter', 'hal', '--backfill']) == 1
+    err = capsys.readouterr().err
+    assert 'does not need --backfill' in err
+    assert '--force-full' in err
 
 
 def test_a_public_raw_store_stops_the_run_before_the_adapter(