From 3b7668e7689434946f68fadc5604c60caaf459d8 Mon Sep 17 00:00:00 2001 From: leshem Date: Thu, 10 Sep 2026 12:20:07 -0400 Subject: [PATCH] Reach a frozen source through the cron, rather than around it An adapter whose upstream is archived is marked unrunnable, so the nightly skips it and whatever it once published stays at whichever schema was current then. Nothing in the repo could move those records forward, which is 4,574 of the 9,751 records stranded on schema 0.2.2 for hfopenllm_v2 alone. `cron run --backfill` reaches exactly those adapters. It refuses one the schedule owns, because publishing outside the cadence skips the freshness checks that set it, and points at --force-full instead. The refusal for an unrunnable adapter now names the way out, so the next person reaching for it does not edit the catalog to get past the gate. The workflow goes through the cron rather than calling an adapter module directly. That matters for a reason worth stating plainly: record filenames come from uuid4, so publishing outside the cron appends a second copy of every record instead of overwriting, which is how a schema migration doubled this datastore once already. Routing through the cron records fingerprints in the raw store, and the ledger makes a second dispatch a no-op. --- .github/workflows/adapter_backfill.yml | 71 ++++++++++++++++++++++++++ every_eval_ever/cron/__main__.py | 32 +++++++++++- tests/test_cron_cli.py | 33 +++++++++++- 3 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/adapter_backfill.yml diff --git a/.github/workflows/adapter_backfill.yml b/.github/workflows/adapter_backfill.yml new file mode 100644 index 000000000..b0eb13bd9 --- /dev/null +++ b/.github/workflows/adapter_backfill.yml @@ -0,0 +1,71 @@ +# Carry one adapter's frozen source into the datastore, once. +# +# The nightly runs what the catalog marks runnable. An adapter whose upstream +# is archived is deliberately excluded, which leaves whatever it once published +# stranded at whichever schema was current then, with nothing to move it +# forward. This is the manual counterpart: `cron run --backfill` reaches +# exactly those adapters and refuses the ones the schedule owns. +# +# It goes through the cron rather than calling the adapter module directly, so +# the run records fingerprints in the raw store. That ledger is what makes a +# second dispatch a no-op instead of a second copy of every record — filenames +# come from uuid4, so re-publishing appends rather than overwrites. + +name: Adapter backfill + +on: + workflow_dispatch: + inputs: + adapter: + description: 'Catalog key of the adapter to backfill' + required: true + type: string + dry_run: + description: 'Convert and validate without publishing' + required: false + type: boolean + default: true + +permissions: + contents: read + +jobs: + backfill: + name: ${{ inputs.adapter }} + runs-on: ubuntu-latest + timeout-minutes: 90 + environment: cron + # Two runs of one adapter would read the same ledger, and the second would + # overwrite the first's fingerprints. + concurrency: + group: adapter-backfill-${{ inputs.adapter }} + cancel-in-progress: false + steps: + - uses: actions/checkout@v6.0.2 + - uses: astral-sh/setup-uv@v7.6.0 + with: + python-version: '3.12' + + - name: Install dependencies + env: + UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/eee-venv + run: uv sync --locked + + - name: Backfill + env: + UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/eee-venv + ADAPTER: ${{ inputs.adapter }} + DRY_RUN: ${{ inputs.dry_run }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + ARTIFICIAL_ANALYSIS_API_KEY: ${{ secrets.ARTIFICIAL_ANALYSIS_API_KEY }} + LLM_STATS_API_KEY: ${{ secrets.LLM_STATS_API_KEY }} + MERCOR_EVAL_API_EVALEVAL_KEY: ${{ secrets.MERCOR_EVAL_API_EVALEVAL_KEY }} + EEE_DATASTORE_REPO_ID: ${{ vars.EEE_DATASTORE_REPO_ID }} + EEE_RAW_REPO_ID: ${{ vars.EEE_RAW_REPO_ID }} + run: | + set -euo pipefail + args=(run --adapter "${ADAPTER}" --backfill --force-full) + if [ "${DRY_RUN}" = "true" ]; then + args+=(--dry-run) + fi + uv run --locked python -m every_eval_ever.cron "${args[@]}" diff --git a/every_eval_ever/cron/__main__.py b/every_eval_ever/cron/__main__.py index d3ea73092..60aa48760 100644 --- a/every_eval_ever/cron/__main__.py +++ b/every_eval_ever/cron/__main__.py @@ -445,12 +445,33 @@ def cmd_run(args: argparse.Namespace) -> int: except catalog.UnknownAdapterError as exc: print(str(exc), file=sys.stderr) return 1 - if not spec.runnable: + if spec.runnable and args.backfill: + # A backfill exists to reach what the schedule cannot. Letting it run a + # scheduled adapter would publish outside the cadence the catalog sets, + # with none of the freshness checks that decide when the source is + # worth refetching. + print( + f'{spec.key} is runnable, so it does not need --backfill. Let the ' + f'schedule run it, or pass --force-full to republish.', + file=sys.stderr, + ) + return 1 + if not spec.runnable and not args.backfill: print( f'{spec.key} is not schedulable: {spec.unrunnable_reason}', file=sys.stderr, ) + print( + 'Pass --backfill to run it anyway, which is how a frozen source ' + 'is carried forward once.', + file=sys.stderr, + ) return 1 + if args.backfill: + # Say what is being overridden, so the reason is in the run log rather + # than only in the catalog. + print(f'backfilling {spec.key}, which the catalog marks unrunnable: ' + f'{spec.unrunnable_reason}') run_date = args.date or _today() run_url = args.run_url or _run_url() @@ -769,6 +790,15 @@ def build_parser() -> argparse.ArgumentParser: 'skip and the de-duplication ledger.' ), ) + run_parser.add_argument( + '--backfill', + action='store_true', + help=( + 'Run an adapter the catalog marks unrunnable. For carrying a ' + 'frozen source the schedule will never revisit; refused for an ' + 'adapter that is runnable, which belongs on the schedule.' + ), + ) run_parser.add_argument( '--datastore-repo', default=( diff --git a/tests/test_cron_cli.py b/tests/test_cron_cli.py index c23e1a7b8..acd0bd490 100644 --- a/tests/test_cron_cli.py +++ b/tests/test_cron_cli.py @@ -214,7 +214,38 @@ def test_an_unknown_adapter_is_rejected(capsys) -> None: def test_an_unschedulable_adapter_is_rejected(capsys) -> None: assert cli.main(['run', '--adapter', 'bfcl']) == 1 - assert 'not schedulable' in capsys.readouterr().err + err = capsys.readouterr().err + assert 'not schedulable' in err + # The refusal has to name the way out, or the next person edits the catalog. + assert '--backfill' in err + + +def test_backfill_reaches_an_adapter_the_schedule_never_will( + monkeypatch, capsys +) -> None: + """A frozen source is exactly what the runnable flag excludes.""" + class Reached(Exception): + pass + + def reached(*args, **kwargs): + raise Reached + + monkeypatch.setattr(cli.runner, 'run', reached) + + # Reaching the adapter is the whole assertion; what it then produces is + # the runner's business and is covered where the runner is. + with pytest.raises(Reached): + cli.main(['run', '--adapter', 'bfcl', '--backfill', '--dry-run']) + + assert 'backfilling bfcl' in capsys.readouterr().out + + +def test_backfill_is_refused_for_an_adapter_the_schedule_owns(capsys) -> None: + """Publishing outside the cadence skips the freshness checks that set it.""" + assert cli.main(['run', '--adapter', 'hal', '--backfill']) == 1 + err = capsys.readouterr().err + assert 'does not need --backfill' in err + assert '--force-full' in err def test_a_public_raw_store_stops_the_run_before_the_adapter(