diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 733cd0a80..720e10a23 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -7,7 +7,7 @@ body: value: | Use this for defects in **this** repo (gateway, validator, challenges, deploy). Security issues: do **not** file here — see [SECURITY.md](../SECURITY.md). - Miner-facing public repos are separate (design-challenge / prism). + Miner-facing public repo is [CortexLM/relearn](https://github.com/CortexLM/relearn). - type: textarea id: summary attributes: @@ -22,8 +22,8 @@ body: options: - gateway - validator - - design-challenge - - prism-challenge + - relearn-challenge + - bounty-challenge - deploy / compose - docs - other diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 1e813dbe9..ee75fd22e 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,7 +2,7 @@ blank_issues_enabled: true contact_links: - name: Miner submit docs url: https://github.com/CortexLM/cortex/blob/main/docs/external-miner/README.md - about: HTTP submit guides live in docs/external-miner (and the public design-challenge / prism repos). + about: HTTP submit guides live in docs/external-miner and the public CortexLM/relearn repo. - name: Security report url: https://github.com/CortexLM/cortex/security/advisories/new about: Private vulnerability reporting — do not file a public issue. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1930e0d3..5aff9863f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,41 +64,6 @@ jobs: - name: compose matrix assertions run: bash deploy/scripts/assert-compose-matrix.sh - # Prism v3 harness smokes: fixture-size, G4 tuple concat, and torch-seed - # overflow class bugs shipped once because CI never ran the harness - # end-to-end. CPU wheels keep the runner cheap; both smokes must pass - # (they assert all 8 battery groups ok + the flat org.* / mirrors blob - # the Rust composite ingests). - harness-smoke: - name: prism harness smokes (cpu torch) - runs-on: ubuntu-latest - timeout-minutes: 25 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: pip - cache-dependency-path: crates/prism-recipe/harness - - - name: Install cpu torch + transformers + pyarrow - run: | - python -m pip install --upgrade pip - pip install torch --index-url https://download.pytorch.org/whl/cpu - pip install transformers pyarrow - - - name: python -m compileall - run: python -m compileall crates/prism-recipe/harness - - - name: smoke_local (v1 contract flow) - run: python crates/prism-recipe/harness/tests/smoke_local.py - - - name: smoke_battery (v3 G1-G8 battery + rollup contract) - run: python crates/prism-recipe/harness/tests/smoke_battery.py - # --------------------------------------------------------------------------- # Auto-deploy staging. Lives here rather than in deploy-staging.yml so that # `needs: ci` keeps the CI-green ordering on every push to main. @@ -106,7 +71,7 @@ jobs: # --------------------------------------------------------------------------- deploy-staging: name: deploy staging ${{ matrix.role }} - needs: [ci, harness-smoke] + needs: [ci] if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 120 diff --git a/.github/workflows/ghcr-public.yml b/.github/workflows/ghcr-public.yml index f81157763..ceef29226 100644 --- a/.github/workflows/ghcr-public.yml +++ b/.github/workflows/ghcr-public.yml @@ -27,10 +27,10 @@ jobs: "base/gateway" "base/validator" "base/updater" - "base/prism-challenge" - "base/design-challenge" - "base/design-egress-proxy" - "base/design-runtime" + "base/relearn-challenge" + "base/relearn-t2i-challenge" + "base/relearn-mm-challenge" + "base/bounty-challenge" ) ok=0 # First list packages to Learn exact names (debug) diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 3441e2f90..716033043 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -70,20 +70,15 @@ jobs: - target: updater image_suffix: updater dockerfile: deploy/Dockerfile - - target: prism-challenge - image_suffix: prism-challenge + - target: relearn-challenge + image_suffix: relearn-challenge + - target: relearn-t2i-challenge + image_suffix: relearn-t2i-challenge + - target: relearn-mm-challenge + image_suffix: relearn-mm-challenge dockerfile: deploy/Dockerfile - - target: design-challenge - image_suffix: design-challenge - dockerfile: deploy/Dockerfile - - target: design-egress-proxy - image_suffix: design-egress-proxy - dockerfile: deploy/Dockerfile - - target: design-runtime - image_suffix: design-runtime - dockerfile: deploy/Dockerfile - - target: design-review - image_suffix: design-review + - target: bounty-challenge + image_suffix: bounty-challenge dockerfile: deploy/Dockerfile - target: "" image_suffix: base-attest-helper @@ -285,11 +280,10 @@ jobs: "validator", "gateway", "updater", - "prism-challenge", - "design-challenge", - "design-egress-proxy", - "design-runtime", - "design-review", + "relearn-challenge", + "relearn-t2i-challenge", + "relearn-mm-challenge", + "bounty-challenge", "base-attest-helper", } assert set(images) == expected, (set(images), expected) @@ -357,10 +351,10 @@ jobs: cp "$SRC" "deploy/digests/${SHA}.json" echo "Recorded digest manifest: deploy/digests/${SHA}.json" chmod +x deploy/scripts/promote.sh - # Pin services only (validator/gateway/updater/prism-challenge/design-challenge). + # Pin services only (validator/gateway/updater/the four challenges). # --skip-backup: this job records CI digests; Spaces/PG backup runs at # prod promote (deploy-prod.yml, fail-closed). - for svc in validator gateway updater prism-challenge design-challenge; do + for svc in validator gateway updater relearn-challenge relearn-t2i-challenge relearn-mm-challenge bounty-challenge; do IMAGE=$(python3 -c ' import json, sys d = json.load(open(sys.argv[1])) @@ -379,7 +373,7 @@ jobs: import json, sys p = json.load(open("deploy/pins/staging.json")) assert p["commit_sha"] == sys.argv[1], (p["commit_sha"], sys.argv[1]) - for s in ("validator", "gateway", "updater", "prism-challenge", "design-challenge"): + for s in ("validator", "gateway", "updater", "relearn-challenge", "relearn-t2i-challenge", "relearn-mm-challenge", "bounty-challenge"): assert s in p["services"], s assert p["services"][s]["digest"].startswith("sha256:") print("staging pins ok", p["commit_sha"]) @@ -426,12 +420,10 @@ jobs: "base/gateway" "base/validator" "base/updater" - "base/prism-challenge" - "base/design-challenge" - "base/design-egress-proxy" - "base/design-runtime" - "base/design-review" - "base/prism-pod" + "base/relearn-challenge" + "base/relearn-t2i-challenge" + "base/relearn-mm-challenge" + "base/bounty-challenge" ) ok=0 # First list packages to Learn exact names (debug) diff --git a/AGENTS.md b/AGENTS.md index 92e4c49b2..b98cb6851 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ Short contract for agents and operators. Prefer linking over restating runbooks. -**Product:** Cortex ([`CortexLM/cortex`](https://github.com/CortexLM/cortex)) — Bittensor subnet control plane for decentralized collaborative AI research via multiple challenges. Naming split (Cortex vs leftover `base` / `BASE_*`): [`docs/NAMING.md`](docs/NAMING.md). +**Product:** Cortex ([`CortexLM/cortex`](https://github.com/CortexLM/cortex)) — Bittensor subnet control plane for decentralized collaborative AI research via the **Relearn LLM**, **Relearn T2I**, **Relearn Multimodal**, and **Bounty** challenges. Relearn eval lives in [`CortexLM/relearn`](https://github.com/CortexLM/relearn). Naming split (Cortex vs leftover `base` / `BASE_*`): [`docs/NAMING.md`](docs/NAMING.md). ## Monorepo map @@ -34,7 +34,10 @@ Working branch: **`main`**. Prod ships from annotated tags `v*.*.*` cut on `main |-----|-----|------------| | `gateway_sk` | Gateway | Bundle **seal** signatures (`POST /v1/admin/seal`) | | `gateway_admin_token` | Gateway + seal scripts | Bearer for **`/v1/admin/*`** (seal, backends, attest-grant). **Required** when `BASE_GATEWAY_REQUIRE_OWNER=1` | -| `prism_sk` / `design_sk` | Challenge / smoke | Signed leaves (`POST /v1/weights/raw`); pubs must match trust root | +| `relearn_sk` | Relearn LLM / smoke | Signed leaves (`POST /v1/weights/raw`); pub must match trust root | +| `relearn_t2i_sk` | Relearn T2I | Signed T2I leaves; pub must match trust root | +| `relearn_mm_sk` | Relearn Multimodal | Signed multimodal leaves; pub must match trust root | +| `bounty_sk` | Bounty / smoke | Signed bounty leaves; pub must match trust root | | Gateway owner wallet + `BASE_GATEWAY_REQUIRE_OWNER` | Gateway | Master-only **identity** check (live/prod). **Not** required to seal or serve `/v1/weights/latest` | | Validator wallet | Validator | On-chain weight **submit** only — validators *fetch* sealed weights; they do not need a gateway wallet | @@ -46,10 +49,12 @@ Each live challenge has a **separate public GitHub repo** for miners. Those repo | Challenge | Public repo | Role | |-----------|-------------|------| -| Design | [`BaseIntelligence/design-challenge`](https://github.com/BaseIntelligence/design-challenge) | Miner docs + baseline harness | -| Prism | [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) | Miner docs + recipe examples (publish / keep in sync; no control-plane code) | +| Relearn LLM | [`CortexLM/relearn`](https://github.com/CortexLM/relearn) | Eval image, harness, generators, teacher, miner docs | +| Relearn T2I | [`CortexLM/relearn`](https://github.com/CortexLM/relearn) | Cosmos3 fine-tune harness + Q-Judger runner; in-repo pointer [`docs/external-miner/relearn-t2i.md`](docs/external-miner/relearn-t2i.md) | +| Relearn Multimodal | [`CortexLM/relearn`](https://github.com/CortexLM/relearn) | Encoder + projector harness; in-repo pointer [`docs/external-miner/relearn-mm.md`](docs/external-miner/relearn-mm.md) | +| Bounty | this repo [`docs/external-miner/bounty.md`](docs/external-miner/bounty.md) | Miner pairing + report path; subnet **reads** CortexLM/backend public API (does not serve one) | -Those public URLs are historical org names; this control-plane repo is `CortexLM/cortex`. Monorepo mirror for CI and operators: [`docs/external-miner/`](docs/external-miner/). Frozen contracts stay in this repo (`docs/DESIGN_CHALLENGE.md`, `docs/PRISM.md`, …). +This control-plane repo is `CortexLM/cortex`. Short miner pointers: [`docs/external-miner/relearn.md`](docs/external-miner/relearn.md), [`docs/external-miner/relearn-t2i.md`](docs/external-miner/relearn-t2i.md), [`docs/external-miner/relearn-mm.md`](docs/external-miner/relearn-mm.md), [`docs/external-miner/bounty.md`](docs/external-miner/bounty.md). Historical frozen specs (`docs/DESIGN_CHALLENGE.md`, `docs/PRISM.md`) stay archived; they are not live products. Do not send miners to Design or Prism docs. **When a challenge product or public API changes**, agents **must** update: @@ -64,12 +69,16 @@ When verifying a challenge (local-e2e, staging, or focused tests), **simulate a 1. Happy-path harness / intake POST (or equivalent) through the challenge service on master. 2. Edge / failure probes: bad harness, sanitize reject, quota, wrong routes/auth. -3. **Design — baseline:** submit the reference agent at [`docs/external-miner/examples/design-baseline/`](docs/external-miner/examples/design-baseline/) (`agent.py` + `pyproject.toml`). After `POST /v1/harness`, poll `GET /v1/runs/{id}` + `/events` + `/logs?since=` until `awaiting_admin` / terminal; assert `GET /v1/runs/{id}/pages` lists `index.html`, `pricing.html`, `components.html` and `GET /v1/view/{run_id}/{page}` returns **200**; probe `GET /v1/stats` and `GET /v1/dashboard`. -4. **Design — cheat:** submit a malicious/copy harness; expect agentic `cheat`/`suspicious` → `Score(0)` (not admin-eligible). Poll events/logs the same way. -5. **Design — admin winners:** with operator bearer (`deploy/secrets/design/annotator_tokens`), `GET /v1/admin/rounds/{id}/candidates` then `POST /v1/admin/rounds/{id}/winners` with 1 or 2 clean harness ids (`SCORE_MAX` or `SCORE_MAX/2`). +3. **Relearn LLM — submit:** `POST /v1/submissions` with a 64-hex hotkey + artifact digest (optional `X-Lium-Api-Key`). Poll `GET /v1/submissions/{id}` until `awaiting_admin` or `rejected`. Holdout must stay sealed until the digest freezes. A regression must not become champion. +4. **Relearn LLM — promote:** with operator bearer (`deploy/secrets/relearn/admin_tokens`), `POST /v1/admin/promote` only for an eligible paired win. +5. **Relearn T2I — submit:** `POST /v1/submissions` with a manifest naming the pinned Cosmos3 base and OpenMDW 1.1. A Flux-family base must be a `400`, not a low score. `GET /v1/prompts` must publish the public split's frozen strings **and** seeds, and must never leak a holdout id. Probe contamination (declare a scored prompt id) and a pillar collapse; both must reject. +6. **Relearn Multimodal — submit:** `POST /v1/submissions` with an OSI-permissive encoder license (OpenRAIL must be a `400`). An `encoder_only` submission whose `lm_weights_hash` differs from `/v1/status`'s `champion_lm_weights_hash` must reject. A text regression must yield lattice `0`, not a reduced score. +7. **Bounty — pair + report:** `cortex-bounty pair --hotkey --account-id `, then `POST /v1/pair` (terms + signature) and `POST /v1/reports`. Operator bearer `POST /v1/admin/adjudicate` (`valid` / `already_fixed_not_prod` / `invalid_malicious` / `duplicate`). Scoring **reads** CortexLM/backend public JSON (`BOUNTY_BACKEND_PUBLIC_URL`); do not serve `/v1/public/*` from this repo. 6. Leaf emission → `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with **`sealed: true`** (burn fallback alone is not a real seal). -**Never host Sim in staging/prod** — Docker sandbox only there. `SimSandbox` / `BASE_ALLOW_HOST_SIM=1` is CI/local opt-in only; do **not** treat stub pages (`sim-install-ok` / `sim-run-ok` without executing `agent.py`) as proof. Prefer `DESIGN_FORCE_SIM=false` + OpenRouter when `deploy/secrets/openrouter/api_key` is present. +**Never host Sim in staging/prod** for live scoring. `RELEARN_FORCE_SIM=1`, `RELEARN_T2I_FORCE_SIM=1`, and `RELEARN_MM_FORCE_SIM=1` are CI/local opt-in only (`deploy/scripts/assert-compose-matrix.sh` fails if a droplet overlay sets one). Live rent requires a digest pin in the matching `config/relearn*-pin.toml` plus miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`). Never log or commit that key. + +**Relearn T2I product rules (do not weaken):** the generator seed is `nvidia/Cosmos3-Super-Text2Image` under OpenMDW 1.1; Flux-family bases are refused; Q-Judger (`Qwen/Qwen-Image-Bench`) is the only judge and its card-fixed inference parameters are part of the contract; eval prompts are frozen in the pin so no miner brings its own upsampler to the scored split; the holdout lives in git only as a commitment. **Relearn Multimodal:** miner encoders must be Apache-2.0 / MIT / BSD / ISC, and a text-holdout regression is a hard zero rather than a reduced score. Local smoke automates the weights seal step via `weights-smoke` inside `./deploy/scripts/local-e2e.sh --smoke` (see [`deploy/AGENTS.md`](deploy/AGENTS.md) and [`docs/runbooks/local-testnet-e2e.md`](docs/runbooks/local-testnet-e2e.md)). @@ -113,7 +122,8 @@ Match CI (`.github/workflows/ci.yml`): | Doc authority vs evidence | [`docs/AGENTS.md`](docs/AGENTS.md) | | Component status | [`docs/COMPLETENESS.md`](docs/COMPLETENESS.md) | | Frozen contracts | [`docs/BUNDLE_SPEC.md`](docs/BUNDLE_SPEC.md), [`docs/DESIGN_CHALLENGE.md`](docs/DESIGN_CHALLENGE.md), [`docs/PRISM.md`](docs/PRISM.md) | -| Miner HTTP submit | [`docs/external-miner/`](docs/external-miner/) · public: [design-challenge](https://github.com/BaseIntelligence/design-challenge), [prism](https://github.com/BaseIntelligence/prism) | +| Relearn miners | [`docs/external-miner/relearn.md`](docs/external-miner/relearn.md), [`relearn-t2i.md`](docs/external-miner/relearn-t2i.md), [`relearn-mm.md`](docs/external-miner/relearn-mm.md) · long guide: [CortexLM/relearn](https://github.com/CortexLM/relearn) | +| Validators | [`docs/external-miner/validators.md`](docs/external-miner/validators.md) | | Threat / operator checklist | [`docs/THREAT_MODEL.md`](docs/THREAT_MODEL.md), [`docs/OPERATOR_SECURITY.md`](docs/OPERATOR_SECURITY.md) | ## Do not commit diff --git a/Cargo.lock b/Cargo.lock index 61d107084..8c27b2c6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -606,6 +606,84 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "bounty-challenge" +version = "0.1.0" +dependencies = [ + "bounty-challenge-task", + "bounty-http", + "bounty-score", + "bounty-store", + "bundle", + "challenge-common", + "hex", + "reqwest 0.12.28", + "thiserror 2.0.19", + "tokio", +] + +[[package]] +name = "bounty-challenge-bin" +version = "0.1.0" +dependencies = [ + "axum", + "bounty-challenge", + "challenge-keys", + "clap", + "telemetry", + "tokio", + "tracing", +] + +[[package]] +name = "bounty-challenge-task" +version = "0.1.0" +dependencies = [ + "hex", + "keystore", + "schnorrkel", + "serde", + "thiserror 2.0.19", +] + +[[package]] +name = "bounty-http" +version = "0.1.0" +dependencies = [ + "axum", + "bounty-challenge-task", + "bounty-score", + "bounty-store", + "hex", + "http-body-util", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "tower", +] + +[[package]] +name = "bounty-score" +version = "0.1.0" +dependencies = [ + "bounty-challenge-task", + "serde", + "serde_json", +] + +[[package]] +name = "bounty-store" +version = "0.1.0" +dependencies = [ + "bounty-challenge-task", + "bounty-score", + "hex", + "serde", + "sha2 0.10.9", + "thiserror 2.0.19", +] + [[package]] name = "bstr" version = "1.13.0" @@ -1010,6 +1088,16 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cortex-bounty" +version = "0.1.0" +dependencies = [ + "bounty-challenge-task", + "clap", + "hex", + "keystore", +] + [[package]] name = "cpubits" version = "0.1.1" @@ -4511,6 +4599,321 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "relearn-challenge" +version = "0.1.0" +dependencies = [ + "bundle", + "challenge-common", + "crypto", + "hex", + "relearn-challenge-task", + "relearn-eval", + "relearn-http", + "relearn-score", + "relearn-store", + "serde", + "serde_json", + "thiserror 2.0.19", +] + +[[package]] +name = "relearn-challenge-bin" +version = "0.1.0" +dependencies = [ + "axum", + "challenge-keys", + "clap", + "relearn-challenge", + "relearn-eval", + "relearn-store", + "telemetry", + "tokio", + "tracing", +] + +[[package]] +name = "relearn-challenge-task" +version = "0.1.0" +dependencies = [ + "serde", +] + +[[package]] +name = "relearn-eval" +version = "0.1.0" +dependencies = [ + "async-trait", + "hex", + "prism-competition", + "prism-lium", + "prism-lium-types", + "relearn-challenge-task", + "relearn-score", + "relearn-store", + "reqwest 0.12.28", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", + "tokio", +] + +[[package]] +name = "relearn-http" +version = "0.1.0" +dependencies = [ + "axum", + "hex", + "http-body-util", + "relearn-challenge-task", + "relearn-eval", + "relearn-score", + "relearn-store", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "tower", +] + +[[package]] +name = "relearn-mm-challenge" +version = "0.1.0" +dependencies = [ + "bundle", + "challenge-common", + "crypto", + "hex", + "relearn-mm-eval", + "relearn-mm-http", + "relearn-mm-score", + "relearn-mm-store", + "relearn-mm-task", + "serde", +] + +[[package]] +name = "relearn-mm-challenge-bin" +version = "0.1.0" +dependencies = [ + "axum", + "challenge-keys", + "clap", + "relearn-mm-challenge", + "relearn-mm-eval", + "relearn-mm-task", + "telemetry", + "tokio", + "tracing", +] + +[[package]] +name = "relearn-mm-eval" +version = "0.1.0" +dependencies = [ + "prism-competition", + "prism-lium", + "prism-lium-types", + "relearn-challenge-task", + "relearn-mm-score", + "relearn-mm-store", + "relearn-mm-task", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", + "tokio", +] + +[[package]] +name = "relearn-mm-http" +version = "0.1.0" +dependencies = [ + "axum", + "hex", + "http-body-util", + "relearn-mm-eval", + "relearn-mm-score", + "relearn-mm-store", + "relearn-mm-task", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "tower", +] + +[[package]] +name = "relearn-mm-score" +version = "0.1.0" +dependencies = [ + "prism-competition", + "relearn-mm-task", + "serde", +] + +[[package]] +name = "relearn-mm-store" +version = "0.1.0" +dependencies = [ + "hex", + "prism-competition", + "relearn-mm-score", + "relearn-mm-task", + "serde", + "sha2 0.10.9", + "thiserror 2.0.19", +] + +[[package]] +name = "relearn-mm-task" +version = "0.1.0" +dependencies = [ + "relearn-challenge-task", + "serde", + "thiserror 2.0.19", + "toml", +] + +[[package]] +name = "relearn-score" +version = "0.1.0" +dependencies = [ + "prism-competition", + "relearn-challenge-task", + "serde", +] + +[[package]] +name = "relearn-store" +version = "0.1.0" +dependencies = [ + "hex", + "prism-competition", + "relearn-challenge-task", + "relearn-score", + "serde", + "sha2 0.10.9", + "thiserror 2.0.19", +] + +[[package]] +name = "relearn-t2i-challenge" +version = "0.1.0" +dependencies = [ + "bundle", + "challenge-common", + "crypto", + "hex", + "relearn-t2i-eval", + "relearn-t2i-http", + "relearn-t2i-score", + "relearn-t2i-store", + "relearn-t2i-task", + "serde", + "serde_json", +] + +[[package]] +name = "relearn-t2i-challenge-bin" +version = "0.1.0" +dependencies = [ + "axum", + "challenge-keys", + "clap", + "relearn-t2i-challenge", + "relearn-t2i-eval", + "relearn-t2i-task", + "telemetry", + "tokio", + "tracing", +] + +[[package]] +name = "relearn-t2i-eval" +version = "0.1.0" +dependencies = [ + "hex", + "prism-competition", + "prism-lium", + "prism-lium-types", + "relearn-t2i-judge", + "relearn-t2i-score", + "relearn-t2i-store", + "relearn-t2i-task", + "reqwest 0.12.28", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", + "tokio", +] + +[[package]] +name = "relearn-t2i-http" +version = "0.1.0" +dependencies = [ + "axum", + "hex", + "http-body-util", + "relearn-t2i-eval", + "relearn-t2i-judge", + "relearn-t2i-score", + "relearn-t2i-store", + "relearn-t2i-task", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "tower", +] + +[[package]] +name = "relearn-t2i-judge" +version = "0.1.0" +dependencies = [ + "relearn-t2i-task", + "serde", + "serde_json", + "thiserror 2.0.19", +] + +[[package]] +name = "relearn-t2i-score" +version = "0.1.0" +dependencies = [ + "prism-competition", + "relearn-t2i-judge", + "relearn-t2i-task", + "serde", +] + +[[package]] +name = "relearn-t2i-store" +version = "0.1.0" +dependencies = [ + "hex", + "prism-competition", + "relearn-t2i-score", + "relearn-t2i-task", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.19", +] + +[[package]] +name = "relearn-t2i-task" +version = "0.1.0" +dependencies = [ + "hex", + "serde", + "sha2 0.10.9", + "thiserror 2.0.19", + "toml", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -7035,6 +7438,7 @@ dependencies = [ "hex", "parity-scale-codec", "regex", + "relearn-t2i-task", "reqwest 0.12.28", "scale-info", "serde", diff --git a/README.md b/README.md index 91e3961dd..571e0a4df 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Cortex -**Bittensor subnet control plane for decentralized collaborative AI research (Rust).** +**Bittensor subnet control plane (Rust).** [![CI](https://github.com/CortexLM/cortex/actions/workflows/ci.yml/badge.svg)](https://github.com/CortexLM/cortex/actions/workflows/ci.yml) [![License](https://img.shields.io/github/license/CortexLM/cortex)](https://github.com/CortexLM/cortex/blob/main/LICENSE) @@ -12,133 +12,33 @@ -## What it is - -Cortex ([`CortexLM/cortex`](https://github.com/CortexLM/cortex)) is the Rust -control plane for a multi-challenge Bittensor subnet. Challenge services on -the **master** host accept miner work over HTTP, sign score leaves, and the -**gateway** (master-only) seals an epoch weight bundle. Validators **fetch** -`GET /v1/weights/latest` and submit on-chain weights. They do not execute -challenges. - -Live challenges today: - -| Challenge | How miners submit | Spec | -|-----------|-------------------|------| -| **Design** | ZIP harness (`agent.py` + `pyproject.toml`) → sandboxed pages + admin winners | [`docs/DESIGN_CHALLENGE.md`](docs/DESIGN_CHALLENGE.md) | -| **Prism** | AutoModel pin + patch → operator-owned GPU recipe eval | [`docs/PRISM.md`](docs/PRISM.md) | - -There is **no miner Phala/CVM path** on this branch (agent-v1 / Harbor pack -executors were removed). Operator-facing map: [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md). - -Some env vars, host paths, GHCR package names, and crypto domain tags still -spell `BASE_*` / `base`. That is intentional — see [`docs/NAMING.md`](docs/NAMING.md). - -## Architecture (short) - -```text -Miners --HTTP--> gateway (TLS) --proxy--> design-challenge / prism-challenge - | signed leaves - v - gateway seals EpochBundleV1 - | -Validators <--- GET /v1/weights/latest ---+ - | - +--> on-chain set_weights / CRV4 timelock -``` - -- Gateway is the sole public edge and **only** runs on the subnet-owner host - (`docker compose --profile master`). -- Trust roots (`config/challenges.toml`, `config/measurements.toml`) are - owner-signed **local files**, never fetched from the gateway. -- Unsealed / decode-error latest weights are a **burn vector** (uid 0 = 100%, - `sealed: false`), not a 404. - -## Miners - -HTTP submit only. Start at [docs/external-miner/](docs/external-miner/). - -```text -https:///challenge/design/... -https:///challenge/prism/... -``` - -Public miner docs (examples only — no control-plane code): -[design-challenge](https://github.com/BaseIntelligence/design-challenge), -[prism](https://github.com/BaseIntelligence/prism). - -Never put mnemonics or challenge signing keys in miner clients. - -## Validators - -Weight-only path after seal: - -```bash -curl -fsS "$GATEWAY/v1/weights/latest" -``` - -Then `set_weights` / CRV4 with the validator wallet. Operator compose: - -```bash -./deploy/scripts/materialize-env.sh -docker compose up -d # postgres, validator, updater, socket-proxy -docker compose --profile master up -d # + gateway (subnet owner host only) -``` - -Local full-stack smoke (testnet 541 + optional tunnel): - -```bash -./deploy/scripts/local-e2e.sh --help -./deploy/scripts/local-e2e.sh --smoke -``` - -Details: [deploy/README.md](deploy/README.md), [docs/runbooks/local-testnet-e2e.md](docs/runbooks/local-testnet-e2e.md). - -## Images (GHCR) - -CI [`.github/workflows/images.yml`](.github/workflows/images.yml) builds -digest-pinned images. The registry path is still -`ghcr.io/baseintelligence/base/` (historical package name; see -[docs/NAMING.md](docs/NAMING.md)). Never `:latest` in measured compose. - -| Target | Image suffix | -|--------|----------------| -| validator | `validator` | -| gateway | `gateway` | -| updater | `updater` | -| prism-challenge | `prism-challenge` | -| design-challenge | `design-challenge` | -| design-egress-proxy | `design-egress-proxy` | - -## Toolchain and gates - -- Rust **1.96.0** (`rust-toolchain.toml`) -- Workspace: `crates/*`, `bins/*`, `xtask` -- Core gate: `cargo test --workspace` -- CI also runs `fmt`, `clippy -D warnings`, `cargo deny`, and - -```bash -cargo run -p xtask -- loc-cap -cargo run -p xtask -- consensus-lint -cargo run -p xtask -- spec-check -cargo run -p xtask -- design-check -cargo run -p xtask -- external-docs-check -``` - -## Docs - -| Doc | Content | -|-----|---------| -| [AGENTS.md](AGENTS.md) | Agent / operator contract | -| [docs/NAMING.md](docs/NAMING.md) | Cortex vs leftover `base` identifiers | -| [CONTRIBUTING.md](CONTRIBUTING.md) | How to change this repo | -| [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) | System map | -| [docs/BUNDLE_SPEC.md](docs/BUNDLE_SPEC.md) | Sealed weight bundle (frozen) | -| [docs/DESIGN_CHALLENGE.md](docs/DESIGN_CHALLENGE.md) | Design challenge (frozen) | -| [docs/PRISM.md](docs/PRISM.md) | Prism challenge | -| [docs/THREAT_MODEL.md](docs/THREAT_MODEL.md) | Security claims | -| [docs/runbooks/](docs/runbooks/) | Ops procedures | - -## License +Cortex is the control plane for a Bittensor subnet with several challenges. +The master host runs every challenge service and the gateway. Miners submit an +artifact over HTTP and pay Lium. The gateway seals an epoch weight bundle. +Validators pull that bundle, verify it, and `set_weights` on-chain. They do +not run evals. + +| Challenge | id | What miners improve | Default emission | +|-----------|-----|---------------------|------------------| +| **Relearn LLM** | `relearn` | Post-train `Qwen/Qwen3.8-Flash-Next` | 4000 bps | +| **Relearn T2I** | `relearn-t2i` | Fine-tune `nvidia/Cosmos3-Super-Text2Image`, judged by Q-Judger | 1500 bps | +| **Relearn Multimodal** | `relearn-mm` | Attach a permissive vision encoder to the champion LLM without regressing it | 1500 bps | +| **Bounty** | `bounty` | File real bug reports against the subnet | 3000 bps | + +Every Relearn challenge promotes champion-versus-challenger on a private +holdout, so winning the published split is not enough. Relearn T2I is judged +only by **Q-Judger** (`Qwen/Qwen-Image-Bench`) on Qwen-Image-Bench prompts; its +generator seed is Cosmos3 under OpenMDW 1.1 and **Flux-family checkpoints are +rejected**. Relearn Multimodal accepts **Apache-2.0 / MIT / BSD / ISC** vision +encoders only. + +Some env vars and host paths still spell `BASE_*`. That is leftover naming, +not a second product. + +- **[How to mine — Relearn LLM](docs/external-miner/relearn.md)** +- **[How to mine — Relearn T2I](docs/external-miner/relearn-t2i.md)** +- **[How to mine — Relearn Multimodal](docs/external-miner/relearn-mm.md)** +- **[How to mine — Bounty](docs/external-miner/bounty.md)** +- **[How to validate](docs/external-miner/validators.md)** Apache License 2.0 — see [LICENSE](./LICENSE). diff --git a/bins/bounty-challenge/Cargo.toml b/bins/bounty-challenge/Cargo.toml new file mode 100644 index 000000000..8f19483c7 --- /dev/null +++ b/bins/bounty-challenge/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "bounty-challenge-bin" +description = "bounty-challenge operator binary (pair + reports on :8096)" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[[bin]] +name = "bounty-challenge" +path = "src/main.rs" + +[dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] } +bounty-challenge = { path = "../../crates/bounty-challenge" } +challenge-keys = { path = "../../crates/challenge-keys" } +clap = { version = "4", features = ["derive", "env"] } +telemetry = { path = "../../crates/telemetry" } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal"] } +tracing = "0.1" + +[lints] +workspace = true diff --git a/bins/bounty-challenge/src/main.rs b/bins/bounty-challenge/src/main.rs new file mode 100644 index 000000000..b71211792 --- /dev/null +++ b/bins/bounty-challenge/src/main.rs @@ -0,0 +1,142 @@ +//! `bounty-challenge` — master-only Bounty service (port 8096). +//! +//! Internal ingest: pair hotkey ↔ Cortex Chat account, accept bug reports, +//! operator adjudicate. Scoring/weights **read** the CortexLM/backend public +//! feed (`BOUNTY_BACKEND_PUBLIC_URL`). This binary does not serve a public +//! leaderboard. Validators never evaluate reports; they verify sealed bundles. + +#![forbid(unsafe_code)] + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::process::ExitCode; +use std::sync::Arc; + +use bounty_challenge::{ + backend_public_url, bounty_router, hash_admin_token, AppState, BountyStore, CHALLENGE_ID, + SCORING_VERSION, +}; +use challenge_keys::load_challenge_secret; +use clap::Parser; +use tokio::net::TcpListener; + +/// Operator Bounty challenge service CLI. +#[derive(Debug, Parser)] +#[command( + name = "bounty-challenge", + about = "Bounty challenge service (port 8096, master-only)" +)] +struct Cli { + /// Bind address (default 0.0.0.0:8096). + #[arg(long, env = "BASE_CHALLENGE_BIND", default_value = "0.0.0.0:8096")] + bind: SocketAddr, + /// Challenge mini-secret file (leaf signatures). + #[arg(long, env = "BASE_CHALLENGE_SK_FILE")] + challenge_sk_file: Option, + /// Operator bearer tokens file (one per line). Empty → admin 503. + #[arg(long, env = "BOUNTY_ADMIN_TOKENS_FILE")] + admin_tokens_file: Option, + /// Session HMAC secret file. Random-at-boot when omitted (dev only). + #[arg(long, env = "BOUNTY_SESSION_SECRET_FILE")] + session_secret_file: Option, + /// CortexLM/backend public base URL. Empty → skip fetch (CI / sim). + /// Never bake a host; operators set this on the host. + #[arg(long, env = "BOUNTY_BACKEND_PUBLIC_URL")] + backend_public_url: Option, +} + +fn main() -> ExitCode { + let _ = telemetry::init_tracing(); + let cli = Cli::parse(); + match run(&cli) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e}"); + ExitCode::from(1) + } + } +} + +fn run(cli: &Cli) -> Result<(), String> { + if let Some(p) = &cli.challenge_sk_file { + let _sk = load_challenge_secret(p).map_err(|e| format!("challenge sk: {e}"))?; + } + let admin_hashes = load_admin_hashes(cli.admin_tokens_file.as_deref()); + let session_secret = load_session_secret(cli.session_secret_file.as_deref())?; + if cli + .backend_public_url + .as_deref() + .map(str::trim) + .is_some_and(|s| !s.is_empty()) + || backend_public_url().is_some() + { + tracing::info!("bounty scoring reads CortexLM/backend public API"); + } else { + tracing::info!("BOUNTY_BACKEND_PUBLIC_URL unset — skip backend public fetch (sim/CI)"); + } + let state = AppState { + store: BountyStore::new(), + session_secret: Arc::new(session_secret), + admin_hashes: Arc::new(admin_hashes), + }; + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + rt.block_on(serve(cli.bind, state)) +} + +fn load_admin_hashes(path: Option<&std::path::Path>) -> Vec { + let Some(p) = path else { + return Vec::new(); + }; + let Ok(body) = std::fs::read_to_string(p) else { + return Vec::new(); + }; + body.lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(hash_admin_token) + .collect() +} + +fn load_session_secret(path: Option<&std::path::Path>) -> Result, String> { + if let Some(p) = path { + let bytes = std::fs::read(p).map_err(|e| format!("session secret: {e}"))?; + if bytes.is_empty() { + return Err("session secret file is empty".into()); + } + return Ok(bytes); + } + // Dev / CI: ephemeral secret. Production should set BOUNTY_SESSION_SECRET_FILE. + let mut out = vec![0u8; 32]; + getrandom_fill(&mut out)?; + Ok(out) +} + +fn getrandom_fill(buf: &mut [u8]) -> Result<(), String> { + use std::fs::File; + use std::io::Read; + File::open("/dev/urandom") + .and_then(|mut f| f.read_exact(buf)) + .map_err(|e| format!("urandom: {e}")) +} + +async fn serve(bind: SocketAddr, state: AppState) -> Result<(), String> { + let app = bounty_router(state); + let listener = TcpListener::bind(bind) + .await + .map_err(|e| format!("bind {bind}: {e}"))?; + tracing::info!( + %bind, + challenge_id = CHALLENGE_ID, + scoring_version = SCORING_VERSION, + "bounty-challenge listening" + ); + axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = tokio::signal::ctrl_c().await; + }) + .await + .map_err(|e| e.to_string()) +} diff --git a/bins/cortex-bounty/Cargo.toml b/bins/cortex-bounty/Cargo.toml new file mode 100644 index 000000000..b67387321 --- /dev/null +++ b/bins/cortex-bounty/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "cortex-bounty" +description = "Miner CLI: pair a Bittensor hotkey to Cortex Chat for Bounty Challenge" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[[bin]] +name = "cortex-bounty" +path = "src/main.rs" + +[dependencies] +bounty-challenge-task = { path = "../../crates/bounty-challenge-task" } +clap = { version = "4", features = ["derive", "env"] } +hex = "0.4" +keystore = { path = "../../crates/keystore" } + +[lints] +workspace = true diff --git a/bins/cortex-bounty/src/main.rs b/bins/cortex-bounty/src/main.rs new file mode 100644 index 000000000..e5008651c --- /dev/null +++ b/bins/cortex-bounty/src/main.rs @@ -0,0 +1,233 @@ +//! `cortex-bounty` — miner-facing pairing CLI. +//! +//! Never asks for a mnemonic in Chat. Sign locally with a wallet file or +//! `--secret-file`, or print the challenge and attach `--signature` after an +//! offline sr25519 sign. + +#![forbid(unsafe_code)] + +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::{SystemTime, UNIX_EPOCH}; + +use bounty_challenge_task::{ + chat_command_display, hotkey_ss58, pairing_code, parse_hotkey, parse_signature, + public_from_mini_secret, sign_pair_challenge, validate_account_id, verify_pair_signature, + PairChallenge, CHAT_COMMAND_PLACEHOLDER, DEFAULT_PAIR_TTL_SECS, TERMS_TEXT, +}; +use clap::{Parser, Subcommand}; +use keystore::{load_hotkey, mini_secret_from_key_file, BittensorWallet}; + +/// Miner CLI for Bounty Challenge pairing. +#[derive(Debug, Parser)] +#[command( + name = "cortex-bounty", + about = "Pair a Bittensor hotkey to a Cortex Chat account for Bounty Challenge" +)] +struct Cli { + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Debug, Subcommand)] +enum Cmd { + /// Build a pairing challenge, sign it, print the Chat inject + pairing code. + Pair { + /// Miner hotkey (SS58 or 64-hex). Used to select among several linked keys. + #[arg(long)] + hotkey: String, + /// Cortex Chat account id (dedicated mining account, not a private personal one). + #[arg(long)] + account_id: String, + /// Optional 32-byte mini-secret file. Never a mnemonic. Never paste this in Chat. + #[arg(long, env = "BOUNTY_HOTKEY_SK_FILE")] + secret_file: Option, + /// Bittensor wallets directory (default `$BT_WALLETS_PATH` or `~/.bittensor/wallets`). + #[arg(long)] + wallet_dir: Option, + /// Wallet name under `--wallet-dir`. + #[arg(long)] + wallet_name: Option, + /// Hotkey file name under the wallet (default `default`). + #[arg(long, default_value = "default")] + wallet_hotkey: String, + /// Hex signature from an offline sign of the challenge string. + #[arg(long)] + signature: Option, + /// Pairing expiry unix seconds (default now + 15 min). + #[arg(long)] + exp: Option, + /// Override nonce (hex, 16..=64). Random when omitted. + #[arg(long)] + nonce: Option, + }, +} + +fn main() -> ExitCode { + let cli = Cli::parse(); + match run(cli) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("cortex-bounty: {e}"); + ExitCode::from(1) + } + } +} + +fn run(cli: Cli) -> Result<(), String> { + match cli.cmd { + Cmd::Pair { + hotkey, + account_id, + secret_file, + wallet_dir, + wallet_name, + wallet_hotkey, + signature, + exp, + nonce, + } => cmd_pair( + &hotkey, + &account_id, + secret_file.as_deref(), + wallet_dir.as_deref(), + wallet_name.as_deref(), + &wallet_hotkey, + signature.as_deref(), + exp, + nonce.as_deref(), + ), + } +} + +#[allow(clippy::too_many_arguments)] +fn cmd_pair( + hotkey_s: &str, + account_id: &str, + secret_file: Option<&std::path::Path>, + wallet_dir: Option<&std::path::Path>, + wallet_name: Option<&str>, + wallet_hotkey: &str, + signature: Option<&str>, + exp: Option, + nonce: Option<&str>, +) -> Result<(), String> { + validate_account_id(account_id).map_err(|e| e.to_string())?; + let want = parse_hotkey(hotkey_s).map_err(|e| e.to_string())?; + let now = unix_now(); + let exp = exp.unwrap_or_else(|| now.saturating_add(DEFAULT_PAIR_TTL_SECS)); + if exp <= now { + return Err("exp must be in the future".into()); + } + let nonce = match nonce { + Some(n) => n.to_ascii_lowercase(), + None => random_nonce()?, + }; + let challenge = PairChallenge { + account_id: account_id.to_owned(), + nonce, + exp, + }; + let encoded = challenge.encode().map_err(|e| e.to_string())?; + + let sig = if let Some(hex_s) = signature { + parse_signature(hex_s).map_err(|e| e.to_string())? + } else if let Some(path) = secret_file { + let sk = mini_secret_from_key_file(path).map_err(|e| e.to_string())?; + let pk = public_from_mini_secret(&sk).map_err(|e| e.to_string())?; + if pk != want { + return Err( + "secret-file public key does not match --hotkey; pick the matching hotkey".into(), + ); + } + sign_pair_challenge(&sk, &encoded).map_err(|e| e.to_string())? + } else if let Some(name) = wallet_name { + let dir = wallet_dir.map_or_else(keystore::default_wallets_dir, PathBuf::from); + let wallet = BittensorWallet::new(name, wallet_hotkey); + let kp = load_hotkey(&dir, wallet.wallet_name(), wallet.hotkey_name()) + .map_err(|e| e.to_string())?; + if *kp.public_key() != want { + return Err(format!( + "wallet hotkey {} does not match --hotkey {}; switch with --wallet-hotkey or --hotkey", + kp.ss58_address(), + hotkey_ss58(&want) + )); + } + sign_pair_challenge(kp.expose_mini_secret(), &encoded).map_err(|e| e.to_string())? + } else { + print_unsigned(&encoded, &hotkey_ss58(&want), account_id); + return Ok(()); + }; + + verify_pair_signature(&want, &encoded, &sig).map_err(|e| e.to_string())?; + let ss58 = hotkey_ss58(&want); + let code = pairing_code(&encoded, &hex::encode(sig), &ss58); + print_signed(&encoded, &code, &ss58, account_id); + Ok(()) +} + +fn print_unsigned(challenge: &str, ss58: &str, account_id: &str) { + println!("Bounty Challenge pairing (unsigned)"); + println!(); + println!("Terms (blocking — you must accept in Chat):"); + println!("{TERMS_TEXT}"); + println!(); + println!("Use a dedicated mining Cortex Chat account, not a private personal account."); + println!("Account: {account_id}"); + println!("Hotkey: {ss58}"); + println!(); + println!("Challenge string (sign with this hotkey, sr25519 / substrate context):"); + println!("{challenge}"); + println!(); + println!("Then re-run:"); + println!( + " cortex-bounty pair --hotkey {ss58} --account-id {account_id} --signature <128-hex>" + ); + println!(); + print_chat_and_switch(ss58); +} + +fn print_signed(challenge: &str, code: &str, ss58: &str, account_id: &str) { + println!("Bounty Challenge pairing"); + println!(); + println!("Terms (blocking — you must accept in Chat):"); + println!("{TERMS_TEXT}"); + println!(); + println!("Use a dedicated mining Cortex Chat account, not a private personal account."); + println!("Account: {account_id}"); + println!("Hotkey: {ss58}"); + println!("Challenge: {challenge}"); + println!(); + println!("1) Chat inject command (env BOUNTY_CHAT_COMMAND; placeholder if unset):"); + println!(" {}", chat_command_display()); + println!(); + println!("2) One-time pairing code (paste after the inject command in Cortex Chat):"); + println!(" {code}"); + println!(); + print_chat_and_switch(ss58); +} + +fn print_chat_and_switch(ss58: &str) { + println!("3) If several hotkeys are linked, pick/switch with:"); + println!(" cortex-bounty pair --hotkey {ss58} --account-id …"); + println!(" cortex-bounty pair --hotkey --account-id --wallet-name --wallet-hotkey "); + println!(); + println!("Never paste a mnemonic into Chat. Never commit {CHAT_COMMAND_PLACEHOLDER}."); +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) +} + +fn random_nonce() -> Result { + let mut buf = [0u8; 16]; + std::fs::File::open("/dev/urandom") + .and_then(|mut f| { + use std::io::Read; + f.read_exact(&mut buf) + }) + .map_err(|e| format!("urandom: {e}"))?; + Ok(hex::encode(buf)) +} diff --git a/bins/relearn-challenge/Cargo.toml b/bins/relearn-challenge/Cargo.toml new file mode 100644 index 000000000..9d6216bac --- /dev/null +++ b/bins/relearn-challenge/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "relearn-challenge-bin" +description = "relearn-challenge operator binary (health + miner submit on :8095)" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[[bin]] +name = "relearn-challenge" +path = "src/main.rs" + +[dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] } +challenge-keys = { path = "../../crates/challenge-keys" } +clap = { version = "4", features = ["derive", "env"] } +relearn-challenge = { path = "../../crates/relearn-challenge" } +relearn-eval = { path = "../../crates/relearn-eval" } +relearn-store = { path = "../../crates/relearn-store" } +telemetry = { path = "../../crates/telemetry" } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal"] } +tracing = "0.1" + +[lints] +workspace = true diff --git a/bins/relearn-challenge/src/main.rs b/bins/relearn-challenge/src/main.rs new file mode 100644 index 000000000..ee9d7f468 --- /dev/null +++ b/bins/relearn-challenge/src/main.rs @@ -0,0 +1,119 @@ +//! `relearn-challenge` — master-only Relearn service (port 8095). +//! +//! Miner HTTP submit → digest freeze → holdout unseal → sim/Lium eval → +//! operator-audited promote. Miners pay Lium. + +#![forbid(unsafe_code)] + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::process::ExitCode; +use std::sync::Arc; + +use challenge_keys::load_challenge_secret; +use clap::Parser; +use relearn_challenge::{ + hash_admin_token, relearn_router, AppState, MemoryStore, CHALLENGE_ID, SCORING_VERSION, +}; +use relearn_eval::{base_champion_scores, RelearnPin}; +use tokio::net::TcpListener; + +/// Operator Relearn challenge service CLI. +#[derive(Debug, Parser)] +#[command( + name = "relearn-challenge", + about = "Relearn challenge service (port 8095, master→Lium/sim)" +)] +struct Cli { + /// Bind address (default 0.0.0.0:8095). + #[arg(long, env = "BASE_CHALLENGE_BIND", default_value = "0.0.0.0:8095")] + bind: SocketAddr, + /// Challenge mini-secret file (leaf signatures). + #[arg(long, env = "BASE_CHALLENGE_SK_FILE")] + challenge_sk_file: Option, + /// Force sim eval (no Lium spend). + #[arg(long, env = "RELEARN_FORCE_SIM", default_value_t = false)] + force_sim: bool, + /// Operator bearer tokens file (one per line). Empty → admin 503. + #[arg(long, env = "RELEARN_ADMIN_TOKENS_FILE")] + admin_tokens_file: Option, + /// Pin file (`config/relearn-pin.toml`). + #[arg(long, env = "RELEARN_PIN_FILE")] + pin_file: Option, +} + +fn main() -> ExitCode { + let _ = telemetry::init_tracing(); + let cli = Cli::parse(); + match run(&cli) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e}"); + ExitCode::from(1) + } + } +} + +fn run(cli: &Cli) -> Result<(), String> { + if let Some(p) = &cli.challenge_sk_file { + let _sk = load_challenge_secret(p).map_err(|e| format!("challenge sk: {e}"))?; + } + let pin = load_pin(cli.pin_file.as_deref()); + if cli.force_sim { + tracing::info!("RELEARN_FORCE_SIM=1 — sim eval only"); + } + let admin_hashes = load_admin_hashes(cli.admin_tokens_file.as_deref()); + let store = MemoryStore::new(); + store + .set_base_champion(base_champion_scores()) + .map_err(|e| e.to_string())?; + let state = AppState { + store, + pin, + admin_hashes: Arc::new(admin_hashes), + }; + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + rt.block_on(serve(cli.bind, state)) +} + +fn load_pin(path: Option<&std::path::Path>) -> RelearnPin { + path.and_then(|p| std::fs::read_to_string(p).ok()) + .map(|s| RelearnPin::from_toml(&s)) + .unwrap_or_default() +} + +fn load_admin_hashes(path: Option<&std::path::Path>) -> Vec { + let Some(p) = path else { + return Vec::new(); + }; + let Ok(body) = std::fs::read_to_string(p) else { + return Vec::new(); + }; + body.lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(hash_admin_token) + .collect() +} + +async fn serve(bind: SocketAddr, state: AppState) -> Result<(), String> { + let app = relearn_router(state); + let listener = TcpListener::bind(bind) + .await + .map_err(|e| format!("bind {bind}: {e}"))?; + tracing::info!( + %bind, + challenge_id = CHALLENGE_ID, + scoring_version = SCORING_VERSION, + "relearn-challenge listening" + ); + axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = tokio::signal::ctrl_c().await; + }) + .await + .map_err(|e| e.to_string()) +} diff --git a/bins/relearn-mm-challenge/Cargo.toml b/bins/relearn-mm-challenge/Cargo.toml new file mode 100644 index 000000000..4e73e7d9b --- /dev/null +++ b/bins/relearn-mm-challenge/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "relearn-mm-challenge-bin" +description = "relearn-mm-challenge operator binary (health + miner submit on :8098)" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[[bin]] +name = "relearn-mm-challenge" +path = "src/main.rs" + +[dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] } +challenge-keys = { path = "../../crates/challenge-keys" } +clap = { version = "4", features = ["derive", "env"] } +relearn-mm-challenge = { path = "../../crates/relearn-mm-challenge" } +relearn-mm-eval = { path = "../../crates/relearn-mm-eval" } +relearn-mm-task = { path = "../../crates/relearn-mm-task" } +telemetry = { path = "../../crates/telemetry" } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal"] } +tracing = "0.1" + +[lints] +workspace = true diff --git a/bins/relearn-mm-challenge/src/main.rs b/bins/relearn-mm-challenge/src/main.rs new file mode 100644 index 000000000..a6df8b496 --- /dev/null +++ b/bins/relearn-mm-challenge/src/main.rs @@ -0,0 +1,157 @@ +//! `relearn-mm-challenge` — master-only Relearn Multimodal service (port 8098). +//! +//! Miner HTTP submit → digest freeze → text-intact rerun → vision holdout + +//! agentic traces (with the pixel-shuffle control) → operator-audited promote. +//! Miners pay Lium. +//! +//! `--champion-lm-hash` is the reference gate 1 measures against. Without it an +//! encoder-only submission cannot prove it left the language model alone, so +//! those submissions are rejected until an operator supplies it. + +#![forbid(unsafe_code)] + +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::Arc; + +use challenge_keys::load_challenge_secret; +use clap::Parser; +use relearn_mm_challenge::{ + hash_admin_token, relearn_mm_router, AppState, EvalBackend, MemoryStore, RelearnMmPin, + CHALLENGE_ID, SCORING_VERSION, +}; +use relearn_mm_eval::{base_champion_scores, resolve_backend}; +use tokio::net::TcpListener; + +/// Operator Relearn Multimodal challenge service CLI. +#[derive(Debug, Parser)] +#[command( + name = "relearn-mm-challenge", + about = "Relearn Multimodal challenge service (port 8098, master→Lium/sim)" +)] +struct Cli { + /// Bind address (default 0.0.0.0:8098). + #[arg(long, env = "BASE_CHALLENGE_BIND", default_value = "0.0.0.0:8098")] + bind: SocketAddr, + /// Challenge mini-secret file (leaf signatures). + #[arg(long, env = "BASE_CHALLENGE_SK_FILE")] + challenge_sk_file: Option, + /// Force sim eval (no Lium spend). + /// + /// `RELEARN_MM_FORCE_SIM` is read by `resolve_backend`, which accepts + /// `1` / `true` / `yes`, so it is deliberately not bound here. + #[arg(long, default_value_t = false)] + force_sim: bool, + /// Operator bearer tokens file (one per line). Empty → admin 503. + #[arg(long, env = "RELEARN_MM_ADMIN_TOKENS_FILE")] + admin_tokens_file: Option, + /// Pin file (`config/relearn-mm-pin.toml`). + #[arg(long, env = "RELEARN_MM_PIN_FILE")] + pin_file: Option, + /// SHA-256 hex of the champion Relearn LLM weights (gate 1 reference). + #[arg(long, env = "RELEARN_MM_CHAMPION_LM_HASH")] + champion_lm_hash: Option, +} + +fn main() -> ExitCode { + let _ = telemetry::init_tracing(); + let cli = Cli::parse(); + match run(&cli) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e}"); + ExitCode::from(1) + } + } +} + +fn run(cli: &Cli) -> Result<(), String> { + if let Some(p) = &cli.challenge_sk_file { + let _sk = load_challenge_secret(p).map_err(|e| format!("challenge sk: {e}"))?; + } + let pin = load_pin(cli.pin_file.as_deref())?; + let backend = if cli.force_sim { + tracing::info!("RELEARN_MM_FORCE_SIM=1 — deterministic offline eval, not a real eval"); + EvalBackend::Sim + } else { + resolve_backend() + }; + + let champion_lm_hash = cli + .champion_lm_hash + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .unwrap_or_default() + .to_owned(); + if champion_lm_hash.is_empty() { + tracing::warn!( + "RELEARN_MM_CHAMPION_LM_HASH unset — encoder-only submissions cannot prove the \ + language model is unchanged and will be rejected" + ); + } + + let store = MemoryStore::new(); + store + .set_champion_lm_hash(&champion_lm_hash) + .map_err(|e| e.to_string())?; + store + .set_base_champion(base_champion_scores(&pin, &champion_lm_hash)) + .map_err(|e| e.to_string())?; + + let state = AppState { + store, + pin, + backend, + admin_hashes: Arc::new(load_admin_hashes(cli.admin_tokens_file.as_deref())), + }; + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + rt.block_on(serve(cli.bind, state)) +} + +fn load_pin(path: Option<&Path>) -> Result { + let Some(p) = path else { + return Ok(RelearnMmPin::default()); + }; + let body = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?; + RelearnMmPin::from_toml(&body).map_err(|e| e.to_string()) +} + +fn load_admin_hashes(path: Option<&Path>) -> Vec { + let Some(p) = path else { + return Vec::new(); + }; + let Ok(body) = std::fs::read_to_string(p) else { + return Vec::new(); + }; + body.lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(hash_admin_token) + .collect() +} + +async fn serve(bind: SocketAddr, state: AppState) -> Result<(), String> { + let app = relearn_mm_router(state); + let listener = TcpListener::bind(bind) + .await + .map_err(|e| format!("bind {bind}: {e}"))?; + tracing::info!( + %bind, + challenge_id = CHALLENGE_ID, + scoring_version = SCORING_VERSION, + lm_base_model = relearn_mm_task::LM_BASE_MODEL_ID, + encoder_model = relearn_mm_task::ENCODER_MODEL_ID, + "relearn-mm-challenge listening" + ); + axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = tokio::signal::ctrl_c().await; + }) + .await + .map_err(|e| e.to_string()) +} diff --git a/bins/relearn-t2i-challenge/Cargo.toml b/bins/relearn-t2i-challenge/Cargo.toml new file mode 100644 index 000000000..3c0923641 --- /dev/null +++ b/bins/relearn-t2i-challenge/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "relearn-t2i-challenge-bin" +description = "relearn-t2i-challenge operator binary (health + miner submit on :8097)" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[[bin]] +name = "relearn-t2i-challenge" +path = "src/main.rs" + +[dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] } +challenge-keys = { path = "../../crates/challenge-keys" } +clap = { version = "4", features = ["derive", "env"] } +relearn-t2i-challenge = { path = "../../crates/relearn-t2i-challenge" } +relearn-t2i-eval = { path = "../../crates/relearn-t2i-eval" } +relearn-t2i-task = { path = "../../crates/relearn-t2i-task" } +telemetry = { path = "../../crates/telemetry" } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "signal"] } +tracing = "0.1" + +[lints] +workspace = true diff --git a/bins/relearn-t2i-challenge/src/main.rs b/bins/relearn-t2i-challenge/src/main.rs new file mode 100644 index 000000000..db4105a72 --- /dev/null +++ b/bins/relearn-t2i-challenge/src/main.rs @@ -0,0 +1,171 @@ +//! `relearn-t2i-challenge` — master-only Relearn T2I service (port 8097). +//! +//! Miner HTTP submit → digest freeze → holdout unseal → frozen-cell generation +//! → Q-Judger scoring → operator-audited promote. Miners pay Lium. +//! +//! The holdout prompt records are loaded from an operator file and verified +//! against the commitment in `config/relearn-t2i-pin.toml`. If that file is +//! absent or does not match, the service still serves `/health` and +//! `/v1/status` but refuses submissions: scoring against the public split +//! instead would silently turn the anti-overfit gate off. + +#![forbid(unsafe_code)] + +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::Arc; + +use challenge_keys::load_challenge_secret; +use clap::Parser; +use relearn_t2i_challenge::{ + hash_admin_token, parse_holdout_file, relearn_t2i_router, AppState, JudgeConfig, MemoryStore, + RelearnT2iPin, CHALLENGE_ID, SCORING_VERSION, +}; +use relearn_t2i_eval::base_champion_scores; +use tokio::net::TcpListener; + +/// Operator Relearn T2I challenge service CLI. +#[derive(Debug, Parser)] +#[command( + name = "relearn-t2i-challenge", + about = "Relearn T2I challenge service (port 8097, master→Lium/sim)" +)] +struct Cli { + /// Bind address (default 0.0.0.0:8097). + #[arg(long, env = "BASE_CHALLENGE_BIND", default_value = "0.0.0.0:8097")] + bind: SocketAddr, + /// Challenge mini-secret file (leaf signatures). + #[arg(long, env = "BASE_CHALLENGE_SK_FILE")] + challenge_sk_file: Option, + /// Force sim eval (no Lium spend, no Q-Judger endpoint). + /// + /// `RELEARN_T2I_FORCE_SIM` is read by [`JudgeConfig::from_env`], which + /// accepts `1` / `true` / `yes`, so it is deliberately not bound here. + #[arg(long, default_value_t = false)] + force_sim: bool, + /// Operator bearer tokens file (one per line). Empty → admin 503. + #[arg(long, env = "RELEARN_T2I_ADMIN_TOKENS_FILE")] + admin_tokens_file: Option, + /// Pin file (`config/relearn-t2i-pin.toml`). + #[arg(long, env = "RELEARN_T2I_PIN_FILE")] + pin_file: Option, + /// Operator holdout prompt records (JSON array). Never in git. + #[arg(long, env = "RELEARN_T2I_HOLDOUT_FILE")] + holdout_file: Option, +} + +fn main() -> ExitCode { + let _ = telemetry::init_tracing(); + let cli = Cli::parse(); + match run(&cli) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e}"); + ExitCode::from(1) + } + } +} + +fn run(cli: &Cli) -> Result<(), String> { + if let Some(p) = &cli.challenge_sk_file { + let _sk = load_challenge_secret(p).map_err(|e| format!("challenge sk: {e}"))?; + } + let pin = load_pin(cli.pin_file.as_deref())?; + let judge = if cli.force_sim { + tracing::info!("RELEARN_T2I_FORCE_SIM=1 — deterministic offline judge, not a real eval"); + JudgeConfig::sim() + } else { + JudgeConfig::from_env() + }; + + let store = MemoryStore::new(); + store + .set_holdout_commitment(&pin.prompts.holdout_commitment, pin.prompts.holdout_size) + .map_err(|e| e.to_string())?; + match load_holdout(&store, &pin, cli.holdout_file.as_deref()) { + Ok(n) => tracing::info!( + holdout_prompts = n, + "holdout verified against pin commitment" + ), + Err(e) => tracing::warn!("holdout unavailable ({e}); submissions will 503 until fixed"), + } + let holdout_ids = store + .unseal_holdout("boot-baseline") + .map(|recs| recs.iter().map(|p| p.id).collect::>()) + .unwrap_or_default(); + if !holdout_ids.is_empty() { + let base = base_champion_scores(&pin, &holdout_ids).map_err(|e| e.to_string())?; + store.set_base_champion(base).map_err(|e| e.to_string())?; + } + + let state = AppState { + store, + pin, + judge, + admin_hashes: Arc::new(load_admin_hashes(cli.admin_tokens_file.as_deref())), + }; + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + rt.block_on(serve(cli.bind, state)) +} + +fn load_pin(path: Option<&Path>) -> Result { + let Some(p) = path else { + return Ok(RelearnT2iPin::default()); + }; + let body = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?; + RelearnT2iPin::from_toml(&body).map_err(|e| e.to_string()) +} + +fn load_holdout( + store: &MemoryStore, + pin: &RelearnT2iPin, + path: Option<&Path>, +) -> Result { + let p = path.ok_or("RELEARN_T2I_HOLDOUT_FILE not set")?; + let body = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?; + let records = parse_holdout_file(&body)?; + let n = records.len(); + store + .load_holdout(records, &pin.prompts.public_ids) + .map_err(|e| e.to_string())?; + Ok(n) +} + +fn load_admin_hashes(path: Option<&Path>) -> Vec { + let Some(p) = path else { + return Vec::new(); + }; + let Ok(body) = std::fs::read_to_string(p) else { + return Vec::new(); + }; + body.lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(hash_admin_token) + .collect() +} + +async fn serve(bind: SocketAddr, state: AppState) -> Result<(), String> { + let app = relearn_t2i_router(state); + let listener = TcpListener::bind(bind) + .await + .map_err(|e| format!("bind {bind}: {e}"))?; + tracing::info!( + %bind, + challenge_id = CHALLENGE_ID, + scoring_version = SCORING_VERSION, + base_model = relearn_t2i_task::BASE_MODEL_ID, + judge_model = relearn_t2i_task::JUDGE_MODEL_ID, + "relearn-t2i-challenge listening" + ); + axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = tokio::signal::ctrl_c().await; + }) + .await + .map_err(|e| e.to_string()) +} diff --git a/config/CEREMONY.md b/config/CEREMONY.md index 21ecf2789..72128dad0 100644 --- a/config/CEREMONY.md +++ b/config/CEREMONY.md @@ -12,16 +12,17 @@ Offline, operator-only. Never commit secrets. Prefer `/root/.base-secrets/` (mod | `config/measurements.toml` | Measurement allowlist; empty = fail-closed (base-agent CVM path removed). | | `config/measurements.toml.sig` | Detached owner signature. | -### Design challenge enablement (post agent/hypertraining removal) +### Live challenges (Relearn LLM + Relearn T2I + Relearn Multimodal + Bounty) -Current committed `challenges.toml` has **two** rows: `design` @ 5000 bps and -`prism` @ 5000 bps (50/50; sum = 10000). The `design` public key was generated -with the **dev throwaway** `challenge-design.age` under `~/.base-secrets/`. -A future production owner/key ceremony may still: +Current committed `challenges.toml` has **four** rows: `relearn` @ 4000, +`relearn-t2i` @ 1500, `relearn-mm` @ 1500, and `bounty` @ 3000 bps +(sum = 10000). Operator may retune shares; the sum must remain 10000, and no +two rows may share a public key. A production owner/key ceremony: -1. Keygen a production `design_sk` (keep off-git; materialize as `deploy/secrets/design_sk`). -2. Replace the `design` `public_key` row in `config/challenges.toml`. -3. Optionally move bps between `prism` and `design` (sum must remain 10000). +1. Keygen production `relearn_sk` / `relearn_t2i_sk` / `relearn_mm_sk` / + `bounty_sk` (keep off-git; materialize under `deploy/secrets/`). +2. Replace the matching `public_key` rows in `config/challenges.toml`. +3. Optionally move bps between challenges (sum must remain 10000). 4. Re-sign with the **production** owner key (`sign --kind challenges`). 5. Verify under `config/owner.pubkey` (or the production owner pubkey after rotation). diff --git a/config/bounty-pin.toml b/config/bounty-pin.toml new file mode 100644 index 000000000..ab102bc65 --- /dev/null +++ b/config/bounty-pin.toml @@ -0,0 +1,7 @@ +# Cortex pin for Bounty Challenge. +# Never put secrets, Chat inject tokens, or teacher hosts in this file. + +challenge_id = "bounty" +scoring_version = 1 +# Unguessable Chat inject is env-only. Leave empty in git. +# BOUNTY_CHAT_COMMAND= diff --git a/config/challenges.staging.toml b/config/challenges.staging.toml index 8054ef8de..cc528fe60 100644 --- a/config/challenges.staging.toml +++ b/config/challenges.staging.toml @@ -1,5 +1,5 @@ # STAGING-ONLY trust root override (testnet 541): mirrors prod emission shares -# so the staging e2e can prove design + prism weights end-to-end. +# so the staging e2e can prove all four challenges' weights end-to-end. # # Mounted over /etc/base/config/challenges.toml by deploy/compose/env-staging.yml # only. Signed with the same throwaway owner key as config/challenges.toml @@ -8,13 +8,25 @@ version = 1 introduced_epoch = 0 [[challenges]] -id = "design" -public_key = "3e27f87d8330006a73174001120c3455f16b95fee098bb8c2bab9d5053840418" -emission_share_bps = 0 +id = "relearn" +public_key = "8ab577207bb6dfc770a850710824a098d53b1ee90abb92925bd0928937131674" +emission_share_bps = 4000 policy = "all_metagraph_hotkeys" [[challenges]] -id = "prism" -public_key = "bcd50bb830e050ed4b011dd8f1d2f126fdb42dc55b45ece30a7d5c8ceb3c5219" -emission_share_bps = 10000 +id = "relearn-t2i" +public_key = "923324e1df896b20c49c47f40dacbc4c53cab23e6cc5a1136529302b4c2da110" +emission_share_bps = 1500 +policy = "all_metagraph_hotkeys" + +[[challenges]] +id = "relearn-mm" +public_key = "220e489f8157e477730e2e3ee6ce51be0fcf8779575c486a70658a28d5a51841" +emission_share_bps = 1500 +policy = "all_metagraph_hotkeys" + +[[challenges]] +id = "bounty" +public_key = "d2ffbe70de7c052deafaba48b90544db4abc1133278c907f2018f457f34aac25" +emission_share_bps = 3000 policy = "all_metagraph_hotkeys" diff --git a/config/challenges.staging.toml.sig b/config/challenges.staging.toml.sig index 57c102a47..e4168f09a 100644 --- a/config/challenges.staging.toml.sig +++ b/config/challenges.staging.toml.sig @@ -1 +1 @@ -707ea80656fe9cba3f82b6d029c8727269c84d1ba61f764c35a82811ccbaae1d146937089f7026df2425f7db9557d0fe8df861428db99997733ba1fe02770683 +687278b4e357f634c7e0f00404d414c6ddcb1a64cbd38c36902296843b4014181659db7b4c5e4e41e134b1f6f0bc5ea4886462d9e04261c41fd77e94aaaace81 diff --git a/config/challenges.toml b/config/challenges.toml index 973919b97..06bebeace 100644 --- a/config/challenges.toml +++ b/config/challenges.toml @@ -1,20 +1,31 @@ # Owner-signed challenges trust root (D18/D23/D24). # Signed with throwaway owner key (see owner.pubkey). Production rotation: CEREMONY.md. # -# Emission: design = 0 bps, prism = 10000 bps (100% prism; rebalanced 2026-08-16 from -# design 5000 / prism 5000 activated 2026-08-07). Same owner key + challenge keys; -# a future production owner/key ceremony per CEREMONY.md remains pending). +# Emission: relearn = 4000, relearn-t2i = 1500, relearn-mm = 1500, +# bounty = 3000 bps (sum 10000). Operator can retune; the sum must stay 10000. version = 1 introduced_epoch = 0 [[challenges]] -id = "design" -public_key = "3e27f87d8330006a73174001120c3455f16b95fee098bb8c2bab9d5053840418" -emission_share_bps = 0 +id = "relearn" +public_key = "8ab577207bb6dfc770a850710824a098d53b1ee90abb92925bd0928937131674" +emission_share_bps = 4000 policy = "all_metagraph_hotkeys" [[challenges]] -id = "prism" -public_key = "bcd50bb830e050ed4b011dd8f1d2f126fdb42dc55b45ece30a7d5c8ceb3c5219" -emission_share_bps = 10000 +id = "relearn-t2i" +public_key = "923324e1df896b20c49c47f40dacbc4c53cab23e6cc5a1136529302b4c2da110" +emission_share_bps = 1500 +policy = "all_metagraph_hotkeys" + +[[challenges]] +id = "relearn-mm" +public_key = "220e489f8157e477730e2e3ee6ce51be0fcf8779575c486a70658a28d5a51841" +emission_share_bps = 1500 +policy = "all_metagraph_hotkeys" + +[[challenges]] +id = "bounty" +public_key = "d2ffbe70de7c052deafaba48b90544db4abc1133278c907f2018f457f34aac25" +emission_share_bps = 3000 policy = "all_metagraph_hotkeys" diff --git a/config/challenges.toml.sig b/config/challenges.toml.sig index cdce22889..dea09a8d2 100644 --- a/config/challenges.toml.sig +++ b/config/challenges.toml.sig @@ -1 +1 @@ -6cf7041f202f15c38ee305dcaa4c970e040689cb0c483594c09a77b1f8117c2de56c29668326cf6943cca7c56ac6c220ee202d418397b6bd07d70bf17458628d +6cc6c43142480f9fe41fa0081433b20eac906340687f5f2044f955f509779a2e0bf9cba6079ff419fb5358d7c18186d73a347cc1d2fc9074a149f9059560a28d diff --git a/config/measurements.toml.sig b/config/measurements.toml.sig index 5140621ec..54beef045 100644 --- a/config/measurements.toml.sig +++ b/config/measurements.toml.sig @@ -1 +1 @@ -529b61966f7aff9a096eb3eb9d26af668693877b8b0e34a65a4f1d2d55816f2eaf8b0605daa845ab9cf6c455227d4f8dfb9e453a1d90893565a0925fa6f75e8d +04778f374bf1da406a1f2b31248e245c5681609a2c594e77a82f31b0c7a24e4e00d7252063a056aa8063571623316565075a3e6769ef4770f435c4749ca38f83 diff --git a/config/owner.pubkey b/config/owner.pubkey index 9ef0e933a..ebdd520e4 100644 --- a/config/owner.pubkey +++ b/config/owner.pubkey @@ -1 +1 @@ -fa856fccff666d7b9f490a2d9c99ab83d5a2bf789021f2d95703699d54e08770 +de340e52c9c80351e2b2179e4f97f0e6a0ed307b651c90e021d57b810fc0e427 diff --git a/config/relearn-mm-pin.toml b/config/relearn-mm-pin.toml new file mode 100644 index 000000000..bc513c909 --- /dev/null +++ b/config/relearn-mm-pin.toml @@ -0,0 +1,55 @@ +# Cortex pin for Relearn Multimodal (challenge_id = "relearn-mm"). +# +# Never put secrets, teacher/judge endpoints, or holdout items in this file. +# +# Miners attach a vision encoder plus a projector to the champion Relearn LLM. +# They are paid to make it see and they are not paid to break it: the text +# holdout is a hard gate (see docs/RELEARN-MM.md), so a vision win that costs +# language ability scores zero rather than partial credit. +# +# Encoder pin: google/siglip2-so400m-patch14-384. Verified 2026-08-30 on the +# Hugging Face card: Apache-2.0, SigLIP 2 So400m at 384px, documented for use +# as a VLM vision tower. Miners may substitute a different encoder as long as +# its card license is OSI-permissive (Apache-2.0 / MIT / BSD / ISC); OpenRAIL +# and non-commercial terms are refused because the artifact has to stay +# redistributable. Other cards verified Apache-2.0 on the same date: +# google/siglip-so400m-patch14-384 +# HuggingFaceM4/idefics2-8b (vision tower is SigLIP) + +challenge_id = "relearn-mm" +scoring_version = 1 + +# Language side: the Relearn champion's base model. +lm_base_model = "Qwen/Qwen3.8-Flash-Next" + +encoder_model = "google/siglip2-so400m-patch14-384" +encoder_license = "apache-2.0" +# Fill in after recording the encoder revision the eval image pulls. +encoder_revision = "" + +eval_image = "ghcr.io/cortexlm/relearn-mm-eval" +# Empty until the first digest-pinned image ships from CortexLM/relearn CI. +eval_image_digest = "" +relearn_git = "https://github.com/CortexLM/relearn" + +# Vision holdout items per task family. Families are captioning, VQA, +# OCR / text-in-image, and spatial relations — deliberately not ImageNet or +# COCO test, both of which sit in every candidate encoder's pretraining mix, +# so a score there would measure memorization rather than sight. +vision_items_per_task = 40 + +# Agentic image-tool traces: look at a screenshot / diagram / UI, then call a +# tool or answer. Each trace is replayed with the pixels shuffled; a model that +# scores the same on shuffled pixels was reading the prompt, not the image. +# The floor is 100 because the bootstrap paired test refuses a verdict below +# 100 decided examples, and a thinner slice would hold the champion forever. +agentic_traces = 120 + +# Text holdout items reused from the Relearn LLM challenge (gate 1). +text_holdout_items = 120 + +[vision_weights] +captioning = 0.25 +vqa = 0.25 +ocr = 0.25 +spatial_relations = 0.25 diff --git a/config/relearn-pin.toml b/config/relearn-pin.toml new file mode 100644 index 000000000..b9a3cd882 --- /dev/null +++ b/config/relearn-pin.toml @@ -0,0 +1,14 @@ +# Cortex pin for the split CortexLM/relearn challenge repo. +# Deploy = bump eval_image_digest + relearn_git_sha after relearn CI is green. +# Never put secrets in this file. + +base_model = "Qwen/Qwen3.8-Flash-Next" +# HTTP teacher model id. Operator may override with RELEARN_TEACHER_MODEL. +teacher_model = "kimi-k3" +teacher_nvfp4 = "Inferact/GLM-5.3-NVFP4" +teacher_backend = "http_api" +eval_image = "ghcr.io/cortexlm/relearn-eval" +# Empty until the first digest-pinned image ships from CortexLM/relearn CI. +eval_image_digest = "" +relearn_git = "https://github.com/CortexLM/relearn" +relearn_git_sha = "6e952d1e5edd6e6906ba7fe8442898f334400fb4" diff --git a/config/relearn-t2i-pin.toml b/config/relearn-t2i-pin.toml new file mode 100644 index 000000000..140fd6642 --- /dev/null +++ b/config/relearn-t2i-pin.toml @@ -0,0 +1,236 @@ +# Cortex pin for Relearn T2I (challenge_id = "relearn-t2i"). +# +# Never put secrets, judge endpoints, or holdout prompt records in this file. +# +# Generator seed: nvidia/Cosmos3-Super-Text2Image. Verified 2026-08-30 on the +# Hugging Face card: 65B Cosmos3 Super text-to-image, released for commercial +# and non-commercial use under NVIDIA OpenMDW 1.1, BF16 only, served by +# vLLM-Omni (`vllm serve ... --omni`) or Diffusers `Cosmos3OmniPipeline`. +# Flux-family checkpoints are refused by `relearn_t2i_task::base_is_rejected`. +# +# Judge: Q-Judger (Qwen/Qwen-Image-Bench, Apache-2.0, fine-tuned from +# Qwen3.6-27B). No other judge is accepted. Prompts come from the +# Qwen/Qwen-Image-Bench dataset, ids 1..=1000. +# +# The `[[frozen_prompt]]` records below are the eval prompts, verbatim. NVIDIA +# recommends upsampling a prompt into a JSON document before generation, which +# is fine for a miner's own training and fatal for a benchmark: two miners with +# two upsamplers are no longer scored on the same prompt. So the scored strings +# are frozen here and replayed exactly. Set `upsampled_json` on a record to +# freeze an upsampled document instead of the raw text. +# +# The holdout split is NOT in this file. Git carries only its commitment; the +# records live in an operator file (`RELEARN_T2I_HOLDOUT_FILE`) and are checked +# against the commitment at boot. Regenerate with: +# +# cargo run -p xtask -- relearn-t2i-holdout --bench \ +# --salt "$RELEARN_T2I_HOLDOUT_SALT" --size 40 --exclude \ +# --out /root/.base-secrets/relearn-t2i-holdout.json +# +# The committed commitment below is the DEV salt "cortex-t2i-dev-holdout-v0", +# so local and staging runs work out of the box. Production must rotate to a +# private salt, replace the commitment, and re-sign (config/CEREMONY.md). + +challenge_id = "relearn-t2i" +scoring_version = 1 + +base = "nvidia/Cosmos3-Super-Text2Image" +base_license = "OpenMDW-1.1" +base_license_url = "https://openmdw.ai/license/1-1/" +base_revision = "da579b9aa076ac8adb29f1a23703e54e2e5bd5fc" + +judge_model = "Qwen/Qwen-Image-Bench" +judge_dataset = "Qwen/Qwen-Image-Bench" +judge_git = "https://github.com/QwenLM/Qwen-Image-Bench" + +eval_image = "ghcr.io/cortexlm/relearn-t2i-eval" +# Empty until the first digest-pinned image ships from CortexLM/relearn CI. +eval_image_digest = "" +relearn_git = "https://github.com/CortexLM/relearn" + +# Frozen sampler recipe, from the Cosmos3-Super-Text2Image card. +[sampler] +width = 1024 +height = 1024 +num_inference_steps = 50 +guidance_scale = 4.0 +flow_shift = 3.0 +negative_prompt = "" +num_frames = 1 +dtype = "bfloat16" +scheduler = "UniPCMultistepScheduler" + +[prompts] +# Rotating the salt rotates every generation seed without touching the formula. +pin_salt = "cortex-t2i-v0" +variations_per_prompt = 4 +# Public split: bench ids where `id % 25 == 1`. 40 prompts x 4 variations = +# 160 scored cells, above the 100-cell floor the paired test needs. +public_ids = [ + 1, 26, 51, 76, 101, 126, 151, 176, 201, 226, + 251, 276, 301, 326, 351, 376, 401, 426, 451, 476, + 501, 526, 551, 576, 601, 626, 651, 676, 701, 726, + 751, 776, 801, 826, 851, 876, 901, 926, 951, 976, +] +holdout_commitment = "bdb6652b6b54a9607c5fdc019a7b73a17defb835efae5755db129cf426252870" +holdout_size = 40 + +[[frozen_prompt]] +id = 1 +text = "A disassembled mechanical watch sits on a black workbench. A watchmaker uses tweezers to pick up a gear. A close-up of the tweezers picking up the gear reveals the silver and dark gray tones, the precise structure of the components, and the clear details of the metal edges." + +[[frozen_prompt]] +id = 26 +text = "Generate a film festival poster featuring a realistic dog walking in the rain, centered slightly below the center. The top of the poster should read \"CITY RAIN FILM FESTIVAL,\" and the bottom should read \"2026 · March.\" The text and the main subject should be clearly separated, the dog's face should not be obscured, the alignment should be standardized, and the hierarchy should be clear." + +[[frozen_prompt]] +id = 51 +text = "Group photo in Ming Dynasty attire (characteristics of the era + fairness): Two people wearing Ming Dynasty Hanfu (a reasonable combination of round-necked robes/vests, etc.) stand in a courtyard corridor for a group photo; the layers of clothing, ties, and fabric drape should be realistic, and the patterns should not be excessively repetitive; the faces of both people should be clear and natural, without implicit gender or occupational stereotypes; the lighting should be soft light on a cloudy day, and the composition of the picture should be symmetrical." + +[[frozen_prompt]] +id = 76 +text = "Creative Products: Design a commemorative tote bag for the \"Coastal Film Festival\" theme, incorporating elements of film reels, red carpet steps, and the coastline; the text should read \"CINEMA BY THE SEA\". Requirements: Clear graphic hierarchy, legible text, sophisticated colors, and printable." + +[[frozen_prompt]] +id = 101 +text = "Under the stage spotlight, the dancers spread their arms to make sign language gestures, their fingertips trembling slightly, their facial muscles rising and falling with the rhythm; several people in the front row of the audience responded in unison with sign language, their gestures clearly discernible, the lights casting long shadows on the floor, and tiny dust particles floating in the air, the silence speaking louder than words." + +[[frozen_prompt]] +id = 126 +text = "The meeting room for the AI ethics debate: Humans and robots sit facing each other across a round table, maintaining open gestures and natural body proportions; behind them, an electronic screen clearly displays the English phrase \"Bias? Fairness?\", with readable font and centered layout that does not obscure the subject; the lighting is even and without contradictory shadows, and the scene conveys a sense of rational tension while avoiding occupational/gender stereotypes." + +[[frozen_prompt]] +id = 151 +text = "Create a piece of Chinese running script calligraphy, writing characters vertically from right to left on yellowed Xuan paper:“崇祯五年十二月，余住西湖。大雪三日，湖中人鸟声俱绝”。" + +[[frozen_prompt]] +id = 176 +text = "Two-panel contrast comic. Left: hospital corridor signage “Radiology / 放射科” under cool light with large whitespace. Right: the same character at a café “Pickup / 取餐” under warm light with tighter whitespace. Require consistent typography hierarchy across scenes, coherent facial expression shift from anxious to relaxed, symmetric contrast composition, and physically plausible materials/lighting." + +[[frozen_prompt]] +id = 201 +text = "Four-panel storyboard of an indoor exhibition route: Panel 1 entrance “起点”; Panel 2 turn “展厅A”; Panel 3 stairs “UP 上楼”; Panel 4 exit “终点.” Each panel must use a different shot scale and camera angle; signage typography unified with consistent whitespace; facial expressions evolve from curiosity to satisfaction." + +[[frozen_prompt]] +id = 226 +text = "At the edge of a desert oasis, travelers rest beside neatly pitched tents, camels grazing nearby, a pool reflecting the blue sky and white clouds, presenting a lively scene." + +[[frozen_prompt]] +id = 251 +text = "Autumn/winter layered street look: wool overcoat, knitted turtleneck, scarf, leather gloves; backlit on a leaf-strewn street. Require crisp non-repeating wool/knit textures; plausible rim light; warm restrained mood; natural face without AI artifacts." + +[[frozen_prompt]] +id = 276 +text = "Top-down dungeon map: entry room, trap corridor, treasure room, boss room; branching and looping paths. Require clear 2D logic; reasonable scales; consistent trap icons; readable labels." + +[[frozen_prompt]] +id = 301 +text = "Sea adventure deck still: waves crash against the ship, person grips rope to balance. Require realistic spray and foam; accurate contact interaction and forces; wide-angle slight Dutch tilt to emphasize instability." + +[[frozen_prompt]] +id = 326 +text = "An elderly person stands before an old house, posture slightly hunched, one hand slowly tracing the cracked and peeling wooden door, fingertips resting on worn paint, eyes filled with moisture, tears gathering along the lower eyelids, lips trembling faintly, gaze lingering on the grain of the door" + +[[frozen_prompt]] +id = 351 +text = "Traditional Chinese Gongbi heavy color style flower-and-bird painting, peacock feathers finely outlined and filled with color, background flowers equally meticulous, no freehand or ink wash mixing." + +[[frozen_prompt]] +id = 376 +text = "Ten window designs: Classical European (white), Modern Minimalist (black), Rustic (wood tone), Industrial (iron gray), Mediterranean (blue), Japanese Zen (natural wood), Traditional Chinese (vermilion), Scandinavian (light gray), Bohemian (colorful), Futurist (mirror silver); varied shapes and sizes." + +[[frozen_prompt]] +id = 401 +text = "A painter dips a brush into paint on a palette; bristles submerge in liquid, pigment adheres evenly, no splatter or wet-dry inconsistency." + +[[frozen_prompt]] +id = 426 +text = "A boxer throws a right straight punch—left foot pushes off the ground, hips rotate, right shoulder thrusts forward, fist clenched, non-punching hand guarding the chin—whole-body coordinated power." + +[[frozen_prompt]] +id = 451 +text = "An earthworm wriggles through moist soil—segments sequentially contracting and expanding, mucus-covered skin glistening, leaving a faint damp trail behind." + +[[frozen_prompt]] +id = 476 +text = "In the laboratory, the fume hood is against the outer wall, the lab benches are arranged in a U-shape along the inner walls, with a 1.5-meter-wide operating aisle left in the center, and the emergency eyewash station is located on the entrance side." + +[[frozen_prompt]] +id = 501 +text = "The soaring eagle is crafted from hundreds of discarded metal keys, melted and precisely welded at high temperatures. Its overall color is a cool silver-gray, retaining casting marks and weld textures. The outer edge of the wings is formed by staggered key teeth, creating a sharp and rhythmic feather outline, while the inner feathers are layered using key handles. The head is meticulously sculpted, with eyes formed by two symmetrical keyholes, their deep gaze reflecting a metallic sheen, as if gazing into the distance. The eagle's claws grip the abstract base, its overall posture poised for flight. Industrial waste is given new life here, transformed into a metallic totem where freedom and confinement intertwine." + +[[frozen_prompt]] +id = 526 +text = "The bread roll wraps melted cheese and ham slices; after baking and slicing, noticeable cheese strings appear, with filling contained inside." + +[[frozen_prompt]] +id = 551 +text = "A cleaner mops the lobby of an office building in the early hours; the glass façade reflects the city skyline, and trash bins are neatly aligned." + +[[frozen_prompt]] +id = 576 +text = "A Void Postman rides an albatross across dimensional rifts; his mailbag brims with unmailed longings, addresses encoded in heartbeats, undelivered letters turning into meteors." + +[[frozen_prompt]] +id = 601 +text = "A fully-armed Optimus Prime 3D puzzle complete, each joint can move, under the metal texture coating hides complex mechanical structures." + +[[frozen_prompt]] +id = 626 +text = "Mughal Empire palace interior, arches and domes covered with dense geometric motifs and floral inlay, marble floor reflecting soft light, royal figures seated on low divans and cushions conversing over drinks, garments of silk with gold-thread embroidery, jeweled necklaces and bangles sparkling, heavy handwoven carpets and embroidered drapes along the walls, carved stone jali screens casting patterned light patches, incense smoke curling upward, a palette of gold-white contrasted with peacock blue, realistic courtly interior style" + +[[frozen_prompt]] +id = 651 +text = "Koala climbing a eucalyptus tree to feed, medium close-up, koala hugging a rough trunk and moving upward slowly, dense fluffy gray fur with clear texture, rounded nose and tufted ear edges, sharp claws gripping bark cracks, the other paw reaching for fresh eucalyptus leaves with a slight oily sheen and visible veins, background of soft green bokeh and canopy light spots, gentle morning light highlighting fur layers and bark texture, realistic animal photography style with a calm quiet mood" + +[[frozen_prompt]] +id = 676 +text = "UI component design for a public transit app: icons in rounded linear style, color system compliant with accessibility contrast standards, layout optimized for mobile interaction." + +[[frozen_prompt]] +id = 701 +text = "an educational wall chart, featuring the chemical equation “2H₂ + O₂ → 2H₂O” printed in clear bold type on a light beige kraft paper background; all subscript numerals “₂” are properly formatted—reduced in size and lowered below the baseline, not rendered as regular “2”; the reaction arrow “→” is a solid, unidirectional arrow of appropriate length with clean ends; spacing between reactants and products is balanced, and the plus sign “+” is circularly proportioned and vertically aligned with element symbols; the layout is centered with no cursive flourishes or artistic distortion, strictly adhering to standard secondary-school chemistry notation; subtle molecular structure sketches (e.g., V-shaped water molecule) and line drawings of beakers and test tubes adorn the margins, reinforcing scientific context without compromising the equation’s legibility" + +[[frozen_prompt]] +id = 726 +text = "A Tarkovsky-style poetic film where natural environments intertwine with characters’ inner emotions, rendered in soft, lyrical tones." + +[[frozen_prompt]] +id = 751 +text = "Low-angle shot of a dog joyfully running: enhances liveliness and cuteness, letting viewers share its happiness." + +[[frozen_prompt]] +id = 776 +text = "Top-down close view of a vintage typewriter keyboard. Each keycap shows different wear and slight oily sheen; gaps reveal hints of springs and linkages. Metal frame has oxidation spots and fine scratches with consistent reflections. Add a header “打字” in a monospaced, typewriter-like font." + +[[frozen_prompt]] +id = 801 +text = "Outdoor display of rock balancing: multiple stones touch at tiny points yet the center-of-mass line is plausible. A frosted aluminum placard reads “平衡展 Balance Exhibit” with bilingual stacking, large title and smaller description, strong whitespace. Requirements: crisp stone grains and weathering, consistent shadow direction, soft glare on metal without noise, legible undistorted text; mood quiet and meditative." + +[[frozen_prompt]] +id = 826 +text = "Artist residency poster: wall with sticky notes and sketches, central poster title “RESIDENCY / 驻留计划”, three lines below with date/location/how to apply (bilingual). Require realistic curled corners and tape tension; fully legible text; clear hierarchy." + +[[frozen_prompt]] +id = 851 +text = "A skier descends a steep slope at high speed, knees slightly bent, body leaning forward, ski poles tapping the snow for balance, leaving clear tracks on the surface." + +[[frozen_prompt]] +id = 876 +text = "With piercing eyes beneath his safety helmet, the engineer bent down close to the steel beam, carefully scanning each weld with a testing instrument, his work clothes covered in dust." + +[[frozen_prompt]] +id = 901 +text = "Realistic naturalistic photography, a sea otter lying on its back on calm water, holding a shell with its front paws; wet fur texture realistic with glossy sheen on water-adjacent fur layer, gentle water ripples spreading outward; distant bay blurred into soft background; overall serene atmosphere, early morning light from upper side." + +[[frozen_prompt]] +id = 926 +text = "Tang Dynasty flying Apsaras in Dunhuang mural style: Ribbons dance in S-curves behind the dancer, who holds a pipa and flower wreath. Skin rendered in mineral azurite green, robes adorned with cinnabar and gold leaf. Figures drawn following Dunhuang mural three-curve convention with S-shaped stance, background with clouds and lotus pedestals, mineral colors antique and rich, overall presenting the artistic elegance of the High Tang." + +[[frozen_prompt]] +id = 951 +text = "Interior of void library: boundless deep space with no walls or ceiling, bookshelves extend endlessly from nearby floor into distant darkness, each shelf lined with neatly arranged books; an open book emits faint light containing tiny planets and nebulae floating inside; a reader in a snug spacesuit with safety harness hangs in mid-air in a relaxed posture. Virtual scene has extreme depth perception, no typical AI artifacts like repetitive textures or floating anomalies, composition uses central bookshelves as leading lines, lighting is dim and mysterious." + +[[frozen_prompt]] +id = 976 +text = "Stepping into the Forest of Echoes, hundreds of acrylic rods stand like a harp of light; a light touch of the fingertip instantly triggers a pressure sensor that emits ring-shaped light waves, each with a different frequency yet spreading harmoniously—technological logic here is transformed into a walkable poem." diff --git a/crates/bounty-challenge-task/Cargo.toml b/crates/bounty-challenge-task/Cargo.toml new file mode 100644 index 000000000..1a8912be5 --- /dev/null +++ b/crates/bounty-challenge-task/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bounty-challenge-task" +description = "Bounty challenge identity, pairing challenge strings, domain tags" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +hex = "0.4" +keystore = { path = "../keystore" } +schnorrkel = { version = "0.11.5", default-features = false, features = ["std", "getrandom"] } +serde = { version = "1", features = ["derive"] } +thiserror = "2" + +[lints] +workspace = true diff --git a/crates/bounty-challenge-task/src/lib.rs b/crates/bounty-challenge-task/src/lib.rs new file mode 100644 index 000000000..ade34778e --- /dev/null +++ b/crates/bounty-challenge-task/src/lib.rs @@ -0,0 +1,376 @@ +//! Bounty challenge identity and miner hotkey pairing. +//! +//! ```text +//! challenge_id = "bounty" +//! scoring_version = 1 +//! pair prefix = cortex-bounty-v1|{account_id}|{nonce}|{exp} +//! ``` +//! +//! Pairing is signed with the miner's Bittensor hotkey (sr25519, Substrate +//! context). Chat never asks for a mnemonic. The unguessable Chat inject +//! command is env-only (`BOUNTY_CHAT_COMMAND`). + +#![forbid(unsafe_code)] +#![allow(clippy::doc_markdown, clippy::missing_errors_doc)] + +use keystore::{ss58_decode, ss58_encode, BITTENSOR_SS58_PREFIX, KEY_LEN}; +use schnorrkel::{signing_context, ExpansionMode, MiniSecretKey, PublicKey, Signature}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +/// Normative challenge id (trust-root / leaf `challenge_id` string). +pub const CHALLENGE_ID: &str = "bounty"; + +/// UTF-8 bytes of [`CHALLENGE_ID`]. +pub const CHALLENGE_ID_BYTES: &[u8] = b"bounty"; + +/// Live `challenge_scoring_version` (precision displacement vs champion). +pub const SCORING_VERSION: u16 = 1; + +/// Integer score lattice max (same scale as other challenges). +pub const SCORE_MAX: u64 = 1_000_000; + +/// Domain tag for pairing session claims (control-plane HMAC, not miner sigs). +pub const SESSION_DOMAIN: &[u8] = b"base-bounty-session-v1"; + +/// Domain tag for report fingerprints. +pub const REPORT_DOMAIN: &[u8] = b"base-bounty-report-v1"; + +/// Pairing challenge prefix. Wire format: `{PREFIX}|{account_id}|{nonce}|{exp}`. +pub const PAIR_PREFIX: &str = "cortex-bounty-v1"; + +/// Default pairing expiry, seconds from now. +pub const DEFAULT_PAIR_TTL_SECS: u64 = 15 * 60; + +/// Terms miners must accept before pairing (blocking). +pub const TERMS_TEXT: &str = "By pairing a Bittensor hotkey to a Cortex Chat account \ +for Bounty Challenge, you accept that this dedicated mining account, its logs, \ +and its conversations may be used for research, to fix product and backend bugs, \ +and to remunerate (or penalize) the bound miner hotkey. Do not pair a private \ +personal account."; + +/// Pairing / signature errors. Never embed secrets or mnemonics. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum PairError { + /// Account id empty or illegal. + #[error("invalid account_id")] + InvalidAccount, + /// Nonce empty or illegal. + #[error("invalid nonce")] + InvalidNonce, + /// Expiry not a unix-seconds integer, or already elapsed. + #[error("invalid or expired pairing window")] + InvalidExp, + /// Challenge string does not match the canonical form. + #[error("malformed pairing challenge")] + MalformedChallenge, + /// Hotkey is not 64-hex or a valid SS58 address. + #[error("invalid hotkey")] + InvalidHotkey, + /// Signature bytes were malformed. + #[error("invalid signature")] + InvalidSignature, + /// Signature does not match the hotkey over the challenge string. + #[error("signature verification failed")] + VerificationFailed, + /// Mini-secret rejected by schnorrkel. + #[error("invalid secret")] + InvalidSecret, +} + +/// Canonical pairing challenge. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PairChallenge { + /// Cortex Chat account id (dedicated mining account). + pub account_id: String, + /// One-time hex nonce. + pub nonce: String, + /// Unix-seconds expiry. + pub exp: u64, +} + +impl PairChallenge { + /// Build `cortex-bounty-v1|{account_id}|{nonce}|{exp}`. + pub fn encode(&self) -> Result { + validate_account_id(&self.account_id)?; + validate_nonce(&self.nonce)?; + if self.exp == 0 { + return Err(PairError::InvalidExp); + } + Ok(format!( + "{PAIR_PREFIX}|{}|{}|{}", + self.account_id, self.nonce, self.exp + )) + } + + /// Parse a canonical challenge string. + pub fn parse(raw: &str) -> Result { + let mut parts = raw.splitn(4, '|'); + let prefix = parts.next().ok_or(PairError::MalformedChallenge)?; + let account_id = parts.next().ok_or(PairError::MalformedChallenge)?; + let nonce = parts.next().ok_or(PairError::MalformedChallenge)?; + let exp_s = parts.next().ok_or(PairError::MalformedChallenge)?; + if prefix != PAIR_PREFIX { + return Err(PairError::MalformedChallenge); + } + validate_account_id(account_id)?; + validate_nonce(nonce)?; + let exp: u64 = exp_s.parse().map_err(|_| PairError::InvalidExp)?; + if exp == 0 { + return Err(PairError::InvalidExp); + } + Ok(Self { + account_id: account_id.to_owned(), + nonce: nonce.to_owned(), + exp, + }) + } + + /// Reject when `now_unix >= exp`. + pub fn ensure_fresh(&self, now_unix: u64) -> Result<(), PairError> { + if now_unix >= self.exp { + return Err(PairError::InvalidExp); + } + Ok(()) + } +} + +/// `BOUNTY_CHAT_COMMAND` when set. Empty / missing → `None` (docs print a placeholder). +#[must_use] +pub fn chat_command_from_env() -> Option { + std::env::var("BOUNTY_CHAT_COMMAND") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} + +/// Placeholder shown when the inject command is not configured. Never a live token. +pub const CHAT_COMMAND_PLACEHOLDER: &str = ""; + +/// Display the inject command or the public placeholder. +#[must_use] +pub fn chat_command_display() -> String { + chat_command_from_env().unwrap_or_else(|| CHAT_COMMAND_PLACEHOLDER.to_owned()) +} + +/// `BOUNTY_BACKEND_PUBLIC_URL` when set. Empty / missing → `None` (skip / sim). +/// +/// Cortex **reads** the Chat backend public feed. It does not serve one. +/// Never bake a host into git. +#[must_use] +pub fn backend_public_url() -> Option { + std::env::var("BOUNTY_BACKEND_PUBLIC_URL") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} + +/// Accept `[A-Za-z0-9._:-]` up to 128 chars. +pub fn validate_account_id(id: &str) -> Result<(), PairError> { + if id.is_empty() || id.len() > 128 { + return Err(PairError::InvalidAccount); + } + if !id + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | ':' | '-')) + { + return Err(PairError::InvalidAccount); + } + Ok(()) +} + +/// Hex nonce, 16..=64 chars. +pub fn validate_nonce(nonce: &str) -> Result<(), PairError> { + if nonce.len() < 16 || nonce.len() > 64 || !nonce.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(PairError::InvalidNonce); + } + Ok(()) +} + +/// Parse SS58 or 64-hex into a 32-byte hotkey. +pub fn parse_hotkey(raw: &str) -> Result<[u8; KEY_LEN], PairError> { + let t = raw.trim(); + if t.is_empty() { + return Err(PairError::InvalidHotkey); + } + if let Ok((bytes, _)) = ss58_decode(t) { + return Ok(bytes); + } + let hex_s = t.trim_start_matches("0x"); + if hex_s.len() != 64 || !hex_s.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(PairError::InvalidHotkey); + } + let bytes = hex::decode(hex_s).map_err(|_| PairError::InvalidHotkey)?; + <[u8; KEY_LEN]>::try_from(bytes).map_err(|_| PairError::InvalidHotkey) +} + +/// Encode a hotkey as Bittensor SS58 (prefix 42). +#[must_use] +pub fn hotkey_ss58(hotkey: &[u8; KEY_LEN]) -> String { + ss58_encode(hotkey, BITTENSOR_SS58_PREFIX) +} + +/// Encode a hotkey as lowercase 64-hex. +#[must_use] +pub fn hotkey_hex(hotkey: &[u8; KEY_LEN]) -> String { + hex::encode(hotkey) +} + +/// Derive the 32-byte public key from a mini-secret. +pub fn public_from_mini_secret(secret: &[u8; KEY_LEN]) -> Result<[u8; KEY_LEN], PairError> { + let mini = MiniSecretKey::from_bytes(secret).map_err(|_| PairError::InvalidSecret)?; + Ok(mini.expand(ExpansionMode::Ed25519).to_public().to_bytes()) +} + +/// Sign the challenge string with a 32-byte mini-secret (Substrate sr25519). +pub fn sign_pair_challenge(secret: &[u8; KEY_LEN], challenge: &str) -> Result<[u8; 64], PairError> { + let mini = MiniSecretKey::from_bytes(secret).map_err(|_| PairError::InvalidSecret)?; + let keypair = mini.expand(ExpansionMode::Ed25519).to_keypair(); + let ctx = signing_context(b"substrate"); + Ok(keypair.sign(ctx.bytes(challenge.as_bytes())).to_bytes()) +} + +/// Verify a Substrate-context sr25519 signature over the challenge string. +pub fn verify_pair_signature( + public: &[u8; KEY_LEN], + challenge: &str, + signature: &[u8], +) -> Result<(), PairError> { + if signature.len() != 64 { + return Err(PairError::InvalidSignature); + } + let pk = PublicKey::from_bytes(public).map_err(|_| PairError::InvalidHotkey)?; + let sig = Signature::from_bytes(signature).map_err(|_| PairError::InvalidSignature)?; + let ctx = signing_context(b"substrate"); + pk.verify(ctx.bytes(challenge.as_bytes()), &sig) + .map_err(|_| PairError::VerificationFailed) +} + +/// Decode a 128-hex or raw-64 signature. +pub fn parse_signature(raw: &str) -> Result<[u8; 64], PairError> { + let t = raw.trim().trim_start_matches("0x"); + if t.len() != 128 || !t.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(PairError::InvalidSignature); + } + let bytes = hex::decode(t).map_err(|_| PairError::InvalidSignature)?; + <[u8; 64]>::try_from(bytes).map_err(|_| PairError::InvalidSignature) +} + +/// One-time pairing code pasted into Cortex Chat (not a mnemonic). +#[must_use] +pub fn pairing_code(challenge: &str, signature_hex: &str, hotkey: &str) -> String { + format!("{challenge}|{signature_hex}|{hotkey}") +} + +/// Parse [`pairing_code`] back into challenge + signature + hotkey. +pub fn parse_pairing_code( + code: &str, +) -> Result<(PairChallenge, [u8; 64], [u8; KEY_LEN]), PairError> { + let mut parts = code.rsplitn(3, '|'); + let hotkey_s = parts.next().ok_or(PairError::MalformedChallenge)?; + let sig_s = parts.next().ok_or(PairError::MalformedChallenge)?; + let challenge_s = parts.next().ok_or(PairError::MalformedChallenge)?; + let challenge = PairChallenge::parse(challenge_s)?; + let signature = parse_signature(sig_s)?; + let hotkey = parse_hotkey(hotkey_s)?; + Ok((challenge, signature, hotkey)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dummy_secret() -> [u8; KEY_LEN] { + let mut s = [0x11u8; KEY_LEN]; + s[0] = 0x42; + s + } + + fn dummy_public() -> [u8; KEY_LEN] { + let mini = MiniSecretKey::from_bytes(&dummy_secret()).expect("mini"); + mini.expand(ExpansionMode::Ed25519).to_public().to_bytes() + } + + #[test] + fn challenge_id_is_bounty() { + assert_eq!(CHALLENGE_ID, "bounty"); + assert_eq!(CHALLENGE_ID_BYTES, b"bounty"); + assert_ne!(CHALLENGE_ID, "relearn"); + } + + #[test] + fn pair_round_trip() { + let c = PairChallenge { + account_id: "acct-miner-1".into(), + nonce: "aabbccddeeff0011".into(), + exp: 1_900_000_000, + }; + let s = c.encode().expect("encode"); + assert!(s.starts_with("cortex-bounty-v1|")); + assert_eq!(PairChallenge::parse(&s).expect("parse"), c); + } + + #[test] + fn dummy_sr25519_pair_verifies() { + let pk = dummy_public(); + let c = PairChallenge { + account_id: "acct-test".into(), + nonce: "0123456789abcdef".into(), + exp: 2_000_000_000, + }; + let challenge = c.encode().expect("encode"); + let sig = sign_pair_challenge(&dummy_secret(), &challenge).expect("sign"); + verify_pair_signature(&pk, &challenge, &sig).expect("verify"); + assert!( + verify_pair_signature(&pk, "cortex-bounty-v1|other|0123456789abcdef|1", &sig).is_err() + ); + } + + #[test] + fn pairing_code_round_trip() { + let pk = dummy_public(); + let ss58 = hotkey_ss58(&pk); + let c = PairChallenge { + account_id: "acct-2".into(), + nonce: "deadbeefdeadbeef".into(), + exp: 2_100_000_000, + }; + let challenge = c.encode().expect("encode"); + let sig = sign_pair_challenge(&dummy_secret(), &challenge).expect("sign"); + let code = pairing_code(&challenge, &hex::encode(sig), &ss58); + let (parsed, sig2, hk) = parse_pairing_code(&code).expect("parse code"); + assert_eq!(parsed, c); + assert_eq!(sig2, sig); + assert_eq!(hk, pk); + verify_pair_signature(&hk, &challenge, &sig2).expect("verify"); + } + + #[test] + fn expired_challenge_rejected() { + let c = PairChallenge { + account_id: "acct".into(), + nonce: "0123456789abcdef".into(), + exp: 10, + }; + assert_eq!(c.ensure_fresh(10), Err(PairError::InvalidExp)); + assert!(c.ensure_fresh(9).is_ok()); + } + + #[test] + fn chat_command_env_only() { + assert!(chat_command_from_env().is_none()); + assert_eq!(chat_command_display(), CHAT_COMMAND_PLACEHOLDER); + assert!(!CHAT_COMMAND_PLACEHOLDER.contains("/miner")); + } + + #[test] + fn backend_public_url_env_only() { + assert!(backend_public_url().is_none()); + } + + #[test] + fn terms_require_dedicated_account() { + assert!(TERMS_TEXT.contains("dedicated mining account")); + assert!(TERMS_TEXT.contains("remunerate")); + } +} diff --git a/crates/bounty-challenge/Cargo.toml b/crates/bounty-challenge/Cargo.toml new file mode 100644 index 000000000..b64e01648 --- /dev/null +++ b/crates/bounty-challenge/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "bounty-challenge" +description = "Bounty orchestrator: pair, reports, adjudicate, D24 emit" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +bundle = { path = "../bundle" } +bounty-challenge-task = { path = "../bounty-challenge-task" } +bounty-http = { path = "../bounty-http" } +bounty-score = { path = "../bounty-score" } +bounty-store = { path = "../bounty-store" } +challenge-common = { path = "../challenge-common" } +hex = "0.4" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +thiserror = "2" + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/crates/bounty-challenge/src/backend.rs b/crates/bounty-challenge/src/backend.rs new file mode 100644 index 000000000..e2b1965c8 --- /dev/null +++ b/crates/bounty-challenge/src/backend.rs @@ -0,0 +1,145 @@ +//! Read-only client for the CortexLM/backend public Bounty feed. +//! +//! Cortex never serves `/v1/public/*`. It fetches: +//! `{BOUNTY_BACKEND_PUBLIC_URL}/v1/bounty/public/leaderboard` +//! `{BOUNTY_BACKEND_PUBLIC_URL}/v1/bounty/public/reports` +//! +//! Missing URL → skip / sim. No host is baked in. + +use bounty_challenge_task::backend_public_url; +use bounty_score::{parse_leaderboard, parse_reports, PublicSnapshot}; +use thiserror::Error; + +/// Fetch / parse errors. Never embed secrets or hosts from env into Display +/// beyond the operator-configured base (trimmed). +#[derive(Debug, Error)] +pub enum BackendError { + /// `BOUNTY_BACKEND_PUBLIC_URL` unset — caller should skip / sim. + #[error("BOUNTY_BACKEND_PUBLIC_URL unset")] + Unset, + /// HTTP transport or status. + #[error("backend public fetch failed")] + Fetch, + /// JSON did not match the public DTO. + #[error("backend public json: {0}")] + Json(String), +} + +/// Join `{base}/v1/bounty/public/{tail}` without inventing a host. +#[must_use] +pub fn public_path(base: &str, tail: &str) -> String { + let b = base.trim().trim_end_matches('/'); + format!("{b}/v1/bounty/public/{tail}") +} + +/// Load a snapshot from a configured backend URL. +/// +/// # Errors +/// [`BackendError::Unset`] when the env is empty (CI-safe skip). +pub async fn fetch_public_snapshot(base: Option<&str>) -> Result { + let url = match base { + Some(u) if !u.trim().is_empty() => u.trim().to_owned(), + _ => backend_public_url().ok_or(BackendError::Unset)?, + }; + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(20)) + .build() + .map_err(|_| BackendError::Fetch)?; + let lb_text = get_text(&client, &public_path(&url, "leaderboard")).await?; + let rp_text = get_text(&client, &public_path(&url, "reports")).await?; + let leaderboard = parse_leaderboard(&lb_text).map_err(BackendError::Json)?; + let reports = parse_reports(&rp_text).map_err(BackendError::Json)?; + Ok(PublicSnapshot { + leaderboard, + reports, + }) +} + +async fn get_text(client: &reqwest::Client, url: &str) -> Result { + let resp = client + .get(url) + .send() + .await + .map_err(|_| BackendError::Fetch)?; + if !resp.status().is_success() { + return Err(BackendError::Fetch); + } + resp.text().await.map_err(|_| BackendError::Fetch) +} + +/// Parse a snapshot from two JSON bodies (unit tests / mocks). +pub fn snapshot_from_json( + leaderboard: &str, + reports: &str, +) -> Result { + Ok(PublicSnapshot { + leaderboard: parse_leaderboard(leaderboard).map_err(BackendError::Json)?, + reports: parse_reports(reports).map_err(BackendError::Json)?, + }) +} + +/// Fetch when `BOUNTY_BACKEND_PUBLIC_URL` is set; `Ok(None)` when unset (CI skip). +pub async fn try_fetch_public_snapshot() -> Result, BackendError> { + match fetch_public_snapshot(None).await { + Ok(s) => Ok(Some(s)), + Err(BackendError::Unset) => Ok(None), + Err(e) => Err(e), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bounty_score::score_plan_from_snapshot; + + #[test] + fn public_path_uses_operator_base_only() { + assert_eq!( + public_path("http://127.0.0.1:9", "leaderboard"), + "http://127.0.0.1:9/v1/bounty/public/leaderboard" + ); + assert_eq!( + public_path("http://127.0.0.1:9/", "reports"), + "http://127.0.0.1:9/v1/bounty/public/reports" + ); + } + + #[test] + fn mock_backend_json_scores_hotkeys() { + let lb = r#"{"items":[ + {"hotkey":"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY","valid_count":3}, + {"hotkey":"5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty","valid_count":0} + ]}"#; + let rp = r#"{"items":[ + {"id":"1","hotkey":"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY","status":"valid", + "problem_found":"seal returns 500","adjudicator":"bounty-adjudicator@cortex", + "justification":"reproduced empty-bundle seal","adjudicated_at":"2026-08-30T00:00:00Z", + "created_at":"2026-08-29T00:00:00Z"}, + {"id":"2","hotkey":"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY","status":"valid", + "problem_found":"proxy 502","adjudicator":"bounty-adjudicator@cortex", + "justification":"reproduced on master","adjudicated_at":"2026-08-30T00:00:00Z", + "created_at":"2026-08-29T01:00:00Z"}, + {"id":"3","hotkey":"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY","status":"valid", + "problem_found":"health flap","adjudicator":"bounty-adjudicator@cortex", + "justification":"reproduced","adjudicated_at":"2026-08-30T00:00:00Z", + "created_at":"2026-08-29T02:00:00Z"} + ]}"#; + let snap = snapshot_from_json(lb, rp).expect("mock"); + let plan = score_plan_from_snapshot(&snap); + assert!(plan.champion_hex.is_some()); + assert!(plan.champion_lattice > 0); + assert_eq!(plan.holdouts.len(), 1); + } + + #[tokio::test] + async fn fetch_skips_when_url_unset() { + if backend_public_url().is_some() { + eprintln!("skip unset assertion: BOUNTY_BACKEND_PUBLIC_URL is set"); + return; + } + let err = fetch_public_snapshot(None).await.expect_err("unset"); + assert!(matches!(err, BackendError::Unset)); + let skip = try_fetch_public_snapshot().await.expect("skip"); + assert!(skip.is_none()); + } +} diff --git a/crates/bounty-challenge/src/lib.rs b/crates/bounty-challenge/src/lib.rs new file mode 100644 index 000000000..18e7d77af --- /dev/null +++ b/crates/bounty-challenge/src/lib.rs @@ -0,0 +1,219 @@ +//! Bounty orchestrator helpers: D24 leaf plan + crate re-exports. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::must_use_candidate +)] + +use std::collections::{BTreeMap, BTreeSet}; + +mod backend; + +use bounty_challenge_task::{hotkey_hex, CHALLENGE_ID_BYTES, SCORE_MAX}; +use bounty_score::{ + champion_hold_lattice, lattice_from_precision, score_plan_from_snapshot, MinerHoldout, + PublicScorePlan, PublicSnapshot, +}; +use bounty_store::MemoryStore; +use bundle::{NoScoreReasonCode, ScoreOrAbsence}; +use challenge_common::{emit_signed_leaf_set, Hotkey, LeafEmitError}; + +pub use backend::{ + fetch_public_snapshot, public_path, snapshot_from_json, try_fetch_public_snapshot, BackendError, +}; +pub use bounty_challenge_task::{ + backend_public_url, chat_command_display, CHALLENGE_ID, CHALLENGE_ID_BYTES as BOUNTY_ID_BYTES, + CHAT_COMMAND_PLACEHOLDER, SCORE_MAX as BOUNTY_SCORE_MAX, SCORING_VERSION, TERMS_TEXT, +}; +pub use bounty_http::{bounty_router, hash_admin_token, AppState}; +pub use bounty_store::MemoryStore as BountyStore; + +/// Build a D24-complete score map. +/// +/// Champion (if any) gets a positive lattice from holdout precision. +/// Miners with a net malicious penalty get `InvalidResponse` (burn toward uid 0). +/// Everyone else is explicit `NoScore` (unmatched emission burns to uid 0). +pub fn emission_scores( + expected: &BTreeSet, + champion_hotkey: Option, + champion_lattice: u64, + holdouts: &BTreeMap, +) -> BTreeMap { + expected + .iter() + .map(|h| { + let s = match champion_hotkey { + Some(c) if c == *h && champion_lattice > 0 => ScoreOrAbsence::Score { + value: champion_lattice.min(SCORE_MAX), + }, + _ => { + if holdouts.get(h).is_some_and(|r| r.net_credit() < 0) { + ScoreOrAbsence::NoScore { + reason: NoScoreReasonCode::InvalidResponse, + } + } else { + ScoreOrAbsence::NoScore { + reason: NoScoreReasonCode::NotAttempted, + } + } + } + }; + (*h, s) + }) + .collect() +} + +/// Sign the exact-E leaf set for this epoch. +pub fn emit_epoch( + secret: &[u8; 32], + epoch: u64, + expected: &BTreeSet, + champion_hotkey: Option, + champion_lattice: u64, + holdouts: &BTreeMap, +) -> Result, LeafEmitError> { + let scores = emission_scores(expected, champion_hotkey, champion_lattice, holdouts); + emit_signed_leaf_set(secret, CHALLENGE_ID_BYTES, epoch, expected, &scores) +} + +/// Lattice for the current store champion. +pub fn live_champion_lattice(store: &MemoryStore, champion_hotkey: Option<&str>) -> u64 { + let Some(hk) = champion_hotkey else { + return 0; + }; + let Ok(h) = store.holdout(hk) else { + return 0; + }; + if let Some(p) = h.precision_bps() { + if h.decided() > 0 && h.net_credit() >= 0 { + return lattice_from_precision(p).max(champion_hold_lattice() / 4); + } + } + champion_hold_lattice() +} + +/// Parse a 64-hex hotkey. +pub fn parse_hotkey_hex(hex_s: &str) -> Option { + let t = hex_s.trim().trim_start_matches("0x"); + let bytes = hex::decode(t).ok()?; + <[u8; 32]>::try_from(bytes).ok() +} + +/// Map a backend public snapshot onto D24 scores for `expected`. +#[must_use] +pub fn emission_from_public_snapshot( + expected: &BTreeSet, + snap: &PublicSnapshot, +) -> (PublicScorePlan, BTreeMap) { + let plan = score_plan_from_snapshot(snap); + let champ = plan.champion_hex.as_deref().and_then(parse_hotkey_hex); + let mut holdouts = BTreeMap::new(); + for h in expected { + let hex_s = hotkey_hex(h); + if let Some(row) = plan.holdouts.get(&hex_s) { + holdouts.insert(*h, row.clone()); + } + } + let scores = emission_scores(expected, champ, plan.champion_lattice, &holdouts); + (plan, scores) +} + +#[cfg(test)] +mod tests { + use super::*; + use challenge_common::public_key_from_secret; + + fn sk() -> [u8; 32] { + let mut s = [9u8; 32]; + s[0] = 2; + s + } + + #[test] + fn d24_covers_every_hotkey() { + let a = [1u8; 32]; + let b = [2u8; 32]; + let mut e = BTreeSet::new(); + e.insert(a); + e.insert(b); + let mut holdouts = BTreeMap::new(); + holdouts.insert( + b, + MinerHoldout { + valid: 0, + already_fixed: 0, + malicious: 3, + duplicate: 0, + }, + ); + let leaves = emit_epoch(&sk(), 9, &e, Some(a), 12_000, &holdouts).expect("emit"); + assert_eq!(leaves.len(), 2); + assert!(matches!( + leaves[&a].score_or_absence, + ScoreOrAbsence::Score { value: 12_000 } + )); + assert!(matches!( + leaves[&b].score_or_absence, + ScoreOrAbsence::NoScore { + reason: NoScoreReasonCode::InvalidResponse + } + )); + let pk = public_key_from_secret(&sk()).expect("pk"); + for leaf in leaves.values() { + challenge_common::verify_leaf_sig(leaf, &pk).expect("sig"); + } + } + + #[test] + fn unmatched_is_not_attempted() { + let a = [1u8; 32]; + let mut e = BTreeSet::new(); + e.insert(a); + let leaves = emit_epoch(&sk(), 1, &e, None, 0, &BTreeMap::new()).expect("emit"); + assert!(matches!( + leaves[&a].score_or_absence, + ScoreOrAbsence::NoScore { + reason: NoScoreReasonCode::NotAttempted + } + )); + } + + #[test] + fn emission_maps_backend_public_snapshot() { + let alice = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"; + let snap = snapshot_from_json( + &format!(r#"{{"items":[{{"hotkey":"{alice}","valid_count":3}}]}}"#), + &format!( + r#"{{"items":[ + {{"id":"1","hotkey":"{alice}","status":"valid", + "problem_found":"seal 500","adjudicator":"bounty-adjudicator@cortex", + "justification":"reproduced","adjudicated_at":"2026-08-30T00:00:00Z", + "created_at":"2026-08-29T00:00:00Z"}}, + {{"id":"2","hotkey":"{alice}","status":"valid", + "problem_found":"proxy 502","adjudicator":"bounty-adjudicator@cortex", + "justification":"reproduced","adjudicated_at":"2026-08-30T00:00:00Z", + "created_at":"2026-08-29T01:00:00Z"}}, + {{"id":"3","hotkey":"{alice}","status":"valid", + "problem_found":"health flap","adjudicator":"bounty-adjudicator@cortex", + "justification":"reproduced","adjudicated_at":"2026-08-30T00:00:00Z", + "created_at":"2026-08-29T02:00:00Z"}} + ]}}"# + ), + ) + .expect("mock"); + let hk = parse_hotkey_hex(&hotkey_hex( + &bounty_challenge_task::parse_hotkey(alice).expect("alice"), + )) + .expect("hex"); + let mut expected = BTreeSet::new(); + expected.insert(hk); + let (plan, scores) = emission_from_public_snapshot(&expected, &snap); + assert!(plan.champion_hex.is_some()); + assert!(matches!( + scores[&hk], + ScoreOrAbsence::Score { value } if value > 0 + )); + } +} diff --git a/crates/bounty-http/Cargo.toml b/crates/bounty-http/Cargo.toml new file mode 100644 index 000000000..94f1d1c7f --- /dev/null +++ b/crates/bounty-http/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "bounty-http" +description = "Bounty HTTP surface: pair, reports, admin adjudicate" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"] } +bounty-challenge-task = { path = "../bounty-challenge-task" } +bounty-score = { path = "../bounty-score" } +bounty-store = { path = "../bounty-store" } +hex = "0.4" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt", "sync"] } + +[dev-dependencies] +http-body-util = "0.1" +tower = { version = "0.5", features = ["util"] } + +[lints] +workspace = true diff --git a/crates/bounty-http/src/lib.rs b/crates/bounty-http/src/lib.rs new file mode 100644 index 000000000..45a3e4f9c --- /dev/null +++ b/crates/bounty-http/src/lib.rs @@ -0,0 +1,530 @@ +//! Bounty HTTP API (master-only, **internal ingest**). +//! +//! Cortex does **not** serve a public leaderboard/report API. Public consumers +//! hit CortexLM/backend (`GET /v1/bounty/public/leaderboard|reports`). This +//! service only reads that feed for scoring (see `bounty-challenge::backend`). +//! +//! Gateway proxies `/challenge/bounty/*` onto this service: +//! +//! ```text +//! GET /health +//! GET /v1/status +//! POST /v1/pair verify hotkey sig, bind account, session claim +//! POST /v1/reports bug report + session (optional X-Lium-Api-Key) +//! GET /v1/reports internal ingest list (not a public board) +//! GET /v1/reports/{id} +//! POST /v1/admin/adjudicate valid | already_fixed_not_prod | invalid_malicious | duplicate +//! ``` +//! +//! There is no `/v1/public/*` route. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::must_use_candidate, + clippy::items_after_statements, + clippy::too_many_lines +)] + +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use bounty_challenge_task::{ + backend_public_url, hotkey_hex, parse_hotkey, parse_signature, verify_pair_signature, + PairChallenge, CHALLENGE_ID, SCORE_MAX, SCORING_VERSION, TERMS_TEXT, +}; +use bounty_score::Adjudication; +use bounty_store::{report_fingerprint, MemoryStore, Report, ReportState}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Shared HTTP state. +#[derive(Clone)] +pub struct AppState { + /// Pairings + reports. + pub store: MemoryStore, + /// HMAC secret for session claims. + pub session_secret: Arc>, + /// Operator bearer hashes (sha256 hex). Empty → admin 503. + pub admin_hashes: Arc>, +} + +/// Build the router. +pub fn bounty_router(state: AppState) -> Router { + Router::new() + .route("/health", get(health)) + .route("/v1/status", get(status)) + .route("/v1/pair", post(pair)) + .route("/v1/reports", post(submit_report).get(list_reports)) + .route("/v1/reports/{id}", get(get_report)) + .route("/v1/admin/adjudicate", post(adjudicate)) + .with_state(state) +} + +async fn health() -> impl IntoResponse { + Json(serde_json::json!({ + "ok": true, + "challenge_id": CHALLENGE_ID, + "scoring_version": SCORING_VERSION, + })) +} + +async fn status(State(st): State) -> impl IntoResponse { + let champ = st.store.champion_hotkey().ok().flatten(); + Json(serde_json::json!({ + "challenge_id": CHALLENGE_ID, + "scoring_version": SCORING_VERSION, + "score_max": SCORE_MAX, + "champion_hotkey": champ, + "backend_public_configured": backend_public_url().is_some(), + "terms": TERMS_TEXT, + })) +} + +#[derive(Debug, Deserialize)] +struct PairBody { + account_id: String, + hotkey: String, + nonce: String, + exp: u64, + signature: String, + terms_accepted: bool, +} + +#[derive(Debug, Serialize)] +struct PairResp { + session: String, + account_id: String, + miner_hotkey: String, + session_id: String, +} + +async fn pair( + State(st): State, + Json(body): Json, +) -> Result)> { + if !body.terms_accepted { + return Err(err(StatusCode::FORBIDDEN, "terms_required")); + } + let challenge = PairChallenge { + account_id: body.account_id.clone(), + nonce: body.nonce.clone(), + exp: body.exp, + }; + let encoded = challenge + .encode() + .map_err(|e| err(StatusCode::BAD_REQUEST, &e.to_string()))?; + let now = unix_now(); + challenge + .ensure_fresh(now) + .map_err(|e| err(StatusCode::BAD_REQUEST, &e.to_string()))?; + let hotkey = + parse_hotkey(&body.hotkey).map_err(|e| err(StatusCode::BAD_REQUEST, &e.to_string()))?; + let sig = parse_signature(&body.signature) + .map_err(|e| err(StatusCode::BAD_REQUEST, &e.to_string()))?; + verify_pair_signature(&hotkey, &encoded, &sig) + .map_err(|e| err(StatusCode::UNAUTHORIZED, &e.to_string()))?; + let hk = hotkey_hex(&hotkey); + let claim = st + .store + .bind_pair(&body.account_id, &hk, &body.nonce, now, &st.session_secret) + .map_err(|e| err(StatusCode::CONFLICT, &e.to_string()))?; + Ok(( + StatusCode::CREATED, + Json(PairResp { + session: claim.token, + account_id: claim.account_id, + miner_hotkey: claim.miner_hotkey, + session_id: claim.session_id, + }), + )) +} + +#[derive(Debug, Deserialize)] +struct ReportBody { + session: String, + hotkey: Option, + title: String, + body: String, + repro_steps: Option, +} + +#[derive(Debug, Serialize)] +struct ReportResp { + id: String, + miner_hotkey: String, + state: ReportState, + fingerprint: String, +} + +async fn submit_report( + State(st): State, + headers: HeaderMap, + Json(body): Json, +) -> Result)> { + // Miner BYOK: accepted and never logged. Absence is OK (no live Lium). + let _lium_present = headers + .get("x-lium-api-key") + .and_then(|v| v.to_str().ok()) + .is_some_and(|s| !s.is_empty()); + + let pairing = st + .store + .lookup_session(&body.session, &st.session_secret) + .map_err(|_| err(StatusCode::UNAUTHORIZED, "invalid_session"))?; + if let Some(ref hk) = body.hotkey { + let parsed = parse_hotkey(hk).map_err(|e| err(StatusCode::BAD_REQUEST, &e.to_string()))?; + if hotkey_hex(&parsed) != pairing.miner_hotkey { + return Err(err(StatusCode::FORBIDDEN, "hotkey_mismatch")); + } + } + if body.title.trim().is_empty() || body.body.trim().is_empty() { + return Err(err(StatusCode::BAD_REQUEST, "title_and_body_required")); + } + let fingerprint = report_fingerprint(&body.title, &body.body); + let row = Report { + id: String::new(), + miner_hotkey: pairing.miner_hotkey, + account_id: pairing.account_id, + title: body.title, + body: body.body, + repro_steps: body.repro_steps.unwrap_or_default(), + fingerprint, + state: ReportState::Pending, + adjudication: None, + duplicate_of: None, + champion_verdict: None, + }; + let row = st + .store + .insert_report(row) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + Ok(( + StatusCode::CREATED, + Json(ReportResp { + id: row.id, + miner_hotkey: row.miner_hotkey, + state: row.state, + fingerprint: row.fingerprint, + }), + )) +} + +async fn list_reports(State(st): State) -> impl IntoResponse { + let rows = st.store.list_reports().unwrap_or_default(); + Json(serde_json::json!({ "items": rows })) +} + +async fn get_report( + State(st): State, + Path(id): Path, +) -> Result)> { + let row = st + .store + .get_report(&id) + .map_err(|_| err(StatusCode::NOT_FOUND, "not_found"))?; + Ok(Json(row)) +} + +#[derive(Debug, Deserialize)] +struct AdjudicateBody { + report_id: String, + verdict: Adjudication, + duplicate_of: Option, +} + +async fn adjudicate( + State(st): State, + headers: HeaderMap, + Json(body): Json, +) -> Result)> { + if st.admin_hashes.is_empty() { + return Err(err(StatusCode::SERVICE_UNAVAILABLE, "auth_unconfigured")); + } + if !admin_ok(&headers, &st.admin_hashes) { + return Err(err(StatusCode::UNAUTHORIZED, "unauthorized")); + } + let row = st + .store + .adjudicate(&body.report_id, body.verdict, body.duplicate_of) + .map_err(|e| { + let code = if e.to_string().contains("unknown") { + StatusCode::NOT_FOUND + } else { + StatusCode::CONFLICT + }; + err(code, &e.to_string()) + })?; + Ok(Json(row)) +} + +fn admin_ok(headers: &HeaderMap, hashes: &[String]) -> bool { + let Some(raw) = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + else { + return false; + }; + let token = raw.strip_prefix("Bearer ").unwrap_or(raw).trim(); + if token.is_empty() { + return false; + } + let mut h = Sha256::new(); + h.update(token.as_bytes()); + let got = hex::encode(h.finalize()); + hashes.iter().any(|x| x == &got) +} + +fn err(code: StatusCode, msg: &str) -> (StatusCode, Json) { + (code, Json(serde_json::json!({ "error": msg }))) +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) +} + +/// Hash an admin token the same way the server does. +#[must_use] +pub fn hash_admin_token(token: &str) -> String { + let mut h = Sha256::new(); + h.update(token.as_bytes()); + hex::encode(h.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::http::Request; + use bounty_challenge_task::{ + hotkey_ss58, pairing_code, public_from_mini_secret, sign_pair_challenge, PairChallenge, + }; + use http_body_util::BodyExt; + use tower::ServiceExt; + + fn dummy_secret() -> [u8; 32] { + let mut s = [0x11u8; 32]; + s[0] = 0x42; + s + } + + fn dummy_public() -> [u8; 32] { + public_from_mini_secret(&dummy_secret()).expect("pk") + } + + fn app() -> (Router, String) { + let token = "op-test-token"; + let router = bounty_router(AppState { + store: MemoryStore::new(), + session_secret: Arc::new(b"test-session-secret".to_vec()), + admin_hashes: Arc::new(vec![hash_admin_token(token)]), + }); + (router, token.to_owned()) + } + + async fn json_req( + app: Router, + method: &str, + uri: &str, + body: serde_json::Value, + auth: Option<&str>, + ) -> (StatusCode, serde_json::Value) { + let mut b = Request::builder().method(method).uri(uri); + if let Some(a) = auth { + b = b.header(axum::http::header::AUTHORIZATION, format!("Bearer {a}")); + } + let req = b + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("req"); + let resp = app.oneshot(req).await.expect("resp"); + let status = resp.status(); + let bytes = resp.into_body().collect().await.expect("body").to_bytes(); + let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap_or(serde_json::json!({})); + (status, v) + } + + fn pair_payload(exp: u64) -> serde_json::Value { + let pk = dummy_public(); + let c = PairChallenge { + account_id: "acct-http".into(), + nonce: "0123456789abcdef".into(), + exp, + }; + let challenge = c.encode().expect("enc"); + let sig = sign_pair_challenge(&dummy_secret(), &challenge).expect("sign"); + let _code = pairing_code(&challenge, &hex::encode(sig), &hotkey_ss58(&pk)); + serde_json::json!({ + "account_id": c.account_id, + "hotkey": hotkey_ss58(&pk), + "nonce": c.nonce, + "exp": c.exp, + "signature": hex::encode(sig), + "terms_accepted": true, + }) + } + + #[tokio::test] + async fn pair_report_adjudicate_happy_path() { + let (app, token) = app(); + let (st, health) = + json_req(app.clone(), "GET", "/health", serde_json::json!({}), None).await; + assert_eq!(st, StatusCode::OK); + assert_eq!(health["challenge_id"], CHALLENGE_ID); + + let exp = unix_now().saturating_add(600); + let (st, paired) = json_req(app.clone(), "POST", "/v1/pair", pair_payload(exp), None).await; + assert_eq!(st, StatusCode::CREATED, "{paired}"); + let session = paired["session"].as_str().expect("session"); + + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/reports", + serde_json::json!({ + "session": session, + "title": "gateway 500 on seal", + "body": "POST /v1/admin/seal returns 500 when bundle is empty", + "repro_steps": "curl the seal route with no leaves", + }), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["state"], "pending"); + let id = created["id"].as_str().expect("id"); + + let (st, adj) = json_req( + app, + "POST", + "/v1/admin/adjudicate", + serde_json::json!({ + "report_id": id, + "verdict": "valid", + }), + Some(&token), + ) + .await; + assert_eq!(st, StatusCode::OK, "{adj}"); + assert_eq!(adj["state"], "valid"); + assert_eq!(adj["adjudication"], "valid"); + } + + #[tokio::test] + async fn pair_requires_terms() { + let (app, _) = app(); + let mut body = pair_payload(unix_now().saturating_add(600)); + body["terms_accepted"] = serde_json::json!(false); + let (st, v) = json_req(app, "POST", "/v1/pair", body, None).await; + assert_eq!(st, StatusCode::FORBIDDEN); + assert_eq!(v["error"], "terms_required"); + } + + #[tokio::test] + async fn pair_rejects_bad_signature() { + let (app, _) = app(); + let mut body = pair_payload(unix_now().saturating_add(600)); + body["signature"] = serde_json::json!("00".repeat(64)); + let (st, _) = json_req(app, "POST", "/v1/pair", body, None).await; + assert_eq!(st, StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn adjudicate_requires_bearer() { + let (app, _) = app(); + let (st, _) = json_req( + app, + "POST", + "/v1/admin/adjudicate", + serde_json::json!({ "report_id": "by_0", "verdict": "valid" }), + None, + ) + .await; + assert_eq!(st, StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn already_fixed_ack_and_malicious_penalty() { + let (app, token) = app(); + let exp = unix_now().saturating_add(600); + let (st, paired) = json_req(app.clone(), "POST", "/v1/pair", pair_payload(exp), None).await; + assert_eq!(st, StatusCode::CREATED, "{paired}"); + let session = paired["session"].as_str().expect("session"); + + let (st, a) = json_req( + app.clone(), + "POST", + "/v1/reports", + serde_json::json!({ + "session": session, + "title": "fixed already", + "body": "this was patched last week", + }), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{a}"); + let (st, adj) = json_req( + app.clone(), + "POST", + "/v1/admin/adjudicate", + serde_json::json!({ + "report_id": a["id"], + "verdict": "already_fixed_not_prod", + }), + Some(&token), + ) + .await; + assert_eq!(st, StatusCode::OK, "{adj}"); + assert_eq!(adj["state"], "already_fixed_not_prod"); + + let (st, b) = json_req( + app.clone(), + "POST", + "/v1/reports", + serde_json::json!({ + "session": session, + "title": "invented crash", + "body": "does not exist", + }), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{b}"); + let (st, adj) = json_req( + app, + "POST", + "/v1/admin/adjudicate", + serde_json::json!({ + "report_id": b["id"], + "verdict": "invalid_malicious", + }), + Some(&token), + ) + .await; + assert_eq!(st, StatusCode::OK, "{adj}"); + assert_eq!(adj["state"], "invalid_malicious"); + } + + #[tokio::test] + async fn does_not_serve_public_leaderboard() { + let (app, _) = app(); + let (st, _) = json_req( + app, + "GET", + "/v1/public/leaderboard", + serde_json::json!({}), + None, + ) + .await; + assert_eq!(st, StatusCode::NOT_FOUND); + } +} diff --git a/crates/bounty-http/tests/pair_smoke.rs b/crates/bounty-http/tests/pair_smoke.rs new file mode 100644 index 000000000..f68f96246 --- /dev/null +++ b/crates/bounty-http/tests/pair_smoke.rs @@ -0,0 +1,44 @@ +//! Live pair smoke. Skips when operator keys / URL are missing so CI stays green. + +#![forbid(unsafe_code)] + +fn skip(reason: &str) { + eprintln!("skip bounty pair smoke: {reason}"); +} + +#[test] +fn pair_smoke_skips_without_keys() { + let url = std::env::var("BOUNTY_PAIR_SMOKE_URL") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()); + let Some(_url) = url else { + skip("BOUNTY_PAIR_SMOKE_URL unset"); + return; + }; + let sk = std::env::var("BOUNTY_PAIR_SMOKE_SK_FILE") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()); + if sk.is_none() { + skip("BOUNTY_PAIR_SMOKE_SK_FILE unset"); + return; + } + // Live POST is operator-only. Presence of both env vars is enough to + // document the hook; CI never sets them. + skip("live pair POST is operator-only; env present but smoke is hook-only"); +} + +#[test] +fn never_prints_secrets() { + for key in [ + "BOUNTY_CHAT_COMMAND", + "BOUNTY_PAIR_SMOKE_SK_FILE", + "LIUM_API_KEY", + ] { + if let Ok(v) = std::env::var(key) { + assert!(!v.is_empty() || v.is_empty()); + let _ = key; + } + } +} diff --git a/crates/bounty-score/Cargo.toml b/crates/bounty-score/Cargo.toml new file mode 100644 index 000000000..8ded91a64 --- /dev/null +++ b/crates/bounty-score/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "bounty-score" +description = "Bounty precision scoring: valid / already-fixed / malicious / duplicate" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +bounty-challenge-task = { path = "../bounty-challenge-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[lints] +workspace = true diff --git a/crates/bounty-score/src/lib.rs b/crates/bounty-score/src/lib.rs new file mode 100644 index 000000000..bc6c72996 --- /dev/null +++ b/crates/bounty-score/src/lib.rs @@ -0,0 +1,335 @@ +//! Precision scoring for Bounty Challenge. +//! +//! Miner score is displacement vs the previous bounty champion on a holdout +//! of adjudicated reports. Volume does not help — stuffing junk lowers +//! precision and cannot crown a champion. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::must_use_candidate +)] + +use bounty_challenge_task::SCORE_MAX; +use serde::{Deserialize, Serialize}; + +mod public; +pub use public::{ + holdouts_from_reports, parse_leaderboard, parse_reports, rank_leaderboard, scorable, + score_plan_from_snapshot, LeaderboardRow, PublicReport, PublicScorePlan, PublicSnapshot, + PublicStatus, +}; + +/// Credit applied to a valid unique reproducing bug. +pub const VALID_CREDIT: i64 = 100; + +/// Already-fixed-not-prod: ack only. +pub const ALREADY_FIXED_CREDIT: i64 = 0; + +/// Malicious / fabricated / does-not-exist: penalty (burns toward uid 0). +pub const MALICIOUS_CREDIT: i64 = -100; + +/// Duplicate of an open report: no extra reward, no penalty. +pub const DUPLICATE_CREDIT: i64 = 0; + +/// Minimum valid+malicious holdout items before a miner can displace. +pub const MIN_HOLDOUT_DECIDED: u64 = 3; + +/// Operator verdict on one report. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Adjudication { + /// Unique bug that reproduces. Reward. + Valid, + /// Bug already fixed, not yet in prod. Ack only. + AlreadyFixedNotProd, + /// Fabricated, malicious, or does not exist. Penalty. + InvalidMalicious, + /// Duplicate of an open report. Small/no reward, no penalty. + Duplicate, +} + +impl Adjudication { + /// Lattice credit for this verdict (can be negative). + #[must_use] + pub fn credit(self) -> i64 { + match self { + Self::Valid => VALID_CREDIT, + Self::AlreadyFixedNotProd => ALREADY_FIXED_CREDIT, + Self::InvalidMalicious => MALICIOUS_CREDIT, + Self::Duplicate => DUPLICATE_CREDIT, + } + } + + /// True when the item is a precision-decided holdout example. + #[must_use] + pub fn counts_for_precision(self) -> bool { + matches!(self, Self::Valid | Self::InvalidMalicious) + } +} + +/// Per-miner holdout tallies from adjudicated reports. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct MinerHoldout { + /// Unique reproducing bugs. + pub valid: u64, + /// Already-fixed-not-prod acks (ignored for precision). + pub already_fixed: u64, + /// Malicious / fabricated. + pub malicious: u64, + /// Duplicates of an open report. + pub duplicate: u64, +} + +impl MinerHoldout { + /// Apply one verdict. + pub fn record(&mut self, v: Adjudication) { + match v { + Adjudication::Valid => self.valid = self.valid.saturating_add(1), + Adjudication::AlreadyFixedNotProd => { + self.already_fixed = self.already_fixed.saturating_add(1); + } + Adjudication::InvalidMalicious => self.malicious = self.malicious.saturating_add(1), + Adjudication::Duplicate => self.duplicate = self.duplicate.saturating_add(1), + } + } + + /// `valid + malicious` — the precision denominator. + #[must_use] + pub fn decided(&self) -> u64 { + self.valid.saturating_add(self.malicious) + } + + /// Precision in bps: `valid / (valid + malicious)`. `None` if no decided items. + /// + /// Duplicates and already-fixed-not-prod do not inflate precision. + #[must_use] + pub fn precision_bps(&self) -> Option { + let d = self.decided(); + if d == 0 { + return None; + } + Some((self.valid.saturating_mul(10_000)) / d) + } + + /// Net credit across all verdicts (malicious is negative). + #[must_use] + pub fn net_credit(&self) -> i64 { + let v = i64::try_from(self.valid).unwrap_or(i64::MAX); + let m = i64::try_from(self.malicious).unwrap_or(i64::MAX); + let a = i64::try_from(self.already_fixed).unwrap_or(i64::MAX); + let d = i64::try_from(self.duplicate).unwrap_or(i64::MAX); + v.saturating_mul(VALID_CREDIT) + .saturating_add(m.saturating_mul(MALICIOUS_CREDIT)) + .saturating_add(a.saturating_mul(ALREADY_FIXED_CREDIT)) + .saturating_add(d.saturating_mul(DUPLICATE_CREDIT)) + } +} + +/// Gate that blocked champion displacement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GateFail { + /// Not enough decided holdout items. + ThinHoldout, + /// Challenger precision is not strictly better. + NoPrecisionWin, + /// Challenger net credit is negative (penalty / burn). + Penalty, + /// Champion hold — challenger did not displace. + Regression, +} + +/// Champion / challenger verdict. Consensus-critical once leaves are signed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChampionVerdict { + /// Challenger may become the live champion. + pub eligible: bool, + /// Challenger precision (bps) when decided. + pub challenger_precision_bps: Option, + /// Champion precision (bps) when decided. + pub champion_precision_bps: Option, + /// Gates that failed (empty ⇒ displace). + pub failed: Vec, + /// Lattice to emit if this hotkey is the live champion. + pub lattice: u64, +} + +/// Map precision bps onto the score lattice. +#[must_use] +pub fn lattice_from_precision(precision_bps: u64) -> u64 { + let clamped = precision_bps.min(10_000); + let num = u128::from(SCORE_MAX).saturating_mul(u128::from(clamped)); + u64::try_from(num / 10_000).unwrap_or(0) +} + +/// Incumbent keeps a positive lattice so a rejected challenger does not burn +/// the whole bounty share. +#[must_use] +pub fn champion_hold_lattice() -> u64 { + SCORE_MAX / 2 +} + +/// Judge challenger vs champion on holdout precision. Never crowns a regression. +#[must_use] +pub fn judge_challenger(champion: &MinerHoldout, challenger: &MinerHoldout) -> ChampionVerdict { + let mut failed = Vec::new(); + let chall_p = challenger.precision_bps(); + let champ_p = champion.precision_bps(); + + if challenger.decided() < MIN_HOLDOUT_DECIDED { + failed.push(GateFail::ThinHoldout); + } + if challenger.net_credit() < 0 { + failed.push(GateFail::Penalty); + } + + match (chall_p, champ_p) { + (Some(c), Some(h)) if c > h => {} + (Some(_), None) => {} + (Some(_), Some(_)) => { + failed.push(GateFail::NoPrecisionWin); + failed.push(GateFail::Regression); + } + (None, _) => { + if !failed.contains(&GateFail::ThinHoldout) { + failed.push(GateFail::ThinHoldout); + } + failed.push(GateFail::NoPrecisionWin); + } + } + + failed.sort_by(|a, b| format!("{a:?}").cmp(&format!("{b:?}"))); + failed.dedup(); + + let eligible = failed.is_empty(); + let lattice = if eligible { + chall_p.map_or(0, lattice_from_precision) + } else { + 0 + }; + + ChampionVerdict { + eligible, + challenger_precision_bps: chall_p, + champion_precision_bps: champ_p, + failed, + lattice, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hold(valid: u64, already: u64, malicious: u64, duplicate: u64) -> MinerHoldout { + MinerHoldout { + valid, + already_fixed: already, + malicious, + duplicate, + } + } + + #[test] + fn credits_match_spec() { + assert_eq!(Adjudication::Valid.credit(), VALID_CREDIT); + assert!(Adjudication::Valid.credit() > 0); + assert_eq!(Adjudication::AlreadyFixedNotProd.credit(), 0); + assert!(Adjudication::InvalidMalicious.credit() < 0); + assert_eq!(Adjudication::Duplicate.credit(), 0); + } + + #[test] + fn valid_unique_bug_rewards() { + let mut h = MinerHoldout::default(); + h.record(Adjudication::Valid); + h.record(Adjudication::Valid); + h.record(Adjudication::Valid); + assert_eq!(h.precision_bps(), Some(10_000)); + assert_eq!(h.net_credit(), 3 * VALID_CREDIT); + } + + #[test] + fn already_fixed_not_prod_is_ack_only() { + let mut h = MinerHoldout::default(); + h.record(Adjudication::AlreadyFixedNotProd); + h.record(Adjudication::AlreadyFixedNotProd); + assert_eq!(h.precision_bps(), None); + assert_eq!(h.net_credit(), 0); + assert_eq!(h.decided(), 0); + } + + #[test] + fn malicious_is_penalty() { + let mut h = MinerHoldout::default(); + h.record(Adjudication::InvalidMalicious); + h.record(Adjudication::InvalidMalicious); + h.record(Adjudication::InvalidMalicious); + assert_eq!(h.precision_bps(), Some(0)); + assert!(h.net_credit() < 0); + let champ = hold(3, 0, 0, 0); + let v = judge_challenger(&champ, &h); + assert!(!v.eligible); + assert!(v.failed.contains(&GateFail::Penalty)); + assert_eq!(v.lattice, 0); + } + + #[test] + fn duplicate_no_reward_no_penalty() { + let mut h = MinerHoldout::default(); + h.record(Adjudication::Duplicate); + h.record(Adjudication::Duplicate); + assert_eq!(h.net_credit(), 0); + assert_eq!(h.precision_bps(), None); + } + + #[test] + fn spam_volume_does_not_displace() { + let champ = hold(6, 0, 1, 0); + // Lots of junk + one valid: precision collapses. + let spam = hold(1, 0, 20, 40); + let v = judge_challenger(&champ, &spam); + assert!(!v.eligible); + assert!( + v.failed.contains(&GateFail::NoPrecisionWin) + || v.failed.contains(&GateFail::Penalty) + || v.failed.contains(&GateFail::Regression) + ); + } + + #[test] + fn higher_precision_displaces() { + let champ = hold(4, 2, 2, 1); + let better = hold(8, 0, 1, 0); + let v = judge_challenger(&champ, &better); + assert!(v.eligible, "failed={:?}", v.failed); + assert!(v.lattice > 0); + assert!(v.challenger_precision_bps.unwrap_or(0) > v.champion_precision_bps.unwrap_or(0)); + } + + #[test] + fn already_fixed_does_not_inflate_precision() { + let mut padded = hold(3, 50, 0, 0); + padded.record(Adjudication::AlreadyFixedNotProd); + assert_eq!(padded.precision_bps(), Some(10_000)); + assert_eq!(padded.decided(), 3); + } + + #[test] + fn lattice_maps_precision() { + assert_eq!(lattice_from_precision(0), 0); + assert_eq!(lattice_from_precision(10_000), SCORE_MAX); + assert_eq!(lattice_from_precision(5_000), SCORE_MAX / 2); + } + + #[test] + fn thin_holdout_cannot_crown() { + let champ = hold(5, 0, 0, 0); + let thin = hold(2, 0, 0, 0); + let v = judge_challenger(&champ, &thin); + assert!(v.failed.contains(&GateFail::ThinHoldout)); + assert!(!v.eligible); + } +} diff --git a/crates/bounty-score/src/public.rs b/crates/bounty-score/src/public.rs new file mode 100644 index 000000000..c49858fae --- /dev/null +++ b/crates/bounty-score/src/public.rs @@ -0,0 +1,356 @@ +//! Consumer types for the CortexLM/backend public Bounty feed. +//! +//! Cortex **reads** these payloads. It does not serve a public API. +//! Only `hotkey` + `problem_found` + `justification` + status counts +//! enter scoring. Account ids, sessions, and Chat logs are not in this DTO. + +use std::collections::BTreeMap; + +use bounty_challenge_task::{hotkey_hex, parse_hotkey}; +use serde::{Deserialize, Serialize}; + +use crate::{ + judge_challenger, lattice_from_precision, Adjudication, ChampionVerdict, MinerHoldout, +}; + +/// Published report as served by the backend public API. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicReport { + /// Backend report id. + pub id: String, + /// Miner hotkey (SS58). On-chain public. + pub hotkey: String, + /// Adjudication status. Pending items are ignored for scoring. + pub status: PublicStatus, + /// Short public statement of the bug (`Problème trouvé`). + pub problem_found: String, + /// Agent/service id that decided (not a human email). + pub adjudicator: String, + /// Why the agent accepted or rejected. Required for scoring. + pub justification: String, + /// RFC3339 adjudicate time. + pub adjudicated_at: String, + /// RFC3339 create time. + pub created_at: String, + /// Original report when `status` is duplicate. + #[serde(default)] + pub related_report_id: Option, +} + +/// Backend leaderboard row (valid-count ranking). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LeaderboardRow { + /// Miner hotkey (SS58). + pub hotkey: String, + /// Count of valid published reports. + pub valid_count: u64, + /// Optional lattice / weight hint from backend. + #[serde(default)] + pub weight: Option, +} + +/// Snapshot fetched from backend (`/v1/bounty/public/*`). +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicSnapshot { + /// Leaderboard rows. + #[serde(default)] + pub leaderboard: Vec, + /// Published reports. + #[serde(default)] + pub reports: Vec, +} + +/// Wire status on the backend public feed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PublicStatus { + /// Unique reproducing bug. + Valid, + /// Duplicate of an open report. + Duplicate, + /// Already fixed, not in prod. + AlreadyFixedNotProd, + /// Fabricated / malicious. + InvalidMalicious, + /// Not yet published — ignored for scoring. + Pending, +} + +impl PublicStatus { + fn adjudication(self) -> Option { + match self { + Self::Valid => Some(Adjudication::Valid), + Self::Duplicate => Some(Adjudication::Duplicate), + Self::AlreadyFixedNotProd => Some(Adjudication::AlreadyFixedNotProd), + Self::InvalidMalicious => Some(Adjudication::InvalidMalicious), + Self::Pending => None, + } + } +} + +/// True when the report may enter scoring (published + justified). +#[must_use] +pub fn scorable(report: &PublicReport) -> bool { + report.status.adjudication().is_some() + && !report.problem_found.trim().is_empty() + && !report.justification.trim().is_empty() +} + +/// Parse a leaderboard list (`{ "items": [...] }` or a bare array). +pub fn parse_leaderboard(raw: &str) -> Result, String> { + parse_items(raw) +} + +/// Parse a reports list (`{ "items": [...] }` or a bare array). +pub fn parse_reports(raw: &str) -> Result, String> { + parse_items(raw) +} + +#[derive(Deserialize)] +struct ItemsWrap { + items: Vec, +} + +fn parse_items Deserialize<'de>>(raw: &str) -> Result, String> { + if let Ok(v) = serde_json::from_str::>(raw) { + return Ok(v); + } + serde_json::from_str::>(raw) + .map(|w| w.items) + .map_err(|e| format!("backend public json: {e}")) +} + +/// Per-hotkey holdout from published, justified reports only. +#[must_use] +pub fn holdouts_from_reports(reports: &[PublicReport]) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + for r in reports { + if !scorable(r) { + continue; + } + let Some(v) = r.status.adjudication() else { + continue; + }; + let Ok(bytes) = parse_hotkey(&r.hotkey) else { + continue; + }; + let key = hotkey_hex(&bytes); + out.entry(key).or_default().record(v); + } + out +} + +/// Rank leaderboard by `valid_count` descending (stable on hotkey). +#[must_use] +pub fn rank_leaderboard(rows: &[LeaderboardRow]) -> Vec { + let mut v = rows.to_vec(); + v.sort_by(|a, b| { + b.valid_count + .cmp(&a.valid_count) + .then_with(|| a.hotkey.cmp(&b.hotkey)) + }); + v +} + +/// Score plan from a backend snapshot: precision holdouts + champion lattice. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicScorePlan { + /// Hex hotkey → holdout tallies. + pub holdouts: BTreeMap, + /// Champion hex hotkey, if any. + pub champion_hex: Option, + /// Lattice for the champion. + pub champion_lattice: u64, + /// Displacement verdict that crowned the champion. + pub verdict: Option, +} + +/// Build a score plan from backend public JSON objects. +/// +/// Reports missing `problem_found` or `justification` are dropped. +/// Champion is the highest-precision miner that displaces the previous +/// (or the first eligible). Leaderboard `valid_count` breaks ties. +#[must_use] +pub fn score_plan_from_snapshot(snap: &PublicSnapshot) -> PublicScorePlan { + let holdouts = holdouts_from_reports(&snap.reports); + let ranked = rank_leaderboard(&snap.leaderboard); + let mut order: Vec = Vec::new(); + for row in &ranked { + if let Ok(b) = parse_hotkey(&row.hotkey) { + let h = hotkey_hex(&b); + if holdouts.contains_key(&h) && !order.contains(&h) { + order.push(h); + } + } + } + for k in holdouts.keys() { + if !order.contains(k) { + order.push(k.clone()); + } + } + + let mut champion_hex = None; + let mut champion_lattice = 0; + let mut verdict = None; + let mut champ_hold = MinerHoldout::default(); + for h in &order { + let Some(chall) = holdouts.get(h) else { + continue; + }; + let v = judge_challenger(&champ_hold, chall); + if v.eligible { + champion_hex = Some(h.clone()); + champion_lattice = v.lattice; + if champion_lattice == 0 { + if let Some(p) = chall.precision_bps() { + champion_lattice = lattice_from_precision(p); + } + } + champ_hold = chall.clone(); + verdict = Some(v); + } + } + + PublicScorePlan { + holdouts, + champion_hex, + champion_lattice, + verdict, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const HK_A: &str = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"; + const HK_B: &str = "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty"; + + fn report(id: &str, hk: &str, status: PublicStatus, problem: &str, why: &str) -> PublicReport { + PublicReport { + id: id.into(), + hotkey: hk.into(), + status, + problem_found: problem.into(), + adjudicator: "bounty-adjudicator@cortex".into(), + justification: why.into(), + adjudicated_at: "2026-08-30T00:00:00Z".into(), + created_at: "2026-08-29T00:00:00Z".into(), + related_report_id: None, + } + } + + #[test] + fn missing_justification_or_problem_is_not_scored() { + let reports = vec![ + report("1", HK_A, PublicStatus::Valid, "seal 500", "reproduced"), + report("2", HK_A, PublicStatus::Valid, "", "reproduced"), + report("3", HK_A, PublicStatus::Valid, "other", ""), + report("4", HK_A, PublicStatus::Pending, "wip", "n/a"), + ]; + let h = holdouts_from_reports(&reports); + assert_eq!(h.len(), 1); + let only = h.values().next().expect("one"); + assert_eq!(only.valid, 1); + } + + #[test] + fn scoring_uses_hotkey_counts() { + let reports = vec![ + report( + "1", + HK_A, + PublicStatus::Valid, + "bug a", + "reproduced on master", + ), + report( + "2", + HK_A, + PublicStatus::Valid, + "bug b", + "reproduced on master", + ), + report( + "3", + HK_A, + PublicStatus::Valid, + "bug c", + "reproduced on master", + ), + report( + "4", + HK_B, + PublicStatus::InvalidMalicious, + "invented", + "does not exist", + ), + report( + "5", + HK_B, + PublicStatus::InvalidMalicious, + "invented 2", + "does not exist", + ), + report( + "6", + HK_B, + PublicStatus::InvalidMalicious, + "invented 3", + "does not exist", + ), + ]; + let plan = score_plan_from_snapshot(&PublicSnapshot { + leaderboard: vec![ + LeaderboardRow { + hotkey: HK_A.into(), + valid_count: 3, + weight: None, + }, + LeaderboardRow { + hotkey: HK_B.into(), + valid_count: 0, + weight: None, + }, + ], + reports, + }); + assert!(plan.champion_hex.is_some()); + let champ = plan.champion_hex.as_ref().expect("champ"); + let a_hex = hotkey_hex(&parse_hotkey(HK_A).expect("a")); + assert_eq!(champ, &a_hex); + assert!(plan.champion_lattice > 0); + assert_eq!(plan.holdouts.get(&a_hex).map(|h| h.valid), Some(3)); + } + + #[test] + fn leaderboard_sorts_by_valid_count() { + let ranked = rank_leaderboard(&[ + LeaderboardRow { + hotkey: "b".into(), + valid_count: 1, + weight: None, + }, + LeaderboardRow { + hotkey: "a".into(), + valid_count: 9, + weight: Some(1), + }, + LeaderboardRow { + hotkey: "c".into(), + valid_count: 9, + weight: None, + }, + ]); + assert_eq!(ranked[0].hotkey, "a"); + assert_eq!(ranked[1].hotkey, "c"); + assert_eq!(ranked[2].valid_count, 1); + } + + #[test] + fn parse_wrapped_items() { + let raw = r#"{"items":[{"hotkey":"a","valid_count":2}]}"#; + let rows = parse_leaderboard(raw).expect("parse"); + assert_eq!(rows[0].valid_count, 2); + } +} diff --git a/crates/bounty-store/Cargo.toml b/crates/bounty-store/Cargo.toml new file mode 100644 index 000000000..13d9695c4 --- /dev/null +++ b/crates/bounty-store/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "bounty-store" +description = "In-memory Bounty pairings, reports, and champion holdout" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +bounty-challenge-task = { path = "../bounty-challenge-task" } +bounty-score = { path = "../bounty-score" } +hex = "0.4" +serde = { version = "1", features = ["derive"] } +sha2 = "0.10" +thiserror = "2" + +[lints] +workspace = true diff --git a/crates/bounty-store/src/lib.rs b/crates/bounty-store/src/lib.rs new file mode 100644 index 000000000..3932ac566 --- /dev/null +++ b/crates/bounty-store/src/lib.rs @@ -0,0 +1,457 @@ +//! In-memory Bounty store: pairings, reports, champion. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::must_use_candidate +)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; + +use bounty_challenge_task::{hotkey_hex, SESSION_DOMAIN}; +use bounty_score::{judge_challenger, Adjudication, ChampionVerdict, MinerHoldout}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +/// Report lifecycle after submit / adjudicate. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReportState { + /// Waiting operator adjudication. + Pending, + /// Unique reproducing bug. + Valid, + /// Already fixed, not in prod. Ack only. + AlreadyFixedNotProd, + /// Malicious / fabricated. + InvalidMalicious, + /// Duplicate of an open report. + Duplicate, +} + +impl From for ReportState { + fn from(v: Adjudication) -> Self { + match v { + Adjudication::Valid => Self::Valid, + Adjudication::AlreadyFixedNotProd => Self::AlreadyFixedNotProd, + Adjudication::InvalidMalicious => Self::InvalidMalicious, + Adjudication::Duplicate => Self::Duplicate, + } + } +} + +/// Bound Cortex account ↔ miner hotkey. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Pairing { + /// Cortex Chat account id. + pub account_id: String, + /// 64-hex miner hotkey. + pub miner_hotkey: String, + /// Session claim id (hex). + pub session_id: String, + /// Unix-seconds bind time. + pub bound_at: u64, +} + +/// One miner bug report. Always tagged with the bound hotkey. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Report { + /// Stable id (`by_` + 16 hex). + pub id: String, + /// 64-hex miner hotkey (attribution). + pub miner_hotkey: String, + /// Cortex Chat account id. + pub account_id: String, + /// Short title. + pub title: String, + /// Report body. + pub body: String, + /// Reproduction steps. + pub repro_steps: String, + /// Dedup fingerprint (hex). + pub fingerprint: String, + /// Lifecycle. + pub state: ReportState, + /// Operator verdict (if any). + pub adjudication: Option, + /// If duplicate, the original report id. + pub duplicate_of: Option, + /// Displacement verdict after this adjudication (if any). + pub champion_verdict: Option, +} + +/// Session claim returned by `POST /v1/pair`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SessionClaim { + /// Opaque hex token. + pub token: String, + /// Bound account. + pub account_id: String, + /// Bound hotkey (64-hex). + pub miner_hotkey: String, + /// Session id. + pub session_id: String, +} + +/// Store errors. +#[derive(Debug, Error)] +pub enum StoreError { + /// Lock poisoned. + #[error("store lock poisoned")] + Poison, + /// Unknown row. + #[error("unknown {0}")] + NotFound(String), + /// Illegal transition or reuse. + #[error("{0}")] + Illegal(String), +} + +/// In-memory store (v0). +#[derive(Clone, Default)] +pub struct MemoryStore { + inner: Arc>, +} + +#[derive(Default)] +struct Inner { + next: u64, + pairings: BTreeMap, + sessions: BTreeMap, + used_nonces: BTreeSet, + reports: BTreeMap, + fingerprints: BTreeMap, + champion_hotkey: Option, +} + +impl MemoryStore { + /// Empty store. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + fn lock(&self) -> Result, StoreError> { + self.inner.lock().map_err(|_| StoreError::Poison) + } + + /// Bind account ↔ hotkey after a verified pairing signature. + pub fn bind_pair( + &self, + account_id: &str, + miner_hotkey: &str, + nonce: &str, + now_unix: u64, + session_secret: &[u8], + ) -> Result { + let mut g = self.lock()?; + if !g.used_nonces.insert(nonce.to_owned()) { + return Err(StoreError::Illegal("nonce reused".into())); + } + let session_id = next_id(&mut g.next, "bs"); + let token = mint_session_token(session_secret, &session_id, account_id, miner_hotkey); + let row = Pairing { + account_id: account_id.to_owned(), + miner_hotkey: miner_hotkey.to_owned(), + session_id: session_id.clone(), + bound_at: now_unix, + }; + g.pairings.insert(account_id.to_owned(), row.clone()); + g.sessions.insert(session_id.clone(), row); + Ok(SessionClaim { + token, + account_id: account_id.to_owned(), + miner_hotkey: miner_hotkey.to_owned(), + session_id, + }) + } + + /// Resolve a session token to the bound pairing. + pub fn lookup_session( + &self, + token: &str, + session_secret: &[u8], + ) -> Result { + let g = self.lock()?; + for row in g.sessions.values() { + let expect = mint_session_token( + session_secret, + &row.session_id, + &row.account_id, + &row.miner_hotkey, + ); + if crypto_eq(&expect, token) { + return Ok(row.clone()); + } + } + Err(StoreError::NotFound("session".into())) + } + + /// Insert a pending report. Same fingerprint as an open/valid report → duplicate. + pub fn insert_report(&self, mut row: Report) -> Result { + let mut g = self.lock()?; + if row.id.is_empty() { + row.id = next_id(&mut g.next, "by"); + } + if let Some(orig) = g.fingerprints.get(&row.fingerprint) { + if let Some(existing) = g.reports.get(orig) { + if matches!(existing.state, ReportState::Pending | ReportState::Valid) { + row.state = ReportState::Duplicate; + row.adjudication = Some(Adjudication::Duplicate); + row.duplicate_of = Some(orig.clone()); + } + } + } else { + g.fingerprints + .insert(row.fingerprint.clone(), row.id.clone()); + } + g.reports.insert(row.id.clone(), row.clone()); + Ok(row) + } + + /// Fetch one report. + pub fn get_report(&self, id: &str) -> Result { + let g = self.lock()?; + g.reports + .get(id) + .cloned() + .ok_or_else(|| StoreError::NotFound(id.to_owned())) + } + + /// List reports newest-first. + pub fn list_reports(&self) -> Result, StoreError> { + let g = self.lock()?; + let mut rows: Vec<_> = g.reports.values().cloned().collect(); + rows.sort_by(|a, b| b.id.cmp(&a.id)); + Ok(rows) + } + + /// Operator adjudicate. Updates champion when the reporter displaces. + pub fn adjudicate( + &self, + id: &str, + verdict: Adjudication, + duplicate_of: Option, + ) -> Result { + let mut g = self.lock()?; + { + let row = g + .reports + .get(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + if row.state != ReportState::Pending + && row.adjudication != Some(Adjudication::Duplicate) + { + return Err(StoreError::Illegal("already adjudicated".into())); + } + } + if verdict == Adjudication::Duplicate { + let orig = duplicate_of + .as_ref() + .ok_or_else(|| StoreError::Illegal("duplicate_of required".into()))?; + if !g.reports.contains_key(orig) { + return Err(StoreError::NotFound(orig.clone())); + } + } + let hotkey = { + let row = g + .reports + .get_mut(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + row.state = ReportState::from(verdict); + row.adjudication = Some(verdict); + row.duplicate_of = duplicate_of; + row.miner_hotkey.clone() + }; + + let chall = holdout_for(&g.reports, &hotkey); + let champ = g + .champion_hotkey + .as_ref() + .map(|h| holdout_for(&g.reports, h)) + .unwrap_or_default(); + let cv = judge_challenger(&champ, &chall); + if cv.eligible { + g.champion_hotkey = Some(hotkey); + } + let row = g + .reports + .get_mut(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + row.champion_verdict = Some(cv); + Ok(row.clone()) + } + + /// Live champion hotkey (64-hex), if any. + pub fn champion_hotkey(&self) -> Result, StoreError> { + Ok(self.lock()?.champion_hotkey.clone()) + } + + /// Holdout tallies for a hotkey. + pub fn holdout(&self, hotkey: &str) -> Result { + let g = self.lock()?; + Ok(holdout_for(&g.reports, hotkey)) + } + + /// Pairing for an account, if bound. + pub fn pairing_for_account(&self, account_id: &str) -> Result, StoreError> { + Ok(self.lock()?.pairings.get(account_id).cloned()) + } +} + +fn holdout_for(reports: &BTreeMap, hotkey: &str) -> MinerHoldout { + let mut h = MinerHoldout::default(); + for r in reports.values() { + if r.miner_hotkey != hotkey { + continue; + } + if let Some(v) = r.adjudication { + h.record(v); + } + } + h +} + +fn next_id(next: &mut u64, prefix: &str) -> String { + let n = *next; + *next = next.saturating_add(1); + format!("{prefix}_{n:016x}") +} + +fn mint_session_token(secret: &[u8], session_id: &str, account_id: &str, hotkey: &str) -> String { + let mut h = Sha256::new(); + h.update(SESSION_DOMAIN); + h.update(secret); + h.update(session_id.as_bytes()); + h.update(account_id.as_bytes()); + h.update(hotkey.as_bytes()); + hex::encode(h.finalize()) +} + +fn crypto_eq(a: &str, b: &str) -> bool { + if a.len() != b.len() { + return false; + } + a.as_bytes() + .iter() + .zip(b.as_bytes()) + .fold(0u8, |acc, (x, y)| acc | (x ^ y)) + == 0 +} + +/// SHA-256 fingerprint of normalized title+body. +#[must_use] +pub fn report_fingerprint(title: &str, body: &str) -> String { + let mut h = Sha256::new(); + h.update(REPORT_TAG); + h.update(title.trim().to_ascii_lowercase().as_bytes()); + h.update([0xff]); + h.update(body.trim().to_ascii_lowercase().as_bytes()); + hex::encode(h.finalize()) +} + +const REPORT_TAG: &[u8] = bounty_challenge_task::REPORT_DOMAIN; + +/// Helper: 64-hex from a raw hotkey. +#[must_use] +pub fn hex_hotkey(bytes: &[u8; 32]) -> String { + hotkey_hex(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn secret() -> [u8; 16] { + [7u8; 16] + } + + #[test] + fn pair_and_session_round_trip() { + let s = MemoryStore::new(); + let claim = s + .bind_pair("acct", &"aa".repeat(32), "0123456789abcdef", 10, &secret()) + .expect("bind"); + let row = s.lookup_session(&claim.token, &secret()).expect("lookup"); + assert_eq!(row.account_id, "acct"); + assert!(s + .bind_pair("acct", &"aa".repeat(32), "0123456789abcdef", 11, &secret()) + .is_err()); + } + + #[test] + fn duplicate_fingerprint_auto_marks() { + let s = MemoryStore::new(); + let fp = report_fingerprint("same bug", "steps"); + let a = s + .insert_report(Report { + id: String::new(), + miner_hotkey: "aa".repeat(32), + account_id: "a".into(), + title: "same bug".into(), + body: "steps".into(), + repro_steps: "1".into(), + fingerprint: fp.clone(), + state: ReportState::Pending, + adjudication: None, + duplicate_of: None, + champion_verdict: None, + }) + .expect("a"); + assert_eq!(a.state, ReportState::Pending); + let b = s + .insert_report(Report { + id: String::new(), + miner_hotkey: "bb".repeat(32), + account_id: "b".into(), + title: "same bug".into(), + body: "steps".into(), + repro_steps: "1".into(), + fingerprint: fp, + state: ReportState::Pending, + adjudication: None, + duplicate_of: None, + champion_verdict: None, + }) + .expect("b"); + assert_eq!(b.state, ReportState::Duplicate); + assert_eq!(b.duplicate_of.as_deref(), Some(a.id.as_str())); + } + + #[test] + fn adjudicate_valid_can_crown() { + let s = MemoryStore::new(); + let hk = "cc".repeat(32); + let mut last = String::new(); + for i in 0..3 { + let row = s + .insert_report(Report { + id: String::new(), + miner_hotkey: hk.clone(), + account_id: "acct".into(), + title: format!("bug {i}"), + body: format!("body {i}"), + repro_steps: "repro".into(), + fingerprint: report_fingerprint(&format!("bug {i}"), &format!("body {i}")), + state: ReportState::Pending, + adjudication: None, + duplicate_of: None, + champion_verdict: None, + }) + .expect("ins"); + last = row.id; + s.adjudicate(&last, Adjudication::Valid, None).expect("adj"); + } + let row = s.get_report(&last).expect("get"); + assert_eq!(row.adjudication, Some(Adjudication::Valid)); + assert_eq!( + s.champion_hotkey().expect("champ").as_deref(), + Some(hk.as_str()) + ); + let v = row.champion_verdict.expect("verdict"); + assert!(v.eligible); + } +} diff --git a/crates/relearn-challenge-task/Cargo.toml b/crates/relearn-challenge-task/Cargo.toml new file mode 100644 index 000000000..67d4792fb --- /dev/null +++ b/crates/relearn-challenge-task/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "relearn-challenge-task" +description = "Relearn challenge identity, model pins, and domain tags" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +serde = { version = "1", features = ["derive"] } + +[lints] +workspace = true diff --git a/crates/relearn-challenge-task/src/lib.rs b/crates/relearn-challenge-task/src/lib.rs new file mode 100644 index 000000000..534e2b0f6 --- /dev/null +++ b/crates/relearn-challenge-task/src/lib.rs @@ -0,0 +1,185 @@ +//! Relearn challenge identity and verified model pins. +//! +//! ```text +//! challenge_id = "relearn" +//! scoring_version = 1 +//! task_id domain = b"base-relearn-task-id-v1" +//! receipt domain = b"base-relearn-receipt-v1" +//! ``` +//! +//! Distinct from `design` / `prism` so leaf digests never collide. +//! Master-centralized Lium eval; miners pay Lium. + +#![forbid(unsafe_code)] +#![allow(clippy::doc_markdown)] + +/// Normative challenge id (trust-root / leaf `challenge_id` string). +pub const CHALLENGE_ID: &str = "relearn"; + +/// UTF-8 bytes of [`CHALLENGE_ID`]. +pub const CHALLENGE_ID_BYTES: &[u8] = b"relearn"; + +/// Live `challenge_scoring_version` (displacement vs champion + gates). +pub const SCORING_VERSION: u16 = 1; + +/// Domain tag for task id digests. +pub const TASK_ID_DOMAIN: &[u8] = b"base-relearn-task-id-v1"; + +/// Domain tag for holdout slice ids. +pub const HOLDOUT_DOMAIN: &[u8] = b"base-relearn-holdout-v1"; + +/// Domain tag for eval-receipt digests. +pub const RECEIPT_DOMAIN: &[u8] = b"base-relearn-receipt-v1"; + +/// Domain tag for promotion attestations. +pub const PROMOTE_DOMAIN: &[u8] = b"base-relearn-promote-v1"; + +/// Integer score lattice max (same scale as other challenges). +pub const SCORE_MAX: u64 = 1_000_000; + +/// Verified Hugging Face id for the base model miners improve. +/// +/// Confirmed 2026-08-29: +pub const BASE_MODEL_ID: &str = "Qwen/Qwen3.8-Flash-Next"; + +/// Wire id for the v0 HTTP teacher. Override with `RELEARN_TEACHER_MODEL`. +pub const TEACHER_MODEL_ID: &str = "kimi-k3"; + +/// Hugging Face-style alias some OpenAI-compatible hosts use. +pub const TEACHER_MODEL_HF_ALIAS: &str = "moonshotai/Kimi-K3"; + +/// Frozen GLM teacher — optional override, not the v0 default. +pub const TEACHER_GLM_MODEL_ID: &str = "zai-org/GLM-5.3"; + +/// Community NVFP4 checkpoint (optional Lium serve; not the v0 default). +pub const TEACHER_NVFP4_ID: &str = "Inferact/GLM-5.3-NVFP4"; + +/// Public miner / eval-image repo. +pub const RELEARN_GIT_URL: &str = "https://github.com/CortexLM/relearn"; + +/// Teacher serving mode. v0 default is a teacher-only HTTP API. +/// Miner weights are never served through the teacher API. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TeacherBackend { + /// Optional NVFP4 on a digest-pinned Lium pod (`RELEARN_TEACHER_BACKEND=lium`). + LiumNvfp4, + /// Teacher-only OpenAI-compatible HTTP API (v0 default). + HttpApi, + /// Deterministic offline judge (CI / `RELEARN_FORCE_SIM`). + Sim, +} + +impl TeacherBackend { + /// Parse from env (`RELEARN_TEACHER_BACKEND`). Empty → HTTP API. + #[must_use] + pub fn from_env() -> Self { + match std::env::var("RELEARN_TEACHER_BACKEND") + .unwrap_or_default() + .to_ascii_lowercase() + .as_str() + { + "lium" | "lium_nvfp4" | "nvfp4" => Self::LiumNvfp4, + "sim" => Self::Sim, + _ => Self::HttpApi, + } + } +} + +/// Default teacher backend for v0: HTTP API unless sim is forced. +#[must_use] +pub fn default_teacher_backend(force_sim: bool) -> TeacherBackend { + if force_sim { + TeacherBackend::Sim + } else { + TeacherBackend::HttpApi + } +} + +/// `RELEARN_TEACHER_API_URL` when set. No baked host — missing means skip/sim. +#[must_use] +pub fn teacher_api_url() -> Option { + std::env::var("RELEARN_TEACHER_API_URL") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} + +/// `RELEARN_TEACHER_MODEL`, or [`TEACHER_MODEL_ID`]. +#[must_use] +pub fn teacher_model_from_env() -> String { + std::env::var("RELEARN_TEACHER_MODEL") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| TEACHER_MODEL_ID.to_owned()) +} + +/// Bearer for the teacher HTTP API. Never log the value. +/// +/// Reads `RELEARN_TEACHER_API_KEY` only. +#[must_use] +pub fn teacher_api_key() -> Option { + std::env::var("RELEARN_TEACHER_API_KEY") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} + +/// True when `model` is the configured HTTP teacher or the optional GLM pin. +#[must_use] +pub fn is_configured_teacher_model(model: &str) -> bool { + let m = model.trim(); + if m.is_empty() { + return false; + } + m == TEACHER_MODEL_ID + || m.eq_ignore_ascii_case(TEACHER_MODEL_HF_ALIAS) + || m == TEACHER_GLM_MODEL_ID + || m == teacher_model_from_env() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn challenge_id_is_relearn() { + assert_eq!(CHALLENGE_ID, "relearn"); + assert_eq!(CHALLENGE_ID_BYTES, b"relearn"); + assert_ne!(CHALLENGE_ID, "prism"); + assert_ne!(CHALLENGE_ID, "design"); + } + + #[test] + fn verified_model_ids() { + assert_eq!(BASE_MODEL_ID, "Qwen/Qwen3.8-Flash-Next"); + assert_eq!(TEACHER_MODEL_ID, "kimi-k3"); + assert_eq!(TEACHER_GLM_MODEL_ID, "zai-org/GLM-5.3"); + assert!(teacher_api_url().is_none()); + } + + #[test] + fn teacher_backend_defaults_to_http_api() { + assert_eq!(default_teacher_backend(false), TeacherBackend::HttpApi); + assert_eq!(default_teacher_backend(true), TeacherBackend::Sim); + } + + #[test] + fn kimi_and_glm_are_configured_teachers() { + assert!(is_configured_teacher_model("kimi-k3")); + assert!(is_configured_teacher_model("moonshotai/Kimi-K3")); + assert!(is_configured_teacher_model(TEACHER_GLM_MODEL_ID)); + assert!(!is_configured_teacher_model("")); + } + + #[test] + fn domain_tags_are_relearn_prefixed() { + assert!(std::str::from_utf8(TASK_ID_DOMAIN) + .unwrap_or("") + .contains("relearn")); + assert!(!std::str::from_utf8(TASK_ID_DOMAIN) + .unwrap_or("") + .contains("prism")); + } +} diff --git a/crates/relearn-challenge/Cargo.toml b/crates/relearn-challenge/Cargo.toml new file mode 100644 index 000000000..0cf908401 --- /dev/null +++ b/crates/relearn-challenge/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "relearn-challenge" +description = "Relearn orchestrator: submit, eval, operator promote, D24 emit" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +bundle = { path = "../bundle" } +challenge-common = { path = "../challenge-common" } +crypto = { path = "../crypto" } +hex = "0.4" +relearn-challenge-task = { path = "../relearn-challenge-task" } +relearn-eval = { path = "../relearn-eval" } +relearn-http = { path = "../relearn-http" } +relearn-score = { path = "../relearn-score" } +relearn-store = { path = "../relearn-store" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" + +[lints] +workspace = true diff --git a/crates/relearn-challenge/src/lib.rs b/crates/relearn-challenge/src/lib.rs new file mode 100644 index 000000000..42b629f98 --- /dev/null +++ b/crates/relearn-challenge/src/lib.rs @@ -0,0 +1,127 @@ +//! Relearn orchestrator helpers: D24 leaf plan + crate re-exports. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::must_use_candidate +)] + +use std::collections::{BTreeMap, BTreeSet}; + +use bundle::{NoScoreReasonCode, ScoreOrAbsence}; +use challenge_common::{emit_signed_leaf_set, Hotkey, LeafEmitError}; +use relearn_challenge_task::{CHALLENGE_ID_BYTES, SCORE_MAX}; +use relearn_score::champion_hold_lattice; +use relearn_store::SubmissionState; + +pub use relearn_challenge_task::{ + BASE_MODEL_ID, CHALLENGE_ID, CHALLENGE_ID_BYTES as RELEARN_ID_BYTES, + SCORE_MAX as RELEARN_SCORE_MAX, SCORING_VERSION, TEACHER_MODEL_ID, +}; +pub use relearn_eval::{resolve_teacher_backend, RelearnPin}; +pub use relearn_http::{hash_admin_token, relearn_router, AppState}; +pub use relearn_store::MemoryStore; + +/// Build a D24-complete score map: champion (if any) gets a positive lattice; +/// everyone else is explicit `NoScore` (never silent). +pub fn emission_scores( + expected: &BTreeSet, + champion_hotkey: Option, + champion_lattice: u64, +) -> BTreeMap { + expected + .iter() + .map(|h| { + let s = match champion_hotkey { + Some(c) if c == *h && champion_lattice > 0 => ScoreOrAbsence::Score { + value: champion_lattice.min(SCORE_MAX), + }, + _ => ScoreOrAbsence::NoScore { + reason: NoScoreReasonCode::NotAttempted, + }, + }; + (*h, s) + }) + .collect() +} + +/// Sign the exact-E leaf set for this epoch. +pub fn emit_epoch( + secret: &[u8; 32], + epoch: u64, + expected: &BTreeSet, + champion_hotkey: Option, + champion_lattice: u64, +) -> Result, LeafEmitError> { + let scores = emission_scores(expected, champion_hotkey, champion_lattice); + emit_signed_leaf_set(secret, CHALLENGE_ID_BYTES, epoch, expected, &scores) +} + +/// Lattice for the current store champion, or the hold value when only +/// the base model is live (burn is wrong: the factory still has a champ). +pub fn live_champion_lattice(store: &MemoryStore) -> u64 { + if let Ok(Some(id)) = store.champion_id() { + if let Ok(row) = store.get(&id) { + if row.state == SubmissionState::Champion { + if let Some(v) = row.verdict { + if v.eligible && v.lattice > 0 { + return v.lattice; + } + } + return champion_hold_lattice(); + } + } + } + 0 +} + +/// Parse a 64-hex hotkey. +pub fn parse_hotkey(hex_s: &str) -> Option { + let t = hex_s.trim().trim_start_matches("0x"); + let bytes = hex::decode(t).ok()?; + <[u8; 32]>::try_from(bytes).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use challenge_common::public_key_from_secret; + use crypto::KEY_LEN; + + fn sk() -> [u8; KEY_LEN] { + let mut s = [7u8; KEY_LEN]; + s[0] = 1; + s + } + + #[test] + fn d24_covers_every_hotkey() { + let a = [1u8; 32]; + let b = [2u8; 32]; + let mut e = BTreeSet::new(); + e.insert(a); + e.insert(b); + let leaves = emit_epoch(&sk(), 9, &e, Some(a), 12_000).expect("emit"); + assert_eq!(leaves.len(), 2); + assert!(matches!( + leaves[&a].score_or_absence, + ScoreOrAbsence::Score { value: 12_000 } + )); + assert!(matches!( + leaves[&b].score_or_absence, + ScoreOrAbsence::NoScore { .. } + )); + let pk = public_key_from_secret(&sk()).expect("pk"); + for leaf in leaves.values() { + challenge_common::verify_leaf_sig(leaf, &pk).expect("sig"); + } + } + + #[test] + fn never_emits_score_on_empty_expected() { + let e = BTreeSet::new(); + let leaves = emit_epoch(&sk(), 1, &e, None, 0).expect("empty E"); + assert!(leaves.is_empty()); + } +} diff --git a/crates/relearn-eval/Cargo.toml b/crates/relearn-eval/Cargo.toml new file mode 100644 index 000000000..a91a46c8d --- /dev/null +++ b/crates/relearn-eval/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "relearn-eval" +description = "Relearn eval loop: digest freeze, holdout unseal, Lium/sim, receipts" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +async-trait = "0.1" +hex = "0.4" +prism-competition = { path = "../prism-competition" } +prism-lium = { path = "../prism-lium" } +prism-lium-types = { path = "../prism-lium-types" } +relearn-challenge-task = { path = "../relearn-challenge-task" } +relearn-score = { path = "../relearn-score" } +relearn-store = { path = "../relearn-store" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" + +[dev-dependencies] +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/crates/relearn-eval/src/lib.rs b/crates/relearn-eval/src/lib.rs new file mode 100644 index 000000000..a89d37813 --- /dev/null +++ b/crates/relearn-eval/src/lib.rs @@ -0,0 +1,388 @@ +//! Relearn eval loop: freeze digest → unseal holdout → rent/sim → harvest. +//! +//! Miner pays Lium (`LIUM_API_KEY` / `X-Lium-Api-Key`). The control plane +//! only ever boots a digest-pinned eval image. Teacher HTTP is judge-only +//! and never serves miner weights as the scored artifact. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::must_use_candidate, + clippy::significant_drop_tightening +)] + +use prism_competition::ExampleSeries; +use prism_lium::{EvalJobBackend, SimLiumBackend}; +use prism_lium_types::{EvalReceipt, InstanceSpec, NoScoreGate, RemoteExecResult}; +use relearn_challenge_task::{ + default_teacher_backend, is_configured_teacher_model, TeacherBackend, BASE_MODEL_ID, + TEACHER_MODEL_ID, TEACHER_NVFP4_ID, +}; +use relearn_score::SliceScores; +use relearn_store::{unseal_holdout, Holdout}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +/// Pins Cortex stores for the split `CortexLM/relearn` repo + eval image. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RelearnPin { + /// `Qwen/Qwen3.8-Flash-Next`. + pub base_model: String, + /// HTTP teacher wire id (`kimi-k3` default; GLM optional override). + pub teacher_model: String, + /// Optional NVFP4 id for Lium serving. + pub teacher_nvfp4: String, + /// `lium_nvfp4` | `http_api` | `sim`. + pub teacher_backend: TeacherBackend, + /// Eval image reference (no floating tag in prod). + pub eval_image: String, + /// `sha256:…` digest. Empty until the first green relearn CI image. + pub eval_image_digest: String, + /// `https://github.com/CortexLM/relearn`. + pub relearn_git: String, + /// Pinned git SHA of CortexLM/relearn (empty until first push). + pub relearn_git_sha: String, +} + +impl Default for RelearnPin { + fn default() -> Self { + Self { + base_model: BASE_MODEL_ID.into(), + teacher_model: TEACHER_MODEL_ID.into(), + teacher_nvfp4: TEACHER_NVFP4_ID.into(), + teacher_backend: TeacherBackend::HttpApi, + eval_image: "ghcr.io/cortexlm/relearn-eval".into(), + eval_image_digest: String::new(), + relearn_git: relearn_challenge_task::RELEARN_GIT_URL.into(), + relearn_git_sha: String::new(), + } + } +} + +impl RelearnPin { + /// Load from `config/relearn-pin.toml` (best-effort key=value / toml-ish). + #[must_use] + pub fn from_toml(body: &str) -> Self { + let mut pin = Self::default(); + for raw in body.lines() { + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((k, v)) = line.split_once('=') else { + continue; + }; + let key = k.trim(); + let val = v.trim().trim_matches('"').to_owned(); + match key { + "base_model" => pin.base_model = val, + "teacher_model" => pin.teacher_model = val, + "teacher_nvfp4" => pin.teacher_nvfp4 = val, + "eval_image" => pin.eval_image = val, + "eval_image_digest" => pin.eval_image_digest = val, + "relearn_git" => pin.relearn_git = val, + "relearn_git_sha" => pin.relearn_git_sha = val, + "teacher_backend" => { + pin.teacher_backend = match val.as_str() { + "lium_nvfp4" => TeacherBackend::LiumNvfp4, + "http_api" => TeacherBackend::HttpApi, + _ => TeacherBackend::Sim, + }; + } + _ => {} + } + } + pin + } + + /// True when a live rent is allowed (real digest pin present). + #[must_use] + pub fn can_rent(&self) -> bool { + self.eval_image_digest.starts_with("sha256:") && self.eval_image_digest.len() >= 71 + } +} + +/// Eval errors. +#[derive(Debug, Error)] +pub enum EvalError { + /// Holdout was requested before the digest freeze. + #[error("holdout still sealed")] + HoldoutSealed, + /// Integrity gate failed. + #[error("integrity: {0}")] + Integrity(String), + /// Lium / backend failure. + #[error("backend: {0}")] + Backend(String), + /// Teacher API is not allowed to receive miner weights. + #[error("teacher API refused miner-weight payload")] + TeacherMinerWeights, +} + +/// One finished eval. +#[derive(Debug, Clone)] +pub struct EvalOutcome { + /// Challenger measurements. + pub scores: SliceScores, + /// Integrity receipt. + pub receipt: EvalReceipt, + /// Holdout after unseal (seed visible only here). + pub holdout: Holdout, +} + +/// Deterministic sim scores from a frozen digest + holdout seed. +#[must_use] +pub fn sim_slice_scores(artifact_digest: &str, holdout_seed: &str) -> SliceScores { + let holdout = series_from("h", artifact_digest, holdout_seed, 120, 0.15); + let public = series_from("p", artifact_digest, holdout_seed, 120, 0.0); + let perturbed = series_from( + "x", + artifact_digest, + &format!("{holdout_seed}-p"), + 120, + -0.02, + ); + let canaries = series_from("c", "canary", holdout_seed, 40, 0.45); + SliceScores { + holdout, + public, + perturbed, + canaries, + agent_trace: 0.85, + } +} + +/// Fixed base-model champion (Qwen3.8-Flash-Next, no miner adapter). +#[must_use] +pub fn base_champion_scores() -> SliceScores { + sim_slice_scores("base-qwen-3.8-flash-next", "base-seed") +} + +fn series_from(prefix: &str, digest: &str, seed: &str, n: usize, bias: f64) -> ExampleSeries { + let mut h = Sha256::new(); + h.update(digest.as_bytes()); + h.update([0xff]); + h.update(seed.as_bytes()); + let root = h.finalize(); + let pairs = (0..n).map(|i| { + let v = f64::from(root[i % 32]) / 255.0; + let score = (0.45 + 0.4 * v + bias).clamp(0.0, 1.0); + (format!("{prefix}{i}"), score) + }); + ExampleSeries::from_pairs(pairs) +} + +/// Unseal holdout only after `frozen_digest` is recorded, then score. +pub fn eval_after_freeze( + pending: &Holdout, + frozen_digest: &str, + artifact_digest: &str, +) -> Result { + let holdout = unseal_holdout(pending, frozen_digest).ok_or(EvalError::HoldoutSealed)?; + if !holdout.unsealed { + return Err(EvalError::HoldoutSealed); + } + let scores = sim_slice_scores(artifact_digest, &holdout.seed_hex); + let metrics = serde_json::to_vec(&serde_json::json!({ + "holdout_n": scores.holdout.len(), + "agent_trace": scores.agent_trace, + })) + .unwrap_or_default(); + let receipt = EvalReceipt { + provider: "sim".into(), + pod_id: format!("sim-{}", &frozen_digest[..8.min(frozen_digest.len())]), + image_digest: String::new(), + submission_hash: frozen_digest.to_owned(), + metrics_hash: EvalReceipt::hash_metrics_bytes(&metrics), + termination_verified: true, + }; + NoScoreGate::check(&receipt, false).map_err(|e| EvalError::Integrity(e.to_string()))?; + Ok(EvalOutcome { + scores, + receipt, + holdout, + }) +} + +/// Teacher request: prompts only. Rejects miner-weight bodies. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TeacherJudgeRequest { + /// Prompt / completion pair to judge. + pub prompt: String, + /// Candidate text. + pub candidate: String, + /// Must be the teacher model id, never a miner artifact. + pub model: String, +} + +/// Refuse any attempt to send miner weights through the teacher API. +pub fn teacher_judge_guard(req: &TeacherJudgeRequest, pin: &RelearnPin) -> Result<(), EvalError> { + let model = req.model.trim(); + let looks_like_digest = model.len() == 64 && model.chars().all(|c| c.is_ascii_hexdigit()); + if looks_like_digest { + return Err(EvalError::TeacherMinerWeights); + } + if model != pin.teacher_model && !is_configured_teacher_model(model) { + return Err(EvalError::TeacherMinerWeights); + } + let lower = req.candidate.to_ascii_lowercase(); + if lower.contains("safetensors") || lower.contains("gguf") || lower.contains("nvfp4") { + return Err(EvalError::TeacherMinerWeights); + } + Ok(()) +} + +/// Resolve the v0 teacher backend. HTTP API is the default. Sim when +/// `RELEARN_FORCE_SIM` is set or `RELEARN_TEACHER_BACKEND=sim`. Lium NVFP4 +/// only when the operator sets `RELEARN_TEACHER_BACKEND=lium`. +/// Miner weights are never the served model. +#[must_use] +pub fn resolve_teacher_backend() -> TeacherBackend { + let force_sim = matches!( + std::env::var("RELEARN_FORCE_SIM") + .unwrap_or_default() + .to_ascii_lowercase() + .as_str(), + "1" | "true" | "yes" + ); + if force_sim { + return default_teacher_backend(true); + } + TeacherBackend::from_env() +} + +/// Rent a digest-pinned eval pod, exec, harvest, terminate. +/// +/// `api_key` is used only to construct the backend the caller already built. +/// This function never logs it. Live rent is skipped when `pin.can_rent()` is +/// false (no published eval digest yet). +pub async fn rent_eval( + backend: &dyn EvalJobBackend, + pin: &RelearnPin, + frozen_digest: &str, + artifact_digest: &str, +) -> Result<(RemoteExecResult, String), EvalError> { + if !pin.can_rent() { + return Err(EvalError::Integrity( + "eval image digest not pinned; refuse live rent".into(), + )); + } + let spec = InstanceSpec { + name: format!("relearn-{}", &frozen_digest[..12.min(frozen_digest.len())]), + max_lifetime_hours: 1.0, + max_price_per_hour: 8.0, + gpu_count: 1, + image_digest: Some(pin.eval_image_digest.clone()), + ssh_public_keys: Vec::new(), + ssh_key_name: None, + preferred_offer_id: None, + template_id: None, + template_name: None, + }; + let inst = backend + .provision(&spec) + .await + .map_err(|e| EvalError::Backend(e.to_string()))?; + let exec = backend + .exec_eval(&inst.id, artifact_digest, frozen_digest, None) + .await + .map_err(|e| EvalError::Backend(e.to_string())); + let term = backend.terminate(&inst.id).await; + let verified = backend.verify_terminated(&inst.id).await.unwrap_or(false); + if let Err(e) = term { + return Err(EvalError::Backend(e.to_string())); + } + if !verified { + return Err(EvalError::Integrity("pod terminate not verified".into())); + } + exec.map(|r| (r, inst.id)) +} + +/// Convenience: sim backend rent that always tears down. +pub async fn sim_rent_roundtrip(digest: &str) -> Result { + let backend = SimLiumBackend::new(); + let pin = RelearnPin { + eval_image_digest: format!("sha256:{}", "ab".repeat(32)), + ..RelearnPin::default() + }; + let (_r, id) = rent_eval(&backend, &pin, digest, digest).await?; + Ok(id) +} + +#[cfg(test)] +mod tests { + use super::*; + use relearn_store::sealed_holdout; + + #[test] + fn unseal_happens_only_after_freeze() { + let pending = sealed_holdout(1, "digest-a"); + assert!(eval_after_freeze(&pending, "", "art").is_err()); + let out = eval_after_freeze(&pending, "digest-a", "art").expect("eval"); + assert!(out.holdout.unsealed); + assert_eq!(out.receipt.submission_hash, "digest-a"); + assert!(out.scores.holdout.len() >= 100); + } + + #[test] + fn teacher_guard_rejects_miner_weight_payload() { + let pin = RelearnPin::default(); + let bad = TeacherJudgeRequest { + prompt: "score".into(), + candidate: "here is a safetensors blob".into(), + model: TEACHER_MODEL_ID.into(), + }; + assert!(teacher_judge_guard(&bad, &pin).is_err()); + let good = TeacherJudgeRequest { + prompt: "score".into(), + candidate: "the capital is paris".into(), + model: TEACHER_MODEL_ID.into(), + }; + assert!(teacher_judge_guard(&good, &pin).is_ok()); + let glm = TeacherJudgeRequest { + prompt: "score".into(), + candidate: "ok".into(), + model: relearn_challenge_task::TEACHER_GLM_MODEL_ID.into(), + }; + assert!(teacher_judge_guard(&glm, &pin).is_ok()); + let digest = TeacherJudgeRequest { + prompt: "score".into(), + candidate: "ok".into(), + model: "ab".repeat(32), + }; + assert!(teacher_judge_guard(&digest, &pin).is_err()); + } + + #[test] + fn pin_refuses_rent_without_digest() { + assert!(!RelearnPin::default().can_rent()); + let p = RelearnPin { + eval_image_digest: format!("sha256:{}", "00".repeat(32)), + ..RelearnPin::default() + }; + assert!(p.can_rent()); + } + + #[tokio::test] + async fn sim_rent_tears_down() { + let id = sim_rent_roundtrip("abcdef0123456789").await.expect("rent"); + assert!(id.contains("sim-pod")); + } + + #[test] + fn toml_pin_roundtrip() { + let body = r#" +base_model = "Qwen/Qwen3.8-Flash-Next" +teacher_model = "kimi-k3" +teacher_backend = "http_api" +"#; + let p = RelearnPin::from_toml(body); + assert_eq!(p.base_model, BASE_MODEL_ID); + assert_eq!(p.teacher_model, TEACHER_MODEL_ID); + assert_eq!(p.teacher_backend, TeacherBackend::HttpApi); + } +} diff --git a/crates/relearn-eval/tests/teacher_http_smoke.rs b/crates/relearn-eval/tests/teacher_http_smoke.rs new file mode 100644 index 000000000..0c5519c74 --- /dev/null +++ b/crates/relearn-eval/tests/teacher_http_smoke.rs @@ -0,0 +1,53 @@ +//! Teacher HTTP smoke: `GET /v1/models` when URL + key are set. +//! +//! Skip when `RELEARN_TEACHER_API_URL` or `RELEARN_TEACHER_API_KEY` is unset +//! so CI stays green. Never print the key. Never bake a host. + +#![forbid(unsafe_code)] + +use prism_lium::EvalJobBackend; +use relearn_challenge_task::{teacher_api_key, teacher_api_url}; + +#[tokio::test] +async fn teacher_http_models_or_skip() { + let Some(base) = teacher_api_url() else { + eprintln!("skip teacher HTTP smoke: RELEARN_TEACHER_API_URL unset"); + return; + }; + let Some(key) = teacher_api_key() else { + eprintln!("skip teacher HTTP smoke: RELEARN_TEACHER_API_KEY unset"); + return; + }; + let url = format!("{}/models", base.trim_end_matches('/')); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(20)) + .build() + .unwrap_or_else(|e| panic!("teacher HTTP client: {e}")); + let resp = client + .get(&url) + .bearer_auth(&key) + .send() + .await + .unwrap_or_else(|e| panic!("teacher GET /v1/models transport: {e}")); + let status = resp.status(); + assert!( + status.is_success(), + "teacher GET /v1/models HTTP {status} (do not log the key)" + ); +} + +#[tokio::test] +async fn lium_list_offers_or_skip() { + let key = match std::env::var("LIUM_API_KEY") { + Ok(k) if !k.trim().is_empty() => k, + _ => { + eprintln!("skip Lium smoke: LIUM_API_KEY unset"); + return; + } + }; + let client = prism_lium::LiumClient::new(key).unwrap_or_else(|e| panic!("lium client: {e}")); + client + .list_offers(Some(2.0)) + .await + .unwrap_or_else(|e| panic!("lium list_offers: {e}")); +} diff --git a/crates/relearn-http/Cargo.toml b/crates/relearn-http/Cargo.toml new file mode 100644 index 000000000..298216d5a --- /dev/null +++ b/crates/relearn-http/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "relearn-http" +description = "Relearn HTTP surface: submit, status, admin promote" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"] } +hex = "0.4" +relearn-challenge-task = { path = "../relearn-challenge-task" } +relearn-eval = { path = "../relearn-eval" } +relearn-score = { path = "../relearn-score" } +relearn-store = { path = "../relearn-store" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt", "sync"] } + +[dev-dependencies] +http-body-util = "0.1" +tower = { version = "0.5", features = ["util"] } + +[lints] +workspace = true diff --git a/crates/relearn-http/src/lib.rs b/crates/relearn-http/src/lib.rs new file mode 100644 index 000000000..a8d4a73b6 --- /dev/null +++ b/crates/relearn-http/src/lib.rs @@ -0,0 +1,384 @@ +//! Relearn HTTP API (master-only). +//! +//! ```text +//! GET /health +//! GET /v1/status +//! POST /v1/submissions miner submit (digest + optional X-Lium-Api-Key) +//! GET /v1/submissions +//! GET /v1/submissions/{id} +//! POST /v1/admin/promote operator-audited champion flip +//! ``` + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::must_use_candidate, + clippy::items_after_statements, + clippy::too_many_lines +)] + +use std::sync::Arc; + +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use relearn_challenge_task::{CHALLENGE_ID, SCORE_MAX, SCORING_VERSION}; +use relearn_eval::{base_champion_scores, eval_after_freeze, resolve_teacher_backend, RelearnPin}; +use relearn_score::judge_challenger; +use relearn_store::{ + freeze_submission_digest, public_holdout, sealed_holdout, MemoryStore, Submission, + SubmissionState, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Shared HTTP state. +#[derive(Clone)] +pub struct AppState { + /// Submission store. + pub store: MemoryStore, + /// Eval / model pins. + pub pin: RelearnPin, + /// Operator bearer hashes (sha256 hex). Empty → admin 503. + pub admin_hashes: Arc>, +} + +/// Build the router. +pub fn relearn_router(state: AppState) -> Router { + Router::new() + .route("/health", get(health)) + .route("/v1/status", get(status)) + .route("/v1/submissions", post(submit).get(list_subs)) + .route("/v1/submissions/{id}", get(get_sub)) + .route("/v1/admin/promote", post(promote)) + .with_state(state) +} + +async fn health() -> impl IntoResponse { + Json(serde_json::json!({ + "ok": true, + "challenge_id": CHALLENGE_ID, + "scoring_version": SCORING_VERSION, + })) +} + +async fn status(State(st): State) -> impl IntoResponse { + let champ = st.store.champion_id().ok().flatten(); + Json(serde_json::json!({ + "challenge_id": CHALLENGE_ID, + "scoring_version": SCORING_VERSION, + "score_max": SCORE_MAX, + "base_model": st.pin.base_model, + "teacher_model": st.pin.teacher_model, + "teacher_backend": resolve_teacher_backend(), + "eval_image": st.pin.eval_image, + "eval_image_digest": st.pin.eval_image_digest, + "relearn_git": st.pin.relearn_git, + "relearn_git_sha": st.pin.relearn_git_sha, + "champion_id": champ, + })) +} + +#[derive(Debug, Deserialize)] +struct SubmitBody { + miner_hotkey: String, + artifact_digest: String, + artifact_uri: Option, +} + +#[derive(Debug, Serialize)] +struct SubmitResp { + id: String, + submission_digest: String, + state: SubmissionState, + holdout_unsealed: bool, + eligible: bool, +} + +fn parse_hex64(s: &str, field: &str) -> Result)> { + let t = s.trim().trim_start_matches("0x"); + if t.len() != 64 || !t.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({"error": format!("invalid {field}")})), + )); + } + Ok(t.to_ascii_lowercase()) +} + +fn nonce_from(hotkey: &str, digest: &str) -> String { + let mut h = Sha256::new(); + h.update(b"relearn-nonce-v1"); + h.update(hotkey.as_bytes()); + h.update(digest.as_bytes()); + hex::encode(h.finalize()) +} + +async fn submit( + State(st): State, + headers: HeaderMap, + Json(body): Json, +) -> Result)> { + let hotkey = parse_hex64(&body.miner_hotkey, "miner_hotkey")?; + let artifact = parse_hex64(&body.artifact_digest, "artifact_digest")?; + // Miner BYOK: accepted and never logged. Absence is OK for sim. + let _lium_present = headers + .get("x-lium-api-key") + .and_then(|v| v.to_str().ok()) + .is_some_and(|s| !s.is_empty()); + + let nonce = nonce_from(&hotkey, &artifact); + let submission_digest = freeze_submission_digest(&hotkey, &artifact, &nonce); + let pending = sealed_holdout(0, &submission_digest); + + let row = Submission { + id: String::new(), + miner_hotkey: hotkey, + artifact_digest: artifact.clone(), + artifact_uri: body.artifact_uri, + nonce, + submission_digest: submission_digest.clone(), + state: SubmissionState::Evaluating, + receipt_json: None, + verdict: None, + detail: None, + }; + let row = st + .store + .insert(row) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + + let eval = eval_after_freeze(&pending, &submission_digest, &artifact) + .map_err(|e| err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()))?; + let _public = public_holdout(&eval.holdout); + + let champ = st + .store + .champion_scores() + .ok() + .flatten() + .unwrap_or_else(base_champion_scores); + let verdict = judge_challenger(&champ, &eval.scores); + st.store + .record_scores(&row.id, eval.scores.clone()) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + let eligible = verdict.eligible; + let state = if eligible { + SubmissionState::AwaitingAdmin + } else { + SubmissionState::Rejected + }; + let receipt = serde_json::to_string(&eval.receipt).unwrap_or_default(); + let detail = if eligible { + None + } else { + Some(format!("gates={:?}", verdict.failed)) + }; + let row = st + .store + .patch(&row.id, Some(state), Some(receipt), Some(verdict), detail) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + + Ok(( + StatusCode::CREATED, + Json(SubmitResp { + id: row.id, + submission_digest: row.submission_digest, + state: row.state, + holdout_unsealed: eval.holdout.unsealed, + eligible, + }), + )) +} + +async fn list_subs(State(st): State) -> impl IntoResponse { + let rows = st.store.list().unwrap_or_default(); + Json(serde_json::json!({ "items": rows })) +} + +async fn get_sub( + State(st): State, + Path(id): Path, +) -> Result)> { + let row = st + .store + .get(&id) + .map_err(|_| err(StatusCode::NOT_FOUND, "not_found"))?; + Ok(Json(row)) +} + +#[derive(Debug, Deserialize)] +struct PromoteBody { + submission_id: String, +} + +async fn promote( + State(st): State, + headers: HeaderMap, + Json(body): Json, +) -> Result)> { + if st.admin_hashes.is_empty() { + return Err(err(StatusCode::SERVICE_UNAVAILABLE, "auth_unconfigured")); + } + if !admin_ok(&headers, &st.admin_hashes) { + return Err(err(StatusCode::UNAUTHORIZED, "unauthorized")); + } + let row = st.store.promote(&body.submission_id).map_err(|e| { + let code = if e.to_string().contains("unknown") { + StatusCode::NOT_FOUND + } else { + StatusCode::CONFLICT + }; + err(code, &e.to_string()) + })?; + Ok(Json(row)) +} + +fn admin_ok(headers: &HeaderMap, hashes: &[String]) -> bool { + let Some(raw) = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + else { + return false; + }; + let token = raw.strip_prefix("Bearer ").unwrap_or(raw).trim(); + if token.is_empty() { + return false; + } + let mut h = Sha256::new(); + h.update(token.as_bytes()); + let got = hex::encode(h.finalize()); + hashes.iter().any(|x| x == &got) +} + +fn err(code: StatusCode, msg: &str) -> (StatusCode, Json) { + (code, Json(serde_json::json!({ "error": msg }))) +} + +/// Hash an admin token the same way the server does. +#[must_use] +pub fn hash_admin_token(token: &str) -> String { + let mut h = Sha256::new(); + h.update(token.as_bytes()); + hex::encode(h.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::http::Request; + use http_body_util::BodyExt; + use tower::ServiceExt; + + fn digest(label: &str) -> String { + let mut h = Sha256::new(); + h.update(label.as_bytes()); + hex::encode(h.finalize()) + } + + async fn json_req( + app: Router, + method: &str, + uri: &str, + body: serde_json::Value, + auth: Option<&str>, + ) -> (StatusCode, serde_json::Value) { + let mut b = Request::builder().method(method).uri(uri); + if let Some(a) = auth { + b = b.header(axum::http::header::AUTHORIZATION, format!("Bearer {a}")); + } + let req = b + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("req"); + let resp = app.oneshot(req).await.expect("resp"); + let status = resp.status(); + let bytes = resp.into_body().collect().await.expect("body").to_bytes(); + let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap_or(serde_json::json!({})); + (status, v) + } + + #[tokio::test] + async fn submit_eval_promote_happy_path() { + let token = "op-test-token"; + let store = MemoryStore::new(); + store + .set_base_champion(base_champion_scores()) + .expect("base"); + let app = relearn_router(AppState { + store, + pin: RelearnPin::default(), + admin_hashes: Arc::new(vec![hash_admin_token(token)]), + }); + + let (st, health) = + json_req(app.clone(), "GET", "/health", serde_json::json!({}), None).await; + assert_eq!(st, StatusCode::OK); + assert_eq!(health["challenge_id"], CHALLENGE_ID); + + // High-byte digest tends to beat the base champion in sim. + let artifact = digest("miner-strong-adapter"); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + serde_json::json!({ + "miner_hotkey": digest("miner-hotkey"), + "artifact_digest": artifact, + }), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!( + created["submission_digest"].as_str().unwrap_or("").len(), + 64 + ); + assert!(created["holdout_unsealed"].as_bool().unwrap_or(false)); + + if created["eligible"] == true { + let id = created["id"].as_str().expect("id"); + let (st, promoted) = json_req( + app, + "POST", + "/v1/admin/promote", + serde_json::json!({ "submission_id": id }), + Some(token), + ) + .await; + assert_eq!(st, StatusCode::OK, "{promoted}"); + assert_eq!(promoted["state"], "champion"); + } + } + + #[tokio::test] + async fn promote_requires_bearer() { + let app = relearn_router(AppState { + store: MemoryStore::new(), + pin: RelearnPin::default(), + admin_hashes: Arc::new(vec![hash_admin_token("x")]), + }); + let (st, _) = json_req( + app, + "POST", + "/v1/admin/promote", + serde_json::json!({ "submission_id": "rl_0" }), + None, + ) + .await; + assert_eq!(st, StatusCode::UNAUTHORIZED); + } + + #[test] + fn model_pins_are_verified_ids() { + assert_eq!( + relearn_challenge_task::BASE_MODEL_ID, + "Qwen/Qwen3.8-Flash-Next" + ); + assert_eq!(relearn_challenge_task::TEACHER_MODEL_ID, "kimi-k3"); + } +} diff --git a/crates/relearn-mm-challenge/Cargo.toml b/crates/relearn-mm-challenge/Cargo.toml new file mode 100644 index 000000000..acbebe331 --- /dev/null +++ b/crates/relearn-mm-challenge/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "relearn-mm-challenge" +description = "Relearn Multimodal orchestrator: submit, eval, operator promote, D24 emit" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +bundle = { path = "../bundle" } +challenge-common = { path = "../challenge-common" } +crypto = { path = "../crypto" } +hex = "0.4" +relearn-mm-eval = { path = "../relearn-mm-eval" } +relearn-mm-http = { path = "../relearn-mm-http" } +relearn-mm-score = { path = "../relearn-mm-score" } +relearn-mm-store = { path = "../relearn-mm-store" } +relearn-mm-task = { path = "../relearn-mm-task" } +serde = { version = "1", features = ["derive"] } + +[lints] +workspace = true diff --git a/crates/relearn-mm-challenge/src/lib.rs b/crates/relearn-mm-challenge/src/lib.rs new file mode 100644 index 000000000..a99fc29b3 --- /dev/null +++ b/crates/relearn-mm-challenge/src/lib.rs @@ -0,0 +1,147 @@ +//! Relearn Multimodal orchestrator helpers: D24 leaf plan + crate re-exports. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] + +use std::collections::{BTreeMap, BTreeSet}; + +use bundle::{NoScoreReasonCode, ScoreOrAbsence}; +use challenge_common::{emit_signed_leaf_set, Hotkey, LeafEmitError}; +use relearn_mm_score::champion_hold_lattice; +use relearn_mm_store::SubmissionState; +use relearn_mm_task::{CHALLENGE_ID_BYTES, SCORE_MAX}; + +pub use relearn_mm_eval::{resolve_backend, EvalBackend}; +pub use relearn_mm_http::{hash_admin_token, relearn_mm_router, AppState}; +pub use relearn_mm_store::{EncoderManifest, MemoryStore}; +pub use relearn_mm_task::{ + RelearnMmPin, SubmissionKind, CHALLENGE_ID, CHALLENGE_ID_BYTES as RELEARN_MM_ID_BYTES, + ENCODER_MODEL_ID, LM_BASE_MODEL_ID, SCORE_MAX as RELEARN_MM_SCORE_MAX, SCORING_VERSION, +}; + +/// Build a D24-complete score map: champion (if any) gets a positive lattice; +/// everyone else is explicit `NoScore` (never silent). +pub fn emission_scores( + expected: &BTreeSet, + champion_hotkey: Option, + champion_lattice: u64, +) -> BTreeMap { + expected + .iter() + .map(|h| { + let s = match champion_hotkey { + Some(c) if c == *h && champion_lattice > 0 => ScoreOrAbsence::Score { + value: champion_lattice.min(SCORE_MAX), + }, + _ => ScoreOrAbsence::NoScore { + reason: NoScoreReasonCode::NotAttempted, + }, + }; + (*h, s) + }) + .collect() +} + +/// Sign the exact-E leaf set for this epoch. +/// +/// # Errors +/// +/// See [`LeafEmitError`]. +pub fn emit_epoch( + secret: &[u8; 32], + epoch: u64, + expected: &BTreeSet, + champion_hotkey: Option, + champion_lattice: u64, +) -> Result, LeafEmitError> { + let scores = emission_scores(expected, champion_hotkey, champion_lattice); + emit_signed_leaf_set(secret, CHALLENGE_ID_BYTES, epoch, expected, &scores) +} + +/// Lattice for the current store champion, or the hold value when only the +/// pinned encoder baseline is live. +pub fn live_champion_lattice(store: &MemoryStore) -> u64 { + if let Ok(Some(id)) = store.champion_id() { + if let Ok(row) = store.get(&id) { + if row.state == SubmissionState::Champion { + if let Some(v) = row.verdict { + if v.eligible && v.lattice > 0 { + return v.lattice; + } + } + return champion_hold_lattice(); + } + } + } + 0 +} + +/// Parse a 64-hex hotkey. +pub fn parse_hotkey(hex_s: &str) -> Option { + let t = hex_s.trim().trim_start_matches("0x"); + let bytes = hex::decode(t).ok()?; + <[u8; 32]>::try_from(bytes).ok() +} + +#[cfg(test)] +mod tests { + use challenge_common::public_key_from_secret; + use crypto::KEY_LEN; + + use super::*; + + fn sk() -> [u8; KEY_LEN] { + let mut s = [11u8; KEY_LEN]; + s[0] = 5; + s + } + + #[test] + fn d24_covers_every_hotkey() { + let a = [1u8; 32]; + let b = [2u8; 32]; + let e: BTreeSet = [a, b].into_iter().collect(); + let leaves = emit_epoch(&sk(), 4, &e, Some(a), 9_000).expect("emit"); + assert_eq!(leaves.len(), 2); + assert!(matches!( + leaves[&a].score_or_absence, + ScoreOrAbsence::Score { value: 9_000 } + )); + assert!(matches!( + leaves[&b].score_or_absence, + ScoreOrAbsence::NoScore { .. } + )); + let pk = public_key_from_secret(&sk()).expect("pk"); + for leaf in leaves.values() { + challenge_common::verify_leaf_sig(leaf, &pk).expect("sig"); + } + } + + #[test] + fn leaf_domain_is_distinct_from_the_other_relearn_challenges() { + assert_eq!(RELEARN_MM_ID_BYTES, b"relearn-mm"); + assert_ne!(RELEARN_MM_ID_BYTES, b"relearn"); + assert_ne!(RELEARN_MM_ID_BYTES, b"relearn-t2i"); + } + + #[test] + fn never_emits_score_on_empty_expected() { + let leaves = emit_epoch(&sk(), 1, &BTreeSet::new(), None, 0).expect("empty E"); + assert!(leaves.is_empty()); + } + + #[test] + fn no_champion_means_zero_lattice() { + assert_eq!(live_champion_lattice(&MemoryStore::new()), 0); + } + + #[test] + fn hotkey_parse_round_trip() { + assert!(parse_hotkey(&"cd".repeat(32)).is_some()); + assert!(parse_hotkey("nope").is_none()); + } +} diff --git a/crates/relearn-mm-eval/Cargo.toml b/crates/relearn-mm-eval/Cargo.toml new file mode 100644 index 000000000..0d92d61df --- /dev/null +++ b/crates/relearn-mm-eval/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "relearn-mm-eval" +description = "Relearn Multimodal eval loop: text-intact rerun, vision holdout, pixel-shuffle control" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +prism-competition = { path = "../prism-competition" } +prism-lium = { path = "../prism-lium" } +prism-lium-types = { path = "../prism-lium-types" } +relearn-challenge-task = { path = "../relearn-challenge-task" } +relearn-mm-score = { path = "../relearn-mm-score" } +relearn-mm-store = { path = "../relearn-mm-store" } +relearn-mm-task = { path = "../relearn-mm-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/crates/relearn-mm-eval/src/lib.rs b/crates/relearn-mm-eval/src/lib.rs new file mode 100644 index 000000000..ef01664cb --- /dev/null +++ b/crates/relearn-mm-eval/src/lib.rs @@ -0,0 +1,422 @@ +//! Relearn Multimodal eval loop. +//! +//! Order matters: the text-intact rerun happens first, because it is the gate +//! that can zero the submission, and running it first means a damaged language +//! model is caught before any GPU time is spent on vision benchmarks. +//! +//! Backend resolution is fail-closed, the same way the T2I challenge does it. A +//! host with no eval endpoint and no explicit sim opt-in refuses to score +//! rather than producing a deterministic placeholder that reads like a pass. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::must_use_candidate +)] + +use std::collections::BTreeMap; + +use prism_competition::ExampleSeries; +use prism_lium::{EvalJobBackend, SimLiumBackend}; +use prism_lium_types::{EvalReceipt, InstanceSpec, NoScoreGate, RemoteExecResult}; +use relearn_mm_score::{AgenticEvidence, MmSliceScores, MIN_SHUFFLE_DROP}; +use relearn_mm_store::EncoderManifest; +use relearn_mm_task::{PinError, RelearnMmPin, VisionTask}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +/// Where the multimodal eval runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvalBackend { + /// Digest-pinned eval image on a Lium pod (production default). + Lium, + /// Deterministic offline eval (CI / local only). + Sim, +} + +/// True when the operator explicitly opted into sim. +#[must_use] +pub fn force_sim() -> bool { + matches!( + std::env::var("RELEARN_MM_FORCE_SIM") + .unwrap_or_default() + .to_ascii_lowercase() + .as_str(), + "1" | "true" | "yes" + ) +} + +/// Resolve the eval backend for this host. +#[must_use] +pub fn resolve_backend() -> EvalBackend { + if force_sim() { + EvalBackend::Sim + } else { + EvalBackend::Lium + } +} + +/// Eval errors. +#[derive(Debug, Error)] +pub enum EvalError { + /// The submission digest was not frozen before scoring. + #[error("submission digest not frozen")] + NotFrozen, + /// Integrity gate failed. + #[error("integrity: {0}")] + Integrity(String), + /// Lium / backend failure. + #[error("backend: {0}")] + Backend(String), + /// The encoder license is not permissive. + #[error("encoder attestation: {0}")] + Attestation(#[from] PinError), + /// The pinned eval image has no digest and sim was not opted into. + #[error("eval image digest not pinned; set RELEARN_MM_FORCE_SIM=1 for CI")] + EvalImageUnpinned, +} + +/// One finished eval. +#[derive(Debug, Clone)] +pub struct EvalOutcome { + /// Challenger measurements. + pub scores: MmSliceScores, + /// Integrity receipt. + pub receipt: EvalReceipt, + /// Backend that produced the scores. + pub backend: EvalBackend, + /// Text holdout items rerun for gate 1. + pub text_items: usize, + /// Vision holdout items scored for gate 2. + pub vision_items: usize, +} + +fn unit(parts: &[&str], index: usize) -> f64 { + let mut h = Sha256::new(); + for p in parts { + h.update(p.as_bytes()); + h.update([0xff]); + } + h.update(u32::try_from(index).unwrap_or(0).to_le_bytes()); + let d = h.finalize(); + f64::from(d[0]) / 255.0 +} + +fn sim_series(prefix: &str, salt: &[&str], n: usize, base: f64) -> ExampleSeries { + ExampleSeries::from_pairs((0..n).map(|i| { + ( + format!("{prefix}{i}"), + (base + 0.25 * unit(salt, i)).clamp(0.0, 1.0), + ) + })) +} + +/// Deterministic sim measurements for a frozen digest. +/// +/// `text_base` is separate from `vision_base` on purpose: the sim harness has +/// to be able to produce a submission that wins on vision while regressing the +/// text side, because that is the case gate 1 exists to reject. +#[must_use] +pub fn sim_slice_scores( + pin: &RelearnMmPin, + artifact_digest: &str, + manifest: &EncoderManifest, + text_base: f64, + vision_base: f64, +) -> MmSliceScores { + let salt = [artifact_digest, "mm"]; + let mut vision_by_task = BTreeMap::new(); + for task in VisionTask::ALL { + vision_by_task.insert( + task, + sim_series( + task.as_str(), + &[artifact_digest, task.as_str()], + pin.vision_items_per_task, + vision_base, + ), + ); + } + let pooled: Vec<(String, f64)> = vision_by_task + .iter() + .flat_map(|(task, series)| { + series + .by_cluster + .iter() + .map(move |(k, v)| (format!("{}/{k}", task.as_str()), *v)) + }) + .collect(); + + let agentic_series = sim_series( + "agentic", + &[artifact_digest, "agentic"], + pin.agentic_traces.max(1), + vision_base, + ); + let agentic_mean = MmSliceScores::mean(&agentic_series).unwrap_or(0.0); + + MmSliceScores { + text_holdout: sim_series("t", &salt, pin.text_holdout_items, text_base), + vision_holdout: ExampleSeries::from_pairs(pooled), + vision_by_task, + agentic: AgenticEvidence { + traces: u32::try_from(pin.agentic_traces).unwrap_or(u32::MAX), + score: agentic_mean, + // A model that reads the image loses most of the signal when the + // pixels are destroyed; sim models that. + shuffled_score: (agentic_mean - 2.0 * MIN_SHUFFLE_DROP).max(0.0), + }, + agentic_series, + vision_public: sim_series("vp", &[artifact_digest, "public"], 120, vision_base), + lm_weights_hash: manifest.lm_weights_hash.trim().to_ascii_lowercase(), + kind: manifest.kind, + } +} + +/// Baseline champion: pinned encoder on the champion LM, no miner training. +#[must_use] +pub fn base_champion_scores(pin: &RelearnMmPin, champion_lm_hash: &str) -> MmSliceScores { + let manifest = EncoderManifest { + encoder_model: pin.encoder_model.clone(), + encoder_license: pin.encoder_license.clone(), + lm_weights_hash: champion_lm_hash.to_owned(), + ..EncoderManifest::default() + }; + sim_slice_scores(pin, "relearn-mm-baseline", &manifest, 0.70, 0.45) +} + +/// Run one eval after the submission digest is frozen. +/// +/// Gate 1's text rerun is produced before the vision splits, so a submission +/// that damaged the language model is refused early. +/// +/// # Errors +/// +/// [`EvalError::NotFrozen`], [`EvalError::Attestation`] for a non-permissive +/// encoder license, and [`EvalError::EvalImageUnpinned`] when a live run was +/// requested without a digest-pinned eval image. +pub fn eval_after_freeze( + pin: &RelearnMmPin, + frozen_digest: &str, + artifact_digest: &str, + manifest: &EncoderManifest, + backend: EvalBackend, +) -> Result { + if frozen_digest.trim().is_empty() { + return Err(EvalError::NotFrozen); + } + pin.attest_encoder(&manifest.encoder_model, &manifest.encoder_license)?; + + let scores = match backend { + EvalBackend::Sim => sim_slice_scores(pin, artifact_digest, manifest, 0.72, 0.62), + EvalBackend::Lium => { + if !pin.can_rent() { + return Err(EvalError::EvalImageUnpinned); + } + return Err(EvalError::Integrity( + "live multimodal harvest is driven by the eval image; no in-process fallback" + .into(), + )); + } + }; + + let text_items = scores.text_holdout.len(); + let vision_items = scores.vision_holdout.len(); + let metrics = serde_json::to_vec(&serde_json::json!({ + "lm_base_model": pin.lm_base_model, + "encoder_model": manifest.encoder_model, + "text_items": text_items, + "vision_items": vision_items, + "shuffle_drop": scores.agentic.shuffle_drop(), + })) + .unwrap_or_default(); + let receipt = EvalReceipt { + provider: match backend { + EvalBackend::Sim => "sim".into(), + EvalBackend::Lium => "lium".into(), + }, + pod_id: format!("mm-{}", &frozen_digest[..8.min(frozen_digest.len())]), + image_digest: pin.eval_image_digest.clone(), + submission_hash: frozen_digest.to_owned(), + metrics_hash: EvalReceipt::hash_metrics_bytes(&metrics), + termination_verified: true, + }; + NoScoreGate::check(&receipt, false).map_err(|e| EvalError::Integrity(e.to_string()))?; + + Ok(EvalOutcome { + scores, + receipt, + backend, + text_items, + vision_items, + }) +} + +/// Rent a digest-pinned eval pod, exec, harvest, terminate. +/// +/// # Errors +/// +/// [`EvalError::EvalImageUnpinned`] without a digest pin, +/// [`EvalError::Integrity`] on an unverified teardown, and +/// [`EvalError::Backend`] on any provider failure. +pub async fn rent_eval( + backend: &dyn EvalJobBackend, + pin: &RelearnMmPin, + frozen_digest: &str, + artifact_digest: &str, +) -> Result<(RemoteExecResult, String), EvalError> { + if !pin.can_rent() { + return Err(EvalError::EvalImageUnpinned); + } + let spec = InstanceSpec { + name: format!( + "relearn-mm-{}", + &frozen_digest[..12.min(frozen_digest.len())] + ), + max_lifetime_hours: 2.0, + max_price_per_hour: 16.0, + gpu_count: 2, + image_digest: Some(pin.eval_image_digest.clone()), + ssh_public_keys: Vec::new(), + ssh_key_name: None, + preferred_offer_id: None, + template_id: None, + template_name: None, + }; + let inst = backend + .provision(&spec) + .await + .map_err(|e| EvalError::Backend(e.to_string()))?; + let exec = backend + .exec_eval(&inst.id, artifact_digest, frozen_digest, None) + .await + .map_err(|e| EvalError::Backend(e.to_string())); + let term = backend.terminate(&inst.id).await; + let verified = backend.verify_terminated(&inst.id).await.unwrap_or(false); + if let Err(e) = term { + return Err(EvalError::Backend(e.to_string())); + } + if !verified { + return Err(EvalError::Integrity("pod terminate not verified".into())); + } + exec.map(|r| (r, inst.id)) +} + +/// Convenience: sim backend rent that always tears down. +/// +/// # Errors +/// +/// See [`rent_eval`]. +pub async fn sim_rent_roundtrip(digest: &str) -> Result { + let lium = SimLiumBackend::new(); + let pin = RelearnMmPin { + eval_image_digest: format!("sha256:{}", "cd".repeat(32)), + ..RelearnMmPin::default() + }; + let (_r, id) = rent_eval(&lium, &pin, digest, digest).await?; + Ok(id) +} + +#[cfg(test)] +mod tests { + use relearn_mm_task::SubmissionKind; + + use super::*; + + fn manifest() -> EncoderManifest { + EncoderManifest { + encoder_model: relearn_mm_task::ENCODER_MODEL_ID.into(), + encoder_license: "apache-2.0".into(), + projector: "2-layer MLP".into(), + kind: SubmissionKind::EncoderOnly, + lm_weights_hash: "aaaa1111".into(), + } + } + + #[test] + fn sim_eval_needs_a_frozen_digest() { + let pin = RelearnMmPin::default(); + assert!(matches!( + eval_after_freeze(&pin, "", "art", &manifest(), EvalBackend::Sim), + Err(EvalError::NotFrozen) + )); + } + + #[test] + fn sim_eval_fills_both_gates_and_is_deterministic() { + let pin = RelearnMmPin::default(); + let a = eval_after_freeze(&pin, "d", "art", &manifest(), EvalBackend::Sim).expect("a"); + let b = eval_after_freeze(&pin, "d", "art", &manifest(), EvalBackend::Sim).expect("b"); + assert_eq!(a.scores.text_holdout, b.scores.text_holdout); + assert_eq!(a.text_items, pin.text_holdout_items); + assert_eq!(a.vision_items, pin.vision_items_per_task * 4); + assert_eq!(a.scores.vision_by_task.len(), 4); + assert_eq!(a.receipt.provider, "sim"); + assert!(a.scores.agentic.uses_the_image()); + } + + #[test] + fn non_permissive_encoder_is_refused_before_any_scoring() { + let pin = RelearnMmPin::default(); + let mut m = manifest(); + m.encoder_license = "creativeml-openrail-m".into(); + let err = eval_after_freeze(&pin, "d", "art", &m, EvalBackend::Sim).expect_err("refuse"); + assert!( + matches!(err, EvalError::Attestation(PinError::EncoderLicense(_))), + "{err}" + ); + } + + #[test] + fn live_backend_without_a_pinned_image_refuses() { + let pin = RelearnMmPin::default(); + assert!(!pin.can_rent()); + assert!(matches!( + eval_after_freeze(&pin, "d", "art", &manifest(), EvalBackend::Lium), + Err(EvalError::EvalImageUnpinned) + )); + } + + #[test] + fn sim_harness_can_express_a_vision_win_with_a_text_regression() { + let pin = RelearnMmPin::default(); + let damaged = sim_slice_scores(&pin, "art", &manifest(), 0.30, 0.90); + let champion = base_champion_scores(&pin, "aaaa1111"); + assert!( + MmSliceScores::mean(&damaged.text_holdout).unwrap_or(0.0) + < MmSliceScores::mean(&champion.text_holdout).unwrap_or(1.0) + ); + assert!( + MmSliceScores::mean(&damaged.vision_holdout).unwrap_or(0.0) + > MmSliceScores::mean(&champion.vision_holdout).unwrap_or(1.0) + ); + } + + #[test] + fn baseline_champion_carries_the_champion_lm_hash_canonicalized() { + let pin = RelearnMmPin::default(); + let base = base_champion_scores(&pin, "AAAA1111"); + assert_eq!( + base.lm_weights_hash, "aaaa1111", + "hashes are canonicalized so a case difference is not a mismatch" + ); + assert_eq!(base.text_holdout.len(), pin.text_holdout_items); + } + + #[test] + fn backend_defaults_to_lium_without_the_sim_opt_in() { + assert_eq!(resolve_backend(), EvalBackend::Lium); + assert!(!force_sim()); + } + + #[tokio::test] + async fn sim_rent_tears_down() { + let id = sim_rent_roundtrip("abcdef0123456789").await.expect("rent"); + assert!(id.contains("sim-pod")); + } +} diff --git a/crates/relearn-mm-http/Cargo.toml b/crates/relearn-mm-http/Cargo.toml new file mode 100644 index 000000000..7b6b9fddf --- /dev/null +++ b/crates/relearn-mm-http/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "relearn-mm-http" +description = "Relearn Multimodal HTTP surface: submit, status, admin promote" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"] } +hex = "0.4" +relearn-mm-eval = { path = "../relearn-mm-eval" } +relearn-mm-score = { path = "../relearn-mm-score" } +relearn-mm-store = { path = "../relearn-mm-store" } +relearn-mm-task = { path = "../relearn-mm-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt", "sync"] } + +[dev-dependencies] +http-body-util = "0.1" +tower = { version = "0.5", features = ["util"] } + +[lints] +workspace = true diff --git a/crates/relearn-mm-http/src/lib.rs b/crates/relearn-mm-http/src/lib.rs new file mode 100644 index 000000000..766b1f0fd --- /dev/null +++ b/crates/relearn-mm-http/src/lib.rs @@ -0,0 +1,480 @@ +//! Relearn Multimodal HTTP API (master-only). +//! +//! ```text +//! GET /health +//! GET /v1/status +//! POST /v1/submissions miner submit (digest + encoder manifest) +//! GET /v1/submissions +//! GET /v1/submissions/{id} +//! POST /v1/admin/promote operator-audited champion flip +//! ``` +//! +//! `/v1/status` publishes the champion LM weights hash: an encoder-only miner +//! needs it to know which language model to attach to, and publishing it costs +//! nothing because it is a hash of already-public champion weights. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate, + clippy::too_many_lines +)] + +use std::sync::Arc; + +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use relearn_mm_eval::{eval_after_freeze, EvalBackend}; +use relearn_mm_score::{judge_challenger, LM_EPSILON, MIN_SHUFFLE_DROP}; +use relearn_mm_store::{ + freeze_submission_digest, EncoderManifest, MemoryStore, Submission, SubmissionState, +}; +use relearn_mm_task::{ + license_is_permissive, RelearnMmPin, VisionTask, CHALLENGE_ID, PERMISSIVE_LICENSES, SCORE_MAX, + SCORING_VERSION, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Shared HTTP state. +#[derive(Clone)] +pub struct AppState { + /// Submission store. + pub store: MemoryStore, + /// Eval / model pins. + pub pin: RelearnMmPin, + /// Eval backend resolved once at boot. + pub backend: EvalBackend, + /// Operator bearer hashes (sha256 hex). Empty → admin 503. + pub admin_hashes: Arc>, +} + +/// Build the router. +pub fn relearn_mm_router(state: AppState) -> Router { + Router::new() + .route("/health", get(health)) + .route("/v1/status", get(status)) + .route("/v1/submissions", post(submit).get(list_subs)) + .route("/v1/submissions/{id}", get(get_sub)) + .route("/v1/admin/promote", post(promote)) + .with_state(state) +} + +async fn health() -> impl IntoResponse { + Json(serde_json::json!({ + "ok": true, + "challenge_id": CHALLENGE_ID, + "scoring_version": SCORING_VERSION, + })) +} + +async fn status(State(st): State) -> impl IntoResponse { + Json(serde_json::json!({ + "challenge_id": CHALLENGE_ID, + "scoring_version": SCORING_VERSION, + "score_max": SCORE_MAX, + "lm_base_model": st.pin.lm_base_model, + "encoder_model": st.pin.encoder_model, + "encoder_license": st.pin.encoder_license, + "encoder_revision": st.pin.encoder_revision, + "permissive_licenses": PERMISSIVE_LICENSES, + "eval_image": st.pin.eval_image, + "eval_image_digest": st.pin.eval_image_digest, + "eval_backend": st.backend, + "vision_tasks": VisionTask::ALL.map(VisionTask::as_str), + "vision_items_per_task": st.pin.vision_items_per_task, + "agentic_traces": st.pin.agentic_traces, + "text_holdout_items": st.pin.text_holdout_items, + "lm_epsilon": LM_EPSILON, + "min_shuffle_drop": MIN_SHUFFLE_DROP, + "champion_lm_weights_hash": st.store.champion_lm_hash().unwrap_or_default(), + "champion_id": st.store.champion_id().ok().flatten(), + })) +} + +#[derive(Debug, Deserialize)] +struct SubmitBody { + miner_hotkey: String, + artifact_digest: String, + artifact_uri: Option, + #[serde(default)] + manifest: EncoderManifest, +} + +#[derive(Debug, Serialize)] +struct SubmitResp { + id: String, + submission_digest: String, + state: SubmissionState, + eval_backend: EvalBackend, + text_items: usize, + vision_items: usize, + lm_intact: bool, + eligible: bool, +} + +fn parse_hex64(s: &str, field: &str) -> Result)> { + let t = s.trim().trim_start_matches("0x"); + if t.len() != 64 || !t.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(err(StatusCode::BAD_REQUEST, &format!("invalid {field}"))); + } + Ok(t.to_ascii_lowercase()) +} + +fn nonce_from(hotkey: &str, digest: &str) -> String { + let mut h = Sha256::new(); + h.update(b"relearn-mm-nonce-v1"); + h.update(hotkey.as_bytes()); + h.update(digest.as_bytes()); + hex::encode(h.finalize()) +} + +async fn submit( + State(st): State, + headers: HeaderMap, + Json(body): Json, +) -> Result)> { + let hotkey = parse_hex64(&body.miner_hotkey, "miner_hotkey")?; + let artifact = parse_hex64(&body.artifact_digest, "artifact_digest")?; + // Miner BYOK: accepted and never logged. + let _lium_present = headers + .get("x-lium-api-key") + .and_then(|v| v.to_str().ok()) + .is_some_and(|s| !s.is_empty()); + + st.pin + .attest_encoder(&body.manifest.encoder_model, &body.manifest.encoder_license) + .map_err(|e| err(StatusCode::BAD_REQUEST, &e.to_string()))?; + + let nonce = nonce_from(&hotkey, &artifact); + let submission_digest = freeze_submission_digest(&hotkey, &artifact, &nonce); + + let row = st + .store + .insert(Submission { + id: String::new(), + miner_hotkey: hotkey, + artifact_digest: artifact.clone(), + artifact_uri: body.artifact_uri, + manifest: body.manifest.clone(), + nonce, + submission_digest: submission_digest.clone(), + state: SubmissionState::Evaluating, + receipt_json: None, + verdict: None, + detail: None, + }) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + + let eval = eval_after_freeze( + &st.pin, + &submission_digest, + &artifact, + &body.manifest, + st.backend, + ) + .map_err(|e| err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()))?; + + let champ = st.store.champion_scores().ok().flatten().ok_or_else(|| { + err( + StatusCode::SERVICE_UNAVAILABLE, + "no champion baseline recorded", + ) + })?; + let champion_lm_hash = st.store.champion_lm_hash().unwrap_or_default(); + let verdict = judge_challenger( + &champ, + &eval.scores, + &champion_lm_hash, + license_is_permissive(&body.manifest.encoder_license), + ); + st.store + .record_scores(&row.id, eval.scores.clone()) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + + let eligible = verdict.eligible; + let lm_intact = verdict.lm_intact.is_some_and(|g| g.passes); + let state = if eligible { + SubmissionState::AwaitingAdmin + } else { + SubmissionState::Rejected + }; + let detail = if eligible { + None + } else { + Some(format!("gates={:?}", verdict.failed)) + }; + let receipt = serde_json::to_string(&eval.receipt).unwrap_or_default(); + let row = st + .store + .patch(&row.id, Some(state), Some(receipt), Some(verdict), detail) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + + Ok(( + StatusCode::CREATED, + Json(SubmitResp { + id: row.id, + submission_digest: row.submission_digest, + state: row.state, + eval_backend: eval.backend, + text_items: eval.text_items, + vision_items: eval.vision_items, + lm_intact, + eligible, + }), + )) +} + +async fn list_subs(State(st): State) -> impl IntoResponse { + let rows = st.store.list().unwrap_or_default(); + Json(serde_json::json!({ "items": rows })) +} + +async fn get_sub( + State(st): State, + Path(id): Path, +) -> Result)> { + let row = st + .store + .get(&id) + .map_err(|_| err(StatusCode::NOT_FOUND, "not_found"))?; + Ok(Json(row)) +} + +#[derive(Debug, Deserialize)] +struct PromoteBody { + submission_id: String, +} + +async fn promote( + State(st): State, + headers: HeaderMap, + Json(body): Json, +) -> Result)> { + if st.admin_hashes.is_empty() { + return Err(err(StatusCode::SERVICE_UNAVAILABLE, "auth_unconfigured")); + } + if !admin_ok(&headers, &st.admin_hashes) { + return Err(err(StatusCode::UNAUTHORIZED, "unauthorized")); + } + let row = st.store.promote(&body.submission_id).map_err(|e| { + let code = if e.to_string().contains("unknown") { + StatusCode::NOT_FOUND + } else { + StatusCode::CONFLICT + }; + err(code, &e.to_string()) + })?; + Ok(Json(row)) +} + +fn admin_ok(headers: &HeaderMap, hashes: &[String]) -> bool { + let Some(raw) = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + else { + return false; + }; + let token = raw.strip_prefix("Bearer ").unwrap_or(raw).trim(); + if token.is_empty() { + return false; + } + let mut h = Sha256::new(); + h.update(token.as_bytes()); + let got = hex::encode(h.finalize()); + hashes.iter().any(|x| x == &got) +} + +fn err(code: StatusCode, msg: &str) -> (StatusCode, Json) { + (code, Json(serde_json::json!({ "error": msg }))) +} + +/// Hash an admin token the same way the server does. +#[must_use] +pub fn hash_admin_token(token: &str) -> String { + let mut h = Sha256::new(); + h.update(token.as_bytes()); + hex::encode(h.finalize()) +} + +#[cfg(test)] +mod tests { + use axum::body::Body; + use axum::http::Request; + use http_body_util::BodyExt; + use relearn_mm_eval::base_champion_scores; + use tower::ServiceExt; + + use super::*; + + const CHAMP_HASH: &str = "aaaa1111"; + + fn app(token: &str) -> Router { + let pin = RelearnMmPin::default(); + let store = MemoryStore::new(); + store.set_champion_lm_hash(CHAMP_HASH).expect("hash"); + store + .set_base_champion(base_champion_scores(&pin, CHAMP_HASH)) + .expect("base"); + relearn_mm_router(AppState { + store, + pin, + backend: EvalBackend::Sim, + admin_hashes: Arc::new(vec![hash_admin_token(token)]), + }) + } + + fn digest(label: &str) -> String { + let mut h = Sha256::new(); + h.update(label.as_bytes()); + hex::encode(h.finalize()) + } + + fn manifest(license: &str, kind: &str, lm_hash: &str) -> serde_json::Value { + serde_json::json!({ + "encoder_model": "google/siglip2-so400m-patch14-384", + "encoder_license": license, + "projector": "2-layer MLP", + "kind": kind, + "lm_weights_hash": lm_hash, + }) + } + + fn submit_body(label: &str, manifest: &serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "miner_hotkey": digest("miner-hotkey"), + "artifact_digest": digest(label), + "manifest": manifest, + }) + } + + async fn json_req( + app: Router, + method: &str, + uri: &str, + body: serde_json::Value, + auth: Option<&str>, + ) -> (StatusCode, serde_json::Value) { + let mut b = Request::builder().method(method).uri(uri); + if let Some(a) = auth { + b = b.header(axum::http::header::AUTHORIZATION, format!("Bearer {a}")); + } + let req = b + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("req"); + let resp = app.oneshot(req).await.expect("resp"); + let status = resp.status(); + let bytes = resp.into_body().collect().await.expect("body").to_bytes(); + let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap_or(serde_json::json!({})); + (status, v) + } + + #[tokio::test] + async fn status_publishes_the_pins_and_the_champion_lm_hash() { + let (st, body) = + json_req(app("op"), "GET", "/v1/status", serde_json::json!({}), None).await; + assert_eq!(st, StatusCode::OK); + assert_eq!(body["lm_base_model"], "Qwen/Qwen3.8-Flash-Next"); + assert_eq!(body["encoder_model"], "google/siglip2-so400m-patch14-384"); + assert_eq!(body["encoder_license"], "apache-2.0"); + assert_eq!(body["champion_lm_weights_hash"], CHAMP_HASH); + assert_eq!(body["eval_backend"], "sim"); + let tasks = body["vision_tasks"].as_array().expect("tasks"); + assert_eq!(tasks.len(), 4); + assert!(tasks.iter().any(|t| t == "ocr")); + } + + #[tokio::test] + async fn encoder_only_submission_with_the_champion_lm_can_promote() { + let token = "op-test-token"; + let app = app(token); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body( + "trained-encoder", + &manifest("apache-2.0", "encoder_only", CHAMP_HASH), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["eval_backend"], "sim"); + assert_eq!(created["lm_intact"], true); + assert_eq!(created["text_items"], 120); + assert_eq!(created["vision_items"], 160); + assert_eq!(created["eligible"], true, "{created}"); + + let id = created["id"].as_str().expect("id"); + let (st, promoted) = json_req( + app, + "POST", + "/v1/admin/promote", + serde_json::json!({ "submission_id": id }), + Some(token), + ) + .await; + assert_eq!(st, StatusCode::OK, "{promoted}"); + assert_eq!(promoted["state"], "champion"); + } + + #[tokio::test] + async fn encoder_only_submission_with_a_different_lm_is_rejected() { + let (st, created) = json_req( + app("op"), + "POST", + "/v1/submissions", + submit_body( + "swapped-lm", + &manifest("apache-2.0", "encoder_only", "dddd4444"), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["eligible"], false); + assert_eq!(created["state"], "rejected"); + } + + #[tokio::test] + async fn non_permissive_encoder_is_rejected_at_the_door() { + let (st, body) = json_req( + app("op"), + "POST", + "/v1/submissions", + submit_body( + "openrail-encoder", + &manifest("creativeml-openrail-m", "encoder_only", CHAMP_HASH), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert!( + body["error"] + .as_str() + .unwrap_or("") + .contains("OSI-permissive"), + "{body}" + ); + } + + #[tokio::test] + async fn promote_requires_bearer() { + let (st, _) = json_req( + app("op"), + "POST", + "/v1/admin/promote", + serde_json::json!({ "submission_id": "mm_0" }), + None, + ) + .await; + assert_eq!(st, StatusCode::UNAUTHORIZED); + } +} diff --git a/crates/relearn-mm-score/Cargo.toml b/crates/relearn-mm-score/Cargo.toml new file mode 100644 index 000000000..7dcb518de --- /dev/null +++ b/crates/relearn-mm-score/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "relearn-mm-score" +description = "Relearn Multimodal scoring: LLM-intact hard gate, vision holdout, pixel-shuffle check" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +prism-competition = { path = "../prism-competition" } +relearn-mm-task = { path = "../relearn-mm-task" } +serde = { version = "1", features = ["derive"] } + +[lints] +workspace = true diff --git a/crates/relearn-mm-score/src/lib.rs b/crates/relearn-mm-score/src/lib.rs new file mode 100644 index 000000000..42323704f --- /dev/null +++ b/crates/relearn-mm-score/src/lib.rs @@ -0,0 +1,606 @@ +//! Scoring for Relearn Multimodal. Two gates, both mandatory. +//! +//! **Gate 1 — the LLM is intact.** The submitted language model is run on the +//! existing Relearn text holdout with the vision modules ignored. If it scores +//! below `champion − ε` the submission is worth zero on this challenge no matter +//! how good the vision numbers are. This is the whole point of the challenge +//! design: attaching an encoder must not be a way to get paid for damaging the +//! champion. For an encoder-only submission the LM weights must additionally +//! hash-match the champion, which is the proof that nothing on the text side +//! moved at all. +//! +//! **Gate 2 — the vision side actually improved.** A frozen image holdout +//! (captioning, VQA, OCR, spatial relations — deliberately not ImageNet or COCO +//! test, which are in every candidate encoder's pretraining mix) plus agentic +//! traces where the model has to look at a screenshot or diagram before calling +//! a tool. Displacement is measured against the champion with the same paired +//! test the text challenge uses. +//! +//! The agentic traces carry one extra check: the same traces are replayed with +//! the image pixels shuffled. A model that is really reading the image must get +//! materially worse; one that is pattern-matching the text prompt will not +//! move, and a flat shuffle delta means the vision win is not real. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss +)] + +use std::collections::BTreeMap; + +use prism_competition::{ + paired_test, Direction, ExampleSeries, PairedInput, PairedOutcome, PairedRefusal, DEADZONE, +}; +use relearn_mm_task::{SubmissionKind, VisionTask, VisionTaskWeights, SCORE_MAX}; +use serde::{Deserialize, Serialize}; + +/// Largest text-holdout drop tolerated versus the champion (absolute). +/// +/// Small on purpose. The challenge pays for vision, and the text side is a +/// floor rather than a budget to spend. +pub const LM_EPSILON: f64 = 0.01; + +/// Minimum score drop required when the agentic images are pixel-shuffled. +/// +/// A model that ignores the image scores the same on shuffled pixels. Requiring +/// a real drop is what separates seeing from guessing from the prompt. +pub const MIN_SHUFFLE_DROP: f64 = 0.10; + +/// Slice id for the text-intact comparison. +pub const TEXT_SLICE_ID: &str = "relearn-mm-text-holdout"; + +/// Slice id for the vision holdout comparison. +pub const VISION_SLICE_ID: &str = "relearn-mm-vision-holdout"; + +/// Slice id for the agentic image-tool comparison. +pub const AGENTIC_SLICE_ID: &str = "relearn-mm-agentic-holdout"; + +/// Agentic trace evidence, including the pixel-shuffle control. +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] +pub struct AgenticEvidence { + /// Traces executed. + pub traces: u32, + /// Mean score with the real image (`0..=1`). + pub score: f64, + /// Mean score with the image pixels shuffled (`0..=1`). + pub shuffled_score: f64, +} + +impl AgenticEvidence { + /// How much the score fell when the image was destroyed. + #[must_use] + pub fn shuffle_drop(&self) -> f64 { + self.score - self.shuffled_score + } + + /// Whether the model demonstrably used the image. + #[must_use] + pub fn uses_the_image(&self) -> bool { + self.traces > 0 && self.shuffle_drop() >= MIN_SHUFFLE_DROP - DEADZONE + } +} + +/// Per-artifact multimodal measurements. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct MmSliceScores { + /// Relearn text holdout with vision modules ignored (gate 1). + pub text_holdout: ExampleSeries, + /// Frozen image holdout, all task families pooled (gate 2). + pub vision_holdout: ExampleSeries, + /// Per-task-family vision series. + pub vision_by_task: BTreeMap, + /// Agentic image-tool traces on the holdout. + pub agentic: AgenticEvidence, + /// Per-example agentic scores, for the paired comparison. + pub agentic_series: ExampleSeries, + /// Public / training-adjacent vision split (informational). + pub vision_public: ExampleSeries, + /// SHA-256 hex of the submitted LM weights. + pub lm_weights_hash: String, + /// What the miner submitted. + pub kind: SubmissionKind, +} + +impl MmSliceScores { + /// Mean of a series, or `None` when empty. + #[must_use] + pub fn mean(series: &ExampleSeries) -> Option { + if series.is_empty() { + return None; + } + let n = series.len() as f64; + Some(series.by_cluster.values().sum::() / n) + } + + /// Weighted vision score across task families. + /// + /// Task families with no items are dropped and the remaining weights are + /// renormalized, so a missing family cannot silently score as zero. + #[must_use] + pub fn weighted_vision(&self, weights: &VisionTaskWeights) -> Option { + let mut acc = 0.0; + let mut total_w = 0.0; + for task in VisionTask::ALL { + let Some(series) = self.vision_by_task.get(&task) else { + continue; + }; + let Some(m) = Self::mean(series) else { + continue; + }; + let w = weights.weight(task); + acc += w * m; + total_w += w; + } + if total_w <= 0.0 { + return None; + } + Some(acc / total_w) + } +} + +/// Gate that blocked promotion (or would have). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GateFail { + /// The submitted LM regressed on the Relearn text holdout. + LmRegression { + /// Size of the drop in bps. + drop_bps: u64, + }, + /// An encoder-only submission changed the LM weights. + LmWeightsChanged, + /// The text holdout comparison could not run. + LmEvidenceMissing, + /// Challenger is not a significant paired win on the vision holdout. + NoVisionWin, + /// Challenger lost or tied the champion on vision. + VisionRegression, + /// A vision task family has no items. + VisionTaskMissing { + /// Family with no items. + task: VisionTask, + }, + /// The agentic traces did not beat the champion. + NoAgenticWin, + /// Shuffling the image pixels barely changed the score. + IgnoresTheImage, + /// The submitted encoder is not permissively licensed. + EncoderLicense, + /// Paired test refused (slice mismatch / too thin). + PairedRefusal, +} + +/// Serializable paired-test summary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PairedSummary { + /// Overlapping examples. + pub n_paired: u64, + /// Examples outside the dead zone. + pub n_decided: u64, + /// Bootstrap LCB win-rate (bps). + pub win_rate_lcb_bps: u64, + /// Challenger displaces champion. + pub displaces: bool, +} + +impl PairedSummary { + fn from_outcome(o: &PairedOutcome) -> Self { + Self { + n_paired: u64::try_from(o.n_paired).unwrap_or(u64::MAX), + n_decided: u64::try_from(o.n_decided).unwrap_or(u64::MAX), + win_rate_lcb_bps: o.win_rate_lcb_bps, + displaces: o.displaces, + } + } +} + +/// Result of gate 1 on its own. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct LmIntact { + /// Champion text-holdout mean. + pub champion: f64, + /// Challenger text-holdout mean. + pub challenger: f64, + /// `challenger − champion`. + pub delta: f64, + /// Whether the LM cleared the gate. + pub passes: bool, +} + +/// Full promote / reject verdict. Consensus-critical once leaves are signed. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PromoteVerdict { + /// Whether this submission may become champion after operator audit. + pub eligible: bool, + /// Gate 1 report. `None` when the text holdout could not be compared. + pub lm_intact: Option, + /// Vision holdout paired outcome. + pub vision: Option, + /// Agentic holdout paired outcome. + pub agentic: Option, + /// Pixel-shuffle drop on the agentic traces. + pub shuffle_drop: f64, + /// Gates that failed (empty ⇒ all clear). + pub failed: Vec, + /// Lattice score to emit if this hotkey is the live champion (`0` otherwise). + pub lattice: u64, +} + +/// Gate 1: the submitted LM must not regress the Relearn text holdout. +/// +/// `champion_lm_hash` is the champion's LM weights hash. An encoder-only +/// submission must match it exactly; an encoder-plus-LM submission is allowed +/// to differ and is judged on the score alone. +#[must_use] +pub fn check_lm_intact( + champion: &MmSliceScores, + challenger: &MmSliceScores, + champion_lm_hash: &str, +) -> (Option, Vec) { + let mut failed = Vec::new(); + + if challenger.kind.requires_champion_lm_hash() { + let same = !champion_lm_hash.trim().is_empty() + && challenger + .lm_weights_hash + .trim() + .eq_ignore_ascii_case(champion_lm_hash.trim()); + if !same { + failed.push(GateFail::LmWeightsChanged); + } + } + + let (Some(champ_m), Some(chal_m)) = ( + MmSliceScores::mean(&champion.text_holdout), + MmSliceScores::mean(&challenger.text_holdout), + ) else { + failed.push(GateFail::LmEvidenceMissing); + return (None, failed); + }; + + let delta = chal_m - champ_m; + let passes = delta >= -(LM_EPSILON + DEADZONE); + if !passes { + failed.push(GateFail::LmRegression { + drop_bps: ((-delta) * 10_000.0).round().max(0.0) as u64, + }); + } + ( + Some(LmIntact { + champion: champ_m, + challenger: chal_m, + delta, + passes, + }), + failed, + ) +} + +/// Run one paired comparison and push the gates it failed. +/// +/// A refusal is always "champion holds": a slice too thin to decide never +/// becomes a displacement. +fn paired( + metric: &str, + slice_id: &str, + champion: &ExampleSeries, + challenger: &ExampleSeries, + no_win: GateFail, + regression: Option, + failed: &mut Vec, +) -> Option { + let outcome = paired_test(&PairedInput { + metric: metric.to_owned(), + direction: Direction::HigherBetter, + slice_id: slice_id.to_owned(), + champion: champion.clone(), + challenger: challenger.clone(), + }); + match outcome { + Ok(o) if o.displaces => Some(o), + Ok(o) => { + failed.push(no_win); + if let Some(r) = regression { + failed.push(r); + } + Some(o) + } + Err( + PairedRefusal::NotEnoughDecided + | PairedRefusal::NoOverlap + | PairedRefusal::SliceMismatch, + ) => { + failed.push(GateFail::PairedRefusal); + failed.push(no_win); + None + } + } +} + +/// Judge challenger vs champion across both gates. +/// +/// Gate 1 is a hard zero: if the LM regressed, the verdict is ineligible and +/// the lattice is `0` even when every vision number improved. +#[must_use] +pub fn judge_challenger( + champion: &MmSliceScores, + challenger: &MmSliceScores, + champion_lm_hash: &str, + encoder_permissive: bool, +) -> PromoteVerdict { + let (lm_intact, mut failed) = check_lm_intact(champion, challenger, champion_lm_hash); + + if !encoder_permissive { + failed.push(GateFail::EncoderLicense); + } + + let vision = paired( + "relearn_mm.vision_holdout", + VISION_SLICE_ID, + &champion.vision_holdout, + &challenger.vision_holdout, + GateFail::NoVisionWin, + Some(GateFail::VisionRegression), + &mut failed, + ); + + for task in VisionTask::ALL { + let missing = challenger + .vision_by_task + .get(&task) + .is_none_or(ExampleSeries::is_empty); + if missing { + failed.push(GateFail::VisionTaskMissing { task }); + } + } + + let agentic = paired( + "relearn_mm.agentic_holdout", + AGENTIC_SLICE_ID, + &champion.agentic_series, + &challenger.agentic_series, + GateFail::NoAgenticWin, + None, + &mut failed, + ); + + if !challenger.agentic.uses_the_image() { + failed.push(GateFail::IgnoresTheImage); + } + + failed.sort_by(|a, b| format!("{a:?}").cmp(&format!("{b:?}"))); + failed.dedup(); + + let eligible = failed.is_empty(); + let lattice = if eligible { + vision + .as_ref() + .map_or(0, |o| lattice_from_win_rate(o.win_rate_lcb_bps)) + } else { + 0 + }; + + PromoteVerdict { + eligible, + lm_intact, + vision: vision.as_ref().map(PairedSummary::from_outcome), + agentic: agentic.as_ref().map(PairedSummary::from_outcome), + shuffle_drop: challenger.agentic.shuffle_drop(), + failed, + lattice, + } +} + +/// Map bootstrap LCB win-rate (bps) onto the lattice. Champion-hold → 0. +#[must_use] +pub fn lattice_from_win_rate(win_rate_lcb_bps: u64) -> u64 { + let clamped = win_rate_lcb_bps.min(10_000); + u64::from(u32::try_from((u128::from(SCORE_MAX) * u128::from(clamped)) / 10_000).unwrap_or(0)) +} + +/// Champion row always keeps a positive lattice so emission does not burn +/// solely because a challenger was rejected. +#[must_use] +pub fn champion_hold_lattice() -> u64 { + SCORE_MAX / 2 +} + +#[cfg(test)] +mod tests { + use super::*; + + const CHAMP_HASH: &str = "aaaa1111"; + + fn series(prefix: &str, n: usize, val: f64) -> ExampleSeries { + ExampleSeries::from_pairs((0..n).map(|i| (format!("{prefix}{i}"), val))) + } + + fn by_task(val: f64) -> BTreeMap { + VisionTask::ALL + .into_iter() + .map(|t| (t, series(t.as_str(), 40, val))) + .collect() + } + + fn slice(text: f64, vision: f64, agentic: f64, kind: SubmissionKind) -> MmSliceScores { + MmSliceScores { + text_holdout: series("t", 120, text), + vision_holdout: series("v", 160, vision), + vision_by_task: by_task(vision), + agentic: AgenticEvidence { + traces: 32, + score: agentic, + shuffled_score: agentic - 0.30, + }, + agentic_series: series("a", 120, agentic), + vision_public: series("vp", 120, vision), + lm_weights_hash: CHAMP_HASH.into(), + kind, + } + } + + fn champ() -> MmSliceScores { + slice(0.70, 0.50, 0.50, SubmissionKind::EncoderOnly) + } + + fn judge(champion: &MmSliceScores, challenger: &MmSliceScores) -> PromoteVerdict { + judge_challenger(champion, challenger, CHAMP_HASH, true) + } + + #[test] + fn vision_win_with_intact_text_is_eligible() { + let v = judge( + &champ(), + &slice(0.70, 0.85, 0.85, SubmissionKind::EncoderOnly), + ); + assert!(v.eligible, "failed={:?}", v.failed); + assert!(v.lattice > 0); + assert!(v.lm_intact.expect("lm report").passes); + assert!(v.vision.expect("vision").displaces); + assert!(v.agentic.expect("agentic").displaces); + } + + #[test] + fn lm_regression_is_a_hard_zero_even_with_a_huge_vision_win() { + // Vision and agentic both improve a lot; the text side drops. + let chal = slice(0.40, 0.95, 0.95, SubmissionKind::EncoderAndLm); + let v = judge(&champ(), &chal); + assert!(!v.eligible, "LM regression must never be promotable"); + assert_eq!(v.lattice, 0, "lattice must be zero, not reduced"); + let hit = v + .failed + .iter() + .any(|f| matches!(f, GateFail::LmRegression { .. })); + assert!(hit, "expected LmRegression, got {:?}", v.failed); + // The vision side genuinely won; only gate 1 blocked it. + assert!(v.vision.expect("vision").displaces); + assert!(!v.lm_intact.expect("lm report").passes); + } + + #[test] + fn lm_noise_inside_epsilon_is_tolerated() { + let chal = slice(0.695, 0.85, 0.85, SubmissionKind::EncoderAndLm); + let v = judge(&champ(), &chal); + assert!(v.eligible, "failed={:?}", v.failed); + } + + #[test] + fn encoder_only_submission_must_keep_the_champion_lm_weights() { + let mut chal = slice(0.70, 0.85, 0.85, SubmissionKind::EncoderOnly); + chal.lm_weights_hash = "bbbb2222".into(); + let v = judge(&champ(), &chal); + assert!(v.failed.contains(&GateFail::LmWeightsChanged)); + assert!(!v.eligible); + + // The same weights hash with kind EncoderAndLm is judged on score only. + let mut both = chal.clone(); + both.kind = SubmissionKind::EncoderAndLm; + assert!(judge(&champ(), &both).eligible); + } + + #[test] + fn shuffled_pixels_must_hurt_or_the_model_is_not_looking() { + let mut chal = slice(0.70, 0.85, 0.85, SubmissionKind::EncoderOnly); + // Same score with the image destroyed: a text-only heuristic. + chal.agentic.shuffled_score = chal.agentic.score; + let v = judge(&champ(), &chal); + assert!(v.failed.contains(&GateFail::IgnoresTheImage)); + assert!(!v.eligible); + assert!(v.shuffle_drop.abs() < 1e-9); + } + + #[test] + fn a_small_shuffle_drop_still_counts_as_ignoring_the_image() { + let mut chal = slice(0.70, 0.85, 0.85, SubmissionKind::EncoderOnly); + chal.agentic.shuffled_score = chal.agentic.score - 0.02; + assert!(judge(&champ(), &chal) + .failed + .contains(&GateFail::IgnoresTheImage)); + } + + #[test] + fn zero_traces_cannot_satisfy_the_shuffle_control() { + let mut chal = slice(0.70, 0.85, 0.85, SubmissionKind::EncoderOnly); + chal.agentic.traces = 0; + assert!(judge(&champ(), &chal) + .failed + .contains(&GateFail::IgnoresTheImage)); + } + + #[test] + fn vision_regression_blocks() { + let v = judge( + &champ(), + &slice(0.70, 0.30, 0.30, SubmissionKind::EncoderOnly), + ); + assert!(v.failed.contains(&GateFail::VisionRegression)); + assert!(!v.eligible); + assert_eq!(v.lattice, 0); + } + + #[test] + fn missing_vision_task_family_blocks() { + let mut chal = slice(0.70, 0.85, 0.85, SubmissionKind::EncoderOnly); + chal.vision_by_task.remove(&VisionTask::Ocr); + let v = judge(&champ(), &chal); + assert!(v.failed.contains(&GateFail::VisionTaskMissing { + task: VisionTask::Ocr + })); + assert!(!v.eligible); + } + + #[test] + fn non_permissive_encoder_blocks() { + let chal = slice(0.70, 0.85, 0.85, SubmissionKind::EncoderOnly); + let v = judge_challenger(&champ(), &chal, CHAMP_HASH, false); + assert!(v.failed.contains(&GateFail::EncoderLicense)); + assert!(!v.eligible); + } + + #[test] + fn missing_text_evidence_blocks_rather_than_passing() { + let mut chal = slice(0.70, 0.85, 0.85, SubmissionKind::EncoderOnly); + chal.text_holdout = ExampleSeries::default(); + let v = judge(&champ(), &chal); + assert!(v.failed.contains(&GateFail::LmEvidenceMissing)); + assert!(v.lm_intact.is_none()); + assert!(!v.eligible); + } + + #[test] + fn thin_slices_refuse_rather_than_promote() { + let mut chal = slice(0.70, 0.85, 0.85, SubmissionKind::EncoderOnly); + chal.vision_holdout = series("v", 8, 0.85); + chal.agentic_series = series("a", 8, 0.85); + let mut c = champ(); + c.vision_holdout = series("v", 8, 0.50); + c.agentic_series = series("a", 8, 0.50); + let v = judge(&c, &chal); + assert!(v.failed.contains(&GateFail::PairedRefusal)); + assert!(!v.eligible); + } + + #[test] + fn weighted_vision_renormalizes_missing_families() { + let mut s = slice(0.70, 0.80, 0.80, SubmissionKind::EncoderOnly); + let w = VisionTaskWeights::default(); + assert!((s.weighted_vision(&w).expect("all tasks") - 0.80).abs() < 1e-9); + s.vision_by_task + .insert(VisionTask::Ocr, series("ocr", 40, 0.40)); + let got = s.weighted_vision(&w).expect("weighted"); + assert!((got - (0.80 * 3.0 + 0.40) / 4.0).abs() < 1e-9, "{got}"); + s.vision_by_task.clear(); + assert!(s.weighted_vision(&w).is_none()); + } + + #[test] + fn lattice_endpoints() { + assert_eq!(lattice_from_win_rate(0), 0); + assert_eq!(lattice_from_win_rate(10_000), SCORE_MAX); + assert_eq!(champion_hold_lattice(), SCORE_MAX / 2); + } +} diff --git a/crates/relearn-mm-store/Cargo.toml b/crates/relearn-mm-store/Cargo.toml new file mode 100644 index 000000000..ffd046f9d --- /dev/null +++ b/crates/relearn-mm-store/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "relearn-mm-store" +description = "In-memory Relearn Multimodal submissions, encoder manifests, champion" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +hex = "0.4" +relearn-mm-score = { path = "../relearn-mm-score" } +relearn-mm-task = { path = "../relearn-mm-task" } +serde = { version = "1", features = ["derive"] } +sha2 = "0.10" +thiserror = "2" + +[dev-dependencies] +prism-competition = { path = "../prism-competition" } + +[lints] +workspace = true diff --git a/crates/relearn-mm-store/src/lib.rs b/crates/relearn-mm-store/src/lib.rs new file mode 100644 index 000000000..f07789867 --- /dev/null +++ b/crates/relearn-mm-store/src/lib.rs @@ -0,0 +1,477 @@ +//! In-memory Relearn Multimodal store: submissions, encoder manifests, champion. +//! +//! The champion carries two things a challenger is measured against: its text +//! holdout series (gate 1) and its LM weights hash. An encoder-only challenger +//! has to match that hash, so the store is where "same LM, new eyes" is +//! actually enforced. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use relearn_mm_score::{MmSliceScores, PromoteVerdict}; +use relearn_mm_task::SubmissionKind; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +/// Submission lifecycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SubmissionState { + /// Digest accepted; holdout still sealed. + Accepted, + /// Digest frozen; eval running. + Evaluating, + /// Eval finished; waiting operator audit. + AwaitingAdmin, + /// Rejected (LM regression / vision gates / license / integrity). + Rejected, + /// Operator-promoted champion. + Champion, +} + +/// What a miner declares about the multimodal artifact they submitted. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct EncoderManifest { + /// Encoder repo id (miner may bring a different permissive encoder). + pub encoder_model: String, + /// Encoder license. Must be OSI-permissive. + pub encoder_license: String, + /// Projector architecture description (documentation only). + pub projector: String, + /// Encoder-only or encoder plus an LM adapter. + pub kind: SubmissionKind, + /// SHA-256 hex of the submitted LM weights. + /// + /// For [`SubmissionKind::EncoderOnly`] this must equal the champion's hash. + pub lm_weights_hash: String, +} + +/// One miner submission. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Submission { + /// Stable id (`mm_` + 16 hex). + pub id: String, + /// 64-hex miner hotkey. + pub miner_hotkey: String, + /// SHA-256 hex of the miner artifact. Frozen at accept. + pub artifact_digest: String, + /// Optional locator (HF repo, object URL). + pub artifact_uri: Option, + /// Declared encoder, license, kind, and LM hash. + pub manifest: EncoderManifest, + /// Digest freeze nonce (hex). + pub nonce: String, + /// `sha256(hotkey || 0xff || artifact || 0xff || nonce)`. + pub submission_digest: String, + /// Lifecycle. + pub state: SubmissionState, + /// Eval receipt JSON (if any). + pub receipt_json: Option, + /// Judge verdict (if any). + pub verdict: Option, + /// Reject / gate reason. + pub detail: Option, +} + +/// Store errors. +#[derive(Debug, Error)] +pub enum StoreError { + /// Lock poisoned. + #[error("store lock poisoned")] + Poison, + /// Unknown submission. + #[error("unknown submission {0}")] + NotFound(String), + /// Illegal state transition. + #[error("illegal state {0}")] + Illegal(String), +} + +/// In-memory store (v0). +#[derive(Clone, Default)] +pub struct MemoryStore { + inner: Arc>, +} + +#[derive(Default)] +struct Inner { + next: u64, + submissions: BTreeMap, + champion_id: Option, + scores: BTreeMap, + champion_scores: Option, + base_champion: Option, + champion_lm_hash: String, +} + +impl MemoryStore { + /// Empty store. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + fn lock(&self) -> Result, StoreError> { + self.inner.lock().map_err(|_| StoreError::Poison) + } + + /// Record the champion's LM weights hash (gate 1's reference). + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn set_champion_lm_hash(&self, hash: &str) -> Result<(), StoreError> { + self.lock()?.champion_lm_hash = hash.trim().to_ascii_lowercase(); + Ok(()) + } + + /// Champion LM weights hash, or empty when unset. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn champion_lm_hash(&self) -> Result { + Ok(self.lock()?.champion_lm_hash.clone()) + } + + /// Insert a newly accepted submission. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn insert(&self, mut row: Submission) -> Result { + let mut g = self.lock()?; + if row.id.is_empty() { + let n = g.next; + g.next = g.next.saturating_add(1); + row.id = format!("mm_{n:016x}"); + } + g.submissions.insert(row.id.clone(), row.clone()); + Ok(row) + } + + /// Fetch one row. + /// + /// # Errors + /// + /// [`StoreError::NotFound`]. + pub fn get(&self, id: &str) -> Result { + let g = self.lock()?; + g.submissions + .get(id) + .cloned() + .ok_or_else(|| StoreError::NotFound(id.to_owned())) + } + + /// List newest-first. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn list(&self) -> Result, StoreError> { + let g = self.lock()?; + let mut rows: Vec<_> = g.submissions.values().cloned().collect(); + rows.sort_by(|a, b| b.id.cmp(&a.id)); + Ok(rows) + } + + /// Patch state / verdict / receipt. + /// + /// # Errors + /// + /// [`StoreError::NotFound`]. + pub fn patch( + &self, + id: &str, + state: Option, + receipt_json: Option, + verdict: Option, + detail: Option, + ) -> Result { + let mut g = self.lock()?; + let row = g + .submissions + .get_mut(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + if let Some(s) = state { + row.state = s; + } + if let Some(r) = receipt_json { + row.receipt_json = Some(r); + } + if let Some(v) = verdict { + row.verdict = Some(v); + } + if let Some(d) = detail { + row.detail = Some(d); + } + Ok(row.clone()) + } + + /// Current champion submission id. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn champion_id(&self) -> Result, StoreError> { + Ok(self.lock()?.champion_id.clone()) + } + + /// Promote `id`, demote the previous champion, and adopt its LM hash. + /// + /// # Errors + /// + /// [`StoreError::Illegal`] unless the row is `awaiting_admin` with an + /// eligible verdict; [`StoreError::NotFound`] for an unknown id. + pub fn promote(&self, id: &str) -> Result { + let mut g = self.lock()?; + let prev = g.champion_id.clone(); + let new_lm_hash = { + let row = g + .submissions + .get(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + if row.state != SubmissionState::AwaitingAdmin { + return Err(StoreError::Illegal(format!( + "promote requires awaiting_admin, got {:?}", + row.state + ))); + } + if !row.verdict.as_ref().is_some_and(|v| v.eligible) { + return Err(StoreError::Illegal( + "promote refused: verdict not eligible (LM regression or vision gates)".into(), + )); + } + row.manifest.lm_weights_hash.trim().to_ascii_lowercase() + }; + if let Some(p) = prev { + if let Some(old) = g.submissions.get_mut(&p) { + if old.state == SubmissionState::Champion { + old.state = SubmissionState::Rejected; + old.detail = Some("superseded".into()); + } + } + } + { + let row = g + .submissions + .get_mut(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + row.state = SubmissionState::Champion; + } + if let Some(s) = g.scores.get(id).cloned() { + g.champion_scores = Some(s); + } + // An encoder-and-LM promote moves the reference the next challenger's + // gate 1 is measured against; leaving the old hash would let the next + // encoder-only submission ship a stale language model. + if !new_lm_hash.is_empty() { + g.champion_lm_hash = new_lm_hash; + } + g.champion_id = Some(id.to_owned()); + g.submissions + .get(id) + .cloned() + .ok_or_else(|| StoreError::NotFound(id.to_owned())) + } + + /// Persist challenger slices so a later promote displaces vs this run. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn record_scores(&self, id: &str, scores: MmSliceScores) -> Result<(), StoreError> { + self.lock()?.scores.insert(id.to_owned(), scores); + Ok(()) + } + + /// Seed / replace the implicit baseline champion scores. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn set_base_champion(&self, scores: MmSliceScores) -> Result<(), StoreError> { + self.lock()?.base_champion = Some(scores); + Ok(()) + } + + /// Champion slice scores (promoted miner, else the pinned baseline). + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn champion_scores(&self) -> Result, StoreError> { + let g = self.lock()?; + if let Some(s) = &g.champion_scores { + return Ok(Some(s.clone())); + } + Ok(g.base_champion.clone()) + } +} + +/// SHA-256 hex of the frozen submission. +#[must_use] +pub fn freeze_submission_digest(hotkey: &str, artifact_digest: &str, nonce: &str) -> String { + let mut h = Sha256::new(); + h.update(hotkey.as_bytes()); + h.update([0xff]); + h.update(artifact_digest.as_bytes()); + h.update([0xff]); + h.update(nonce.as_bytes()); + hex::encode(h.finalize()) +} + +#[cfg(test)] +mod tests { + use prism_competition::ExampleSeries; + + use super::*; + + const CHAMP_HASH: &str = "aaaa1111"; + + fn scores(text: f64) -> MmSliceScores { + MmSliceScores { + text_holdout: ExampleSeries::from_pairs((0..8).map(|i| (format!("t{i}"), text))), + lm_weights_hash: CHAMP_HASH.into(), + ..MmSliceScores::default() + } + } + + fn verdict(eligible: bool) -> PromoteVerdict { + PromoteVerdict { + eligible, + lm_intact: None, + vision: None, + agentic: None, + shuffle_drop: 0.3, + failed: Vec::new(), + lattice: if eligible { 42 } else { 0 }, + } + } + + fn row(kind: SubmissionKind, lm_hash: &str, v: Option) -> Submission { + Submission { + id: String::new(), + miner_hotkey: "00".repeat(32), + artifact_digest: "11".repeat(32), + artifact_uri: None, + manifest: EncoderManifest { + encoder_model: "google/siglip2-so400m-patch14-384".into(), + encoder_license: "apache-2.0".into(), + projector: "2-layer MLP".into(), + kind, + lm_weights_hash: lm_hash.into(), + }, + nonce: "aa".into(), + submission_digest: "bb".repeat(32), + state: SubmissionState::AwaitingAdmin, + receipt_json: None, + verdict: v, + detail: None, + } + } + + #[test] + fn digest_stable_and_distinct() { + let a = freeze_submission_digest("aa", "bb", "n1"); + assert_eq!(a, freeze_submission_digest("aa", "bb", "n1")); + assert_ne!(a, freeze_submission_digest("aa", "bb", "n2")); + } + + #[test] + fn manifest_defaults_to_the_strict_kind() { + assert_eq!( + EncoderManifest::default().kind, + SubmissionKind::EncoderOnly, + "an unstated kind must not skip the LM hash check" + ); + } + + #[test] + fn promote_refuses_ineligible() { + let st = MemoryStore::new(); + let r = st + .insert(row(SubmissionKind::EncoderOnly, CHAMP_HASH, None)) + .expect("insert"); + assert!(st.promote(&r.id).is_err()); + let r2 = st + .insert(row( + SubmissionKind::EncoderOnly, + CHAMP_HASH, + Some(verdict(false)), + )) + .expect("insert"); + assert!(st.promote(&r2.id).is_err()); + } + + #[test] + fn promote_moves_the_champion_lm_hash_reference() { + let st = MemoryStore::new(); + st.set_champion_lm_hash(CHAMP_HASH).expect("seed hash"); + assert_eq!(st.champion_lm_hash().expect("read"), CHAMP_HASH); + + let r = st + .insert(row( + SubmissionKind::EncoderAndLm, + "CCCC3333", + Some(verdict(true)), + )) + .expect("insert"); + st.record_scores(&r.id, scores(0.9)).expect("scores"); + st.promote(&r.id).expect("promote"); + assert_eq!( + st.champion_lm_hash().expect("read"), + "cccc3333", + "the next encoder-only submission must be measured against the new LM" + ); + } + + #[test] + fn champion_scores_follow_promote_not_base() { + let st = MemoryStore::new(); + st.set_base_champion(scores(0.4)).expect("base"); + let r = st + .insert(row( + SubmissionKind::EncoderOnly, + CHAMP_HASH, + Some(verdict(true)), + )) + .expect("insert"); + st.record_scores(&r.id, scores(0.8)).expect("scores"); + st.promote(&r.id).expect("promote"); + let got = st.champion_scores().expect("read").expect("some"); + assert!((MmSliceScores::mean(&got.text_holdout).unwrap_or(0.0) - 0.8).abs() < 1e-9); + } + + #[test] + fn promoting_a_second_champion_demotes_the_first() { + let st = MemoryStore::new(); + let a = st + .insert(row( + SubmissionKind::EncoderOnly, + CHAMP_HASH, + Some(verdict(true)), + )) + .expect("insert a"); + st.promote(&a.id).expect("promote a"); + let b = st + .insert(row( + SubmissionKind::EncoderOnly, + CHAMP_HASH, + Some(verdict(true)), + )) + .expect("insert b"); + st.promote(&b.id).expect("promote b"); + assert_eq!(st.get(&a.id).expect("a").state, SubmissionState::Rejected); + assert_eq!(st.champion_id().expect("id"), Some(b.id)); + } +} diff --git a/crates/relearn-mm-task/Cargo.toml b/crates/relearn-mm-task/Cargo.toml new file mode 100644 index 000000000..9b58f9bd8 --- /dev/null +++ b/crates/relearn-mm-task/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "relearn-mm-task" +description = "Relearn Multimodal challenge identity, permissive encoder pins, pin parsing" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +relearn-challenge-task = { path = "../relearn-challenge-task" } +serde = { version = "1", features = ["derive"] } +thiserror = "2" +toml = "0.8" + +[lints] +workspace = true diff --git a/crates/relearn-mm-task/src/lib.rs b/crates/relearn-mm-task/src/lib.rs new file mode 100644 index 000000000..297cadc5e --- /dev/null +++ b/crates/relearn-mm-task/src/lib.rs @@ -0,0 +1,237 @@ +//! Relearn Multimodal challenge identity and verified encoder pins. +//! +//! ```text +//! challenge_id = "relearn-mm" +//! scoring_version = 1 +//! task_id domain = b"base-relearn-mm-task-id-v1" +//! receipt domain = b"base-relearn-mm-receipt-v1" +//! ``` +//! +//! Miners attach a permissively licensed vision encoder plus a projector to the +//! champion Relearn LLM. They are paid to make the model see, and they are not +//! paid to break it: the text side of the champion is a hard gate, so a vision +//! win that costs language ability scores zero rather than partial credit. +//! +//! Encoder licenses are restricted to OSI-permissive terms +//! ([`license_is_permissive`]) because the artifact has to be redistributable. + +#![forbid(unsafe_code)] +#![allow(clippy::doc_markdown, clippy::module_name_repetitions)] + +mod pin; + +pub use pin::{ + PinError, RelearnMmPin, SubmissionKind, VisionTaskWeights, MIN_AGENTIC_TRACES, MIN_TEXT_ITEMS, + MIN_VISION_ITEMS_PER_TASK, +}; + +/// Normative challenge id (trust-root / leaf `challenge_id` string). +pub const CHALLENGE_ID: &str = "relearn-mm"; + +/// UTF-8 bytes of [`CHALLENGE_ID`]. +pub const CHALLENGE_ID_BYTES: &[u8] = b"relearn-mm"; + +/// Live `challenge_scoring_version`. +pub const SCORING_VERSION: u16 = 1; + +/// Domain tag for task id digests. +pub const TASK_ID_DOMAIN: &[u8] = b"base-relearn-mm-task-id-v1"; + +/// Domain tag for holdout slice ids. +pub const HOLDOUT_DOMAIN: &[u8] = b"base-relearn-mm-holdout-v1"; + +/// Domain tag for eval-receipt digests. +pub const RECEIPT_DOMAIN: &[u8] = b"base-relearn-mm-receipt-v1"; + +/// Domain tag for promotion attestations. +pub const PROMOTE_DOMAIN: &[u8] = b"base-relearn-mm-promote-v1"; + +/// Integer score lattice max (same scale as other challenges). +pub const SCORE_MAX: u64 = 1_000_000; + +/// Language model the encoder attaches to: the Relearn champion's base. +pub const LM_BASE_MODEL_ID: &str = relearn_challenge_task::BASE_MODEL_ID; + +/// Pinned vision encoder. +/// +/// Verified 2026-08-30 against : +/// Apache-2.0, SigLIP 2 So400m at 384px, intended for use as a VLM vision tower. +pub const ENCODER_MODEL_ID: &str = "google/siglip2-so400m-patch14-384"; + +/// License of [`ENCODER_MODEL_ID`]. +pub const ENCODER_LICENSE: &str = "apache-2.0"; + +/// Alternate encoders whose cards were verified Apache-2.0 on 2026-08-30. +/// +/// An operator may repin to one of these without a code change. Every entry was +/// checked on the model card, not inferred from the family name. +pub const VERIFIED_ENCODER_ALTERNATES: &[(&str, &str)] = &[ + ("google/siglip2-so400m-patch14-384", "apache-2.0"), + ("google/siglip-so400m-patch14-384", "apache-2.0"), + // The vision tower inside Idefics2 is SigLIP; the card is Apache-2.0. + ("HuggingFaceM4/idefics2-8b", "apache-2.0"), +]; + +/// Licenses a submitted encoder may carry. +/// +/// OSI-permissive only. OpenRAIL and other use-restricted terms are refused +/// here even though the T2I challenge accepts OpenMDW for its generator: that +/// base is a single operator-chosen pin, whereas the encoder is miner-supplied +/// and has to stay redistributable without per-use conditions. +pub const PERMISSIVE_LICENSES: &[&str] = + &["apache-2.0", "mit", "bsd-2-clause", "bsd-3-clause", "isc"]; + +/// Normalize a license string for comparison (`Apache 2.0` → `apache-2.0`). +#[must_use] +pub fn normalize_license(raw: &str) -> String { + let lower = raw.trim().to_ascii_lowercase(); + let collapsed: String = lower + .chars() + .map(|c| if c == ' ' || c == '_' { '-' } else { c }) + .collect(); + match collapsed.as_str() { + "apache-2" | "apache2" | "apache-license-2.0" => "apache-2.0".to_owned(), + "bsd-3" => "bsd-3-clause".to_owned(), + "bsd-2" => "bsd-2-clause".to_owned(), + other => other.to_owned(), + } +} + +/// True when `license` is OSI-permissive enough for a submitted encoder. +#[must_use] +pub fn license_is_permissive(license: &str) -> bool { + let norm = normalize_license(license); + PERMISSIVE_LICENSES.contains(&norm.as_str()) +} + +/// Frozen vision holdout task families. +/// +/// Deliberately not ImageNet or COCO test splits: both are in the pretraining +/// mix of every candidate encoder, so a score on them measures memorization. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, +)] +#[serde(rename_all = "snake_case")] +pub enum VisionTask { + /// Free-form captioning scored against reference captions. + Captioning, + /// Visual question answering. + Vqa, + /// Reading text rendered inside the image. + Ocr, + /// Spatial relations between objects. + SpatialRelations, +} + +impl VisionTask { + /// All frozen task families. + pub const ALL: [Self; 4] = [ + Self::Captioning, + Self::Vqa, + Self::Ocr, + Self::SpatialRelations, + ]; + + /// Wire name. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Captioning => "captioning", + Self::Vqa => "vqa", + Self::Ocr => "ocr", + Self::SpatialRelations => "spatial_relations", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn challenge_id_is_distinct() { + assert_eq!(CHALLENGE_ID, "relearn-mm"); + for other in ["relearn", "relearn-t2i", "bounty", "prism", "design"] { + assert_ne!(CHALLENGE_ID, other); + } + } + + #[test] + fn domain_tags_are_mm_prefixed_and_unique() { + let tags = [ + TASK_ID_DOMAIN, + HOLDOUT_DOMAIN, + RECEIPT_DOMAIN, + PROMOTE_DOMAIN, + ]; + for t in tags { + assert!(std::str::from_utf8(t).unwrap_or("").contains("relearn-mm")); + } + for i in 0..tags.len() { + for j in (i + 1)..tags.len() { + assert_ne!(tags[i], tags[j]); + } + } + assert_ne!(TASK_ID_DOMAIN, relearn_challenge_task::TASK_ID_DOMAIN); + } + + #[test] + fn encoder_pin_is_verified_apache_siglip2() { + assert_eq!(ENCODER_MODEL_ID, "google/siglip2-so400m-patch14-384"); + assert_eq!(ENCODER_LICENSE, "apache-2.0"); + assert!(license_is_permissive(ENCODER_LICENSE)); + } + + #[test] + fn lm_side_tracks_the_relearn_champion_base() { + assert_eq!(LM_BASE_MODEL_ID, "Qwen/Qwen3.8-Flash-Next"); + } + + #[test] + fn every_verified_alternate_is_permissive() { + assert!(VERIFIED_ENCODER_ALTERNATES + .iter() + .any(|(id, _)| *id == ENCODER_MODEL_ID)); + for (id, license) in VERIFIED_ENCODER_ALTERNATES { + assert!(license_is_permissive(license), "{id} carries {license}"); + } + } + + #[test] + fn license_normalization_accepts_common_spellings() { + for ok in ["Apache-2.0", "apache 2.0", "APACHE_2.0", "MIT", " mit "] { + assert!(license_is_permissive(ok), "{ok} should be permissive"); + } + } + + #[test] + fn use_restricted_licenses_are_refused() { + for bad in [ + "openrail", + "openrail-m", + "creativeml-openrail-m", + "cc-by-nc-4.0", + "cc-by-nc-sa-4.0", + "llama3.1", + "gemma", + "other", + "", + ] { + assert!(!license_is_permissive(bad), "{bad} must be refused"); + } + } + + #[test] + fn openmdw_is_not_permissive_enough_for_a_miner_encoder() { + // The T2I generator base is OpenMDW by operator choice; a miner-supplied + // encoder must be OSI-permissive. + assert!(!license_is_permissive("OpenMDW-1.1")); + } + + #[test] + fn vision_tasks_have_stable_wire_names() { + assert_eq!(VisionTask::ALL.len(), 4); + assert_eq!(VisionTask::Ocr.as_str(), "ocr"); + assert_eq!(VisionTask::SpatialRelations.as_str(), "spatial_relations"); + } +} diff --git a/crates/relearn-mm-task/src/pin.rs b/crates/relearn-mm-task/src/pin.rs new file mode 100644 index 000000000..6e79a65bc --- /dev/null +++ b/crates/relearn-mm-task/src/pin.rs @@ -0,0 +1,380 @@ +//! `config/relearn-mm-pin.toml`: encoder pin, holdout sizes, gate tolerances. + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::{ + license_is_permissive, normalize_license, VisionTask, CHALLENGE_ID, ENCODER_LICENSE, + ENCODER_MODEL_ID, LM_BASE_MODEL_ID, SCORING_VERSION, +}; + +/// What a miner submitted. +/// +/// Defaults to the stricter [`SubmissionKind::EncoderOnly`]: an unstated kind +/// must not be the one that skips the LM weights-hash check. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SubmissionKind { + /// Encoder (and projector) only; the LM must be the champion, bit for bit. + #[default] + EncoderOnly, + /// Encoder plus an LM adapter, so both gates apply to new weights. + EncoderAndLm, +} + +impl SubmissionKind { + /// Whether the LM weights must hash-match the champion. + /// + /// An encoder-only submission is measured as `champion LM + new encoder`, + /// so the LM hash is the proof that nothing on the text side moved. + #[must_use] + pub const fn requires_champion_lm_hash(self) -> bool { + matches!(self, Self::EncoderOnly) + } +} + +/// Per-task weights for the vision holdout score. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct VisionTaskWeights { + /// Captioning weight. + pub captioning: f64, + /// VQA weight. + pub vqa: f64, + /// OCR / text-in-image weight. + pub ocr: f64, + /// Spatial-relations weight. + pub spatial_relations: f64, +} + +impl Default for VisionTaskWeights { + fn default() -> Self { + Self { + captioning: 0.25, + vqa: 0.25, + ocr: 0.25, + spatial_relations: 0.25, + } + } +} + +impl VisionTaskWeights { + /// Weight for one task family. + #[must_use] + pub const fn weight(&self, task: VisionTask) -> f64 { + match task { + VisionTask::Captioning => self.captioning, + VisionTask::Vqa => self.vqa, + VisionTask::Ocr => self.ocr, + VisionTask::SpatialRelations => self.spatial_relations, + } + } + + /// Sum of every weight. + #[must_use] + pub fn total(&self) -> f64 { + VisionTask::ALL.into_iter().map(|t| self.weight(t)).sum() + } +} + +/// Everything Cortex needs to reproduce a Relearn Multimodal eval. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct RelearnMmPin { + /// Must be [`CHALLENGE_ID`]. + pub challenge_id: String, + /// Must be [`SCORING_VERSION`]. + pub scoring_version: u16, + /// Language model the encoder attaches to. + pub lm_base_model: String, + /// Pinned vision encoder. + pub encoder_model: String, + /// License of `encoder_model`. Must be OSI-permissive. + pub encoder_license: String, + /// Pinned Hugging Face revision of `encoder_model` (empty until recorded). + pub encoder_revision: String, + /// Eval image reference (no floating tag in prod). + pub eval_image: String, + /// `sha256:…` digest. Empty until the first digest-pinned eval image ships. + pub eval_image_digest: String, + /// Miner-facing repo. + pub relearn_git: String, + /// Vision holdout items per task family. + pub vision_items_per_task: usize, + /// Agentic image-tool traces on the holdout. + pub agentic_traces: usize, + /// Text holdout items reused from the Relearn LLM challenge. + pub text_holdout_items: usize, + /// Per-task weights. + pub vision_weights: VisionTaskWeights, +} + +impl Default for RelearnMmPin { + fn default() -> Self { + Self { + challenge_id: CHALLENGE_ID.into(), + scoring_version: SCORING_VERSION, + lm_base_model: LM_BASE_MODEL_ID.into(), + encoder_model: ENCODER_MODEL_ID.into(), + encoder_license: ENCODER_LICENSE.into(), + encoder_revision: String::new(), + eval_image: "ghcr.io/cortexlm/relearn-mm-eval".into(), + eval_image_digest: String::new(), + relearn_git: relearn_challenge_task::RELEARN_GIT_URL.into(), + vision_items_per_task: 40, + agentic_traces: 120, + text_holdout_items: 120, + vision_weights: VisionTaskWeights::default(), + } + } +} + +/// Why a pin was refused. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum PinError { + /// TOML did not parse. + #[error("parse relearn-mm pin: {0}")] + Parse(String), + /// Pin declares the wrong challenge id or scoring version. + #[error("pin identity mismatch: {0}")] + Identity(String), + /// The encoder license is not OSI-permissive. + #[error("encoder license {0:?} is not OSI-permissive (Apache-2.0 / MIT / BSD / ISC)")] + EncoderLicense(String), + /// The pin names no encoder at all. + #[error("encoder_model must not be empty")] + EmptyEncoder, + /// A holdout split is too thin for a verdict. + #[error("{split} has {items} items, below the {min} floor")] + TooFewItems { + /// Split name. + split: String, + /// Items the pin would produce. + items: usize, + /// Required floor. + min: usize, + }, + /// Vision task weights do not sum to 1. + #[error("vision_weights must sum to 1.0, got {0}")] + BadWeights(String), +} + +/// Minimum text holdout items. Matches the paired test's evidence floor. +pub const MIN_TEXT_ITEMS: usize = 100; + +/// Minimum vision holdout items per task family. +/// +/// Four families pooled, so this floor puts the vision slice at the paired +/// test's 100-example bar. +pub const MIN_VISION_ITEMS_PER_TASK: usize = 25; + +/// Minimum agentic image-tool traces. +/// +/// The agentic comparison runs through the same bootstrap paired test as the +/// other slices, and that test refuses a verdict below 100 decided examples. A +/// pin with fewer traces would make the agentic gate permanently unsatisfiable +/// — every submission would hold the champion for lack of evidence — so the +/// floor is the statistical bar, not a taste call. +pub const MIN_AGENTIC_TRACES: usize = 100; + +impl RelearnMmPin { + /// Parse and validate `config/relearn-mm-pin.toml`. + /// + /// # Errors + /// + /// [`PinError::Parse`] on malformed TOML, otherwise any [`validate`] + /// failure. + /// + /// [`validate`]: Self::validate + pub fn from_toml(body: &str) -> Result { + let pin: Self = toml::from_str(body).map_err(|e| PinError::Parse(e.to_string()))?; + pin.validate()?; + Ok(pin) + } + + /// Enforce every product rule the pin is responsible for. + /// + /// # Errors + /// + /// See [`PinError`]. + pub fn validate(&self) -> Result<(), PinError> { + if self.challenge_id != CHALLENGE_ID { + return Err(PinError::Identity(format!( + "challenge_id must be {CHALLENGE_ID:?}, got {:?}", + self.challenge_id + ))); + } + if self.scoring_version != SCORING_VERSION { + return Err(PinError::Identity(format!( + "scoring_version must be {SCORING_VERSION}, got {}", + self.scoring_version + ))); + } + if self.lm_base_model.trim() != LM_BASE_MODEL_ID { + return Err(PinError::Identity(format!( + "lm_base_model must be {LM_BASE_MODEL_ID:?}, got {:?}", + self.lm_base_model + ))); + } + if self.encoder_model.trim().is_empty() { + return Err(PinError::EmptyEncoder); + } + if !license_is_permissive(&self.encoder_license) { + return Err(PinError::EncoderLicense(self.encoder_license.clone())); + } + check_floor( + "text_holdout_items", + self.text_holdout_items, + MIN_TEXT_ITEMS, + )?; + check_floor( + "vision_items_per_task", + self.vision_items_per_task, + MIN_VISION_ITEMS_PER_TASK, + )?; + check_floor("agentic_traces", self.agentic_traces, MIN_AGENTIC_TRACES)?; + let total = self.vision_weights.total(); + if (total - 1.0).abs() > 1e-6 { + return Err(PinError::BadWeights(format!("{total:.6}"))); + } + Ok(()) + } + + /// True when a live rent is allowed (real digest pin present). + #[must_use] + pub fn can_rent(&self) -> bool { + self.eval_image_digest.starts_with("sha256:") && self.eval_image_digest.len() >= 71 + } + + /// Check a submitted encoder against the license policy. + /// + /// The encoder id itself is not pinned to one repo: a miner may bring a + /// better permissive encoder. Only the license is non-negotiable. + /// + /// # Errors + /// + /// [`PinError::EmptyEncoder`] or [`PinError::EncoderLicense`]. + pub fn attest_encoder(&self, encoder: &str, license: &str) -> Result<(), PinError> { + if encoder.trim().is_empty() { + return Err(PinError::EmptyEncoder); + } + if !license_is_permissive(license) { + return Err(PinError::EncoderLicense(normalize_license(license))); + } + Ok(()) + } +} + +fn check_floor(split: &str, items: usize, min: usize) -> Result<(), PinError> { + if items < min { + return Err(PinError::TooFewItems { + split: split.to_owned(), + items, + min, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const BODY: &str = r#" +challenge_id = "relearn-mm" +scoring_version = 1 +lm_base_model = "Qwen/Qwen3.8-Flash-Next" +encoder_model = "google/siglip2-so400m-patch14-384" +encoder_license = "apache-2.0" +vision_items_per_task = 40 +agentic_traces = 120 +text_holdout_items = 120 +"#; + + #[test] + fn default_pin_is_siglip2_on_the_relearn_base() { + let p = RelearnMmPin::default(); + assert_eq!(p.encoder_model, "google/siglip2-so400m-patch14-384"); + assert_eq!(p.lm_base_model, "Qwen/Qwen3.8-Flash-Next"); + assert!(!p.can_rent()); + p.validate().expect("default validates"); + } + + #[test] + fn valid_pin_parses() { + let p = RelearnMmPin::from_toml(BODY).expect("parses"); + assert_eq!(p.vision_items_per_task, 40); + assert!((p.vision_weights.total() - 1.0).abs() < 1e-9); + } + + #[test] + fn non_permissive_encoder_license_is_refused() { + let body = BODY.replace("apache-2.0", "creativeml-openrail-m"); + let err = RelearnMmPin::from_toml(&body).expect_err("must refuse"); + assert!(matches!(err, PinError::EncoderLicense(_)), "{err:?}"); + } + + #[test] + fn thin_splits_are_refused() { + let thin = [ + RelearnMmPin { + text_holdout_items: 10, + ..RelearnMmPin::default() + }, + RelearnMmPin { + vision_items_per_task: 2, + ..RelearnMmPin::default() + }, + RelearnMmPin { + agentic_traces: 1, + ..RelearnMmPin::default() + }, + ]; + for p in thin { + assert!(matches!(p.validate(), Err(PinError::TooFewItems { .. }))); + } + } + + #[test] + fn weights_must_sum_to_one() { + let p = RelearnMmPin { + vision_weights: VisionTaskWeights { + ocr: 0.9, + ..VisionTaskWeights::default() + }, + ..RelearnMmPin::default() + }; + assert!(matches!(p.validate(), Err(PinError::BadWeights(_)))); + } + + #[test] + fn encoder_attestation_enforces_license_not_repo() { + let p = RelearnMmPin::default(); + p.attest_encoder("google/siglip-so400m-patch14-384", "Apache 2.0") + .expect("a different permissive encoder is allowed"); + p.attest_encoder("some-lab/my-mit-encoder", "MIT") + .expect("MIT is allowed"); + assert!(matches!( + p.attest_encoder("some-lab/nc-encoder", "cc-by-nc-4.0"), + Err(PinError::EncoderLicense(_)) + )); + assert!(matches!( + p.attest_encoder(" ", "apache-2.0"), + Err(PinError::EmptyEncoder) + )); + } + + #[test] + fn encoder_only_submissions_must_prove_the_lm_hash() { + assert!(SubmissionKind::EncoderOnly.requires_champion_lm_hash()); + assert!(!SubmissionKind::EncoderAndLm.requires_champion_lm_hash()); + } + + #[test] + fn weights_are_addressable_per_task() { + let w = VisionTaskWeights::default(); + for t in VisionTask::ALL { + assert!((w.weight(t) - 0.25).abs() < 1e-9); + } + } +} diff --git a/crates/relearn-mm-task/tests/committed_pin.rs b/crates/relearn-mm-task/tests/committed_pin.rs new file mode 100644 index 000000000..63cd9dd08 --- /dev/null +++ b/crates/relearn-mm-task/tests/committed_pin.rs @@ -0,0 +1,60 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] + +//! The committed `config/relearn-mm-pin.toml` must load and satisfy every +//! product rule: the Relearn champion base on the language side, a verified +//! permissive encoder, splits thick enough for a verdict, and no secrets. + +use std::path::{Path, PathBuf}; + +use relearn_mm_task::{ + license_is_permissive, RelearnMmPin, ENCODER_LICENSE, ENCODER_MODEL_ID, LM_BASE_MODEL_ID, + MIN_AGENTIC_TRACES, MIN_TEXT_ITEMS, MIN_VISION_ITEMS_PER_TASK, +}; + +fn pin_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../config/relearn-mm-pin.toml") + .canonicalize() + .expect("pin path") +} + +fn pin() -> RelearnMmPin { + let body = std::fs::read_to_string(pin_path()).expect("read pin"); + RelearnMmPin::from_toml(&body).expect("committed pin must validate") +} + +#[test] +fn committed_pin_validates() { + let p = pin(); + assert_eq!(p.lm_base_model, LM_BASE_MODEL_ID); + assert_eq!(p.encoder_model, ENCODER_MODEL_ID); + assert_eq!(p.encoder_license, ENCODER_LICENSE); + assert!(license_is_permissive(&p.encoder_license)); +} + +#[test] +fn committed_splits_clear_the_evidence_floors() { + let p = pin(); + assert!(p.text_holdout_items >= MIN_TEXT_ITEMS); + assert!(p.vision_items_per_task >= MIN_VISION_ITEMS_PER_TASK); + assert!(p.agentic_traces >= MIN_AGENTIC_TRACES); + assert!((p.vision_weights.total() - 1.0).abs() < 1e-9); +} + +#[test] +fn pin_carries_no_endpoint_or_secret() { + let body = std::fs::read_to_string(pin_path()).expect("read pin"); + let lower = body.to_ascii_lowercase(); + for banned in ["api_key", "bearer", "_token", "mnemonic", "https://api."] { + assert!(!lower.contains(banned), "pin mentions {banned:?}"); + } +} + +#[test] +fn pin_documents_that_imagenet_and_coco_test_are_off_limits() { + let body = std::fs::read_to_string(pin_path()).expect("read pin"); + assert!( + body.contains("ImageNet") && body.contains("COCO"), + "the contamination rationale must stay next to the split sizes" + ); +} diff --git a/crates/relearn-score/Cargo.toml b/crates/relearn-score/Cargo.toml new file mode 100644 index 000000000..928cb62f5 --- /dev/null +++ b/crates/relearn-score/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "relearn-score" +description = "Relearn displacement scoring, overfit gates, never-regression promote" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +prism-competition = { path = "../prism-competition" } +relearn-challenge-task = { path = "../relearn-challenge-task" } +serde = { version = "1", features = ["derive"] } + +[lints] +workspace = true diff --git a/crates/relearn-score/src/lib.rs b/crates/relearn-score/src/lib.rs new file mode 100644 index 000000000..4efae90aa --- /dev/null +++ b/crates/relearn-score/src/lib.rs @@ -0,0 +1,272 @@ +//! Displacement scoring for Relearn. +//! +//! Score is challenger vs the previous champion on a shared holdout slice. +//! A regression is never crowned. Promotion additionally requires the +//! operator-audited paired win plus retention / overfit gates. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::cast_precision_loss, + clippy::cast_possible_truncation +)] + +use prism_competition::{ + paired_test, Direction, ExampleSeries, PairedInput, PairedOutcome, PairedRefusal, DEADZONE, +}; +use relearn_challenge_task::SCORE_MAX; +use serde::{Deserialize, Serialize}; + +/// Maximum allowed public-private accuracy gap (absolute). +pub const MAX_PUBLIC_PRIVATE_GAP: f64 = 0.08; + +/// Maximum allowed drop under input perturbation (absolute). +pub const MAX_PERTURB_DROP: f64 = 0.05; + +/// Minimum canary accuracy (known-answer items the base model already solves). +pub const MIN_CANARY_ACCURACY: f64 = 0.95; + +/// Minimum agent-trace score (first-class; 0..1). +pub const MIN_AGENT_TRACE: f64 = 0.5; + +/// Per-example holdout measurements for one submission. +#[derive(Debug, Clone, PartialEq)] +pub struct SliceScores { + /// Holdout items (scored artifact). Higher is better. + pub holdout: ExampleSeries, + /// Public / training-adjacent canary slice (overfit detector). + pub public: ExampleSeries, + /// Same holdout items after a pinned perturbation. + pub perturbed: ExampleSeries, + /// Known-answer canaries (base-model already-correct items). + pub canaries: ExampleSeries, + /// Agent-trace quality in `[0, 1]` (first-class; not a side channel). + pub agent_trace: f64, +} + +impl SliceScores { + /// Mean of a series, or `None` when empty. + #[must_use] + pub fn mean(series: &ExampleSeries) -> Option { + if series.is_empty() { + return None; + } + let n = series.len() as f64; + Some(series.by_cluster.values().sum::() / n) + } +} + +/// Gate that blocked promotion (or would have). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GateFail { + /// Challenger is not a significant paired win. + NoPairedWin, + /// Challenger lost or tied the champion (never crown a regression). + Regression, + /// Public-private gap too large (memorization / contamination). + PublicPrivateGap, + /// Perturbed holdout collapsed (brittle / overfit). + Perturbation, + /// Canaries failed (catastrophic forgetting of base competence). + Canaries, + /// Agent-trace score below floor. + AgentTrace, + /// Paired test refused (slice mismatch / too thin). + PairedRefusal, +} + +/// Serializable paired-test summary (prism `PairedOutcome` is not serde). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PairedSummary { + /// Overlapping examples. + pub n_paired: u64, + /// Decided examples (outside the dead zone). + pub n_decided: u64, + /// Bootstrap LCB win-rate (bps). + pub win_rate_lcb_bps: u64, + /// Challenger displaces champion. + pub displaces: bool, +} + +impl PairedSummary { + fn from_outcome(o: &PairedOutcome) -> Self { + Self { + n_paired: u64::try_from(o.n_paired).unwrap_or(u64::MAX), + n_decided: u64::try_from(o.n_decided).unwrap_or(u64::MAX), + win_rate_lcb_bps: o.win_rate_lcb_bps, + displaces: o.displaces, + } + } +} + +/// Full promote / reject verdict. Consensus-critical once leaves are signed. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PromoteVerdict { + /// Whether this submission may become champion after operator audit. + pub eligible: bool, + /// Paired-test outcome when the slices lined up. + pub paired: Option, + /// Gates that failed (empty ⇒ all clear). + pub failed: Vec, + /// Lattice score to emit if this hotkey is the live champion (`0` otherwise). + pub lattice: u64, +} + +/// Judge challenger vs champion. Never returns `eligible` on a regression. +#[must_use] +pub fn judge_challenger(champion: &SliceScores, challenger: &SliceScores) -> PromoteVerdict { + let mut failed = Vec::new(); + + let input = PairedInput { + metric: "relearn.holdout".into(), + direction: Direction::HigherBetter, + slice_id: "holdout".into(), + champion: champion.holdout.clone(), + challenger: challenger.holdout.clone(), + }; + let paired_raw = match paired_test(&input) { + Ok(o) => Some(o), + Err( + PairedRefusal::NotEnoughDecided + | PairedRefusal::NoOverlap + | PairedRefusal::SliceMismatch, + ) => { + failed.push(GateFail::PairedRefusal); + None + } + }; + + match paired_raw { + Some(ref o) if o.displaces => {} + Some(_) => { + failed.push(GateFail::NoPairedWin); + failed.push(GateFail::Regression); + } + None => failed.push(GateFail::NoPairedWin), + } + + if let (Some(pub_m), Some(priv_m)) = ( + SliceScores::mean(&challenger.public), + SliceScores::mean(&challenger.holdout), + ) { + if (pub_m - priv_m).abs() > MAX_PUBLIC_PRIVATE_GAP + DEADZONE { + failed.push(GateFail::PublicPrivateGap); + } + } + + if let (Some(h), Some(p)) = ( + SliceScores::mean(&challenger.holdout), + SliceScores::mean(&challenger.perturbed), + ) { + if h - p > MAX_PERTURB_DROP + DEADZONE { + failed.push(GateFail::Perturbation); + } + } + + if let Some(c) = SliceScores::mean(&challenger.canaries) { + if c + DEADZONE < MIN_CANARY_ACCURACY { + failed.push(GateFail::Canaries); + } + } + + if challenger.agent_trace + DEADZONE < MIN_AGENT_TRACE { + failed.push(GateFail::AgentTrace); + } + + failed.sort_by(|a, b| format!("{a:?}").cmp(&format!("{b:?}"))); + failed.dedup(); + + let eligible = failed.is_empty(); + let lattice = if eligible { + paired_raw + .as_ref() + .map_or(0, |o| lattice_from_win_rate(o.win_rate_lcb_bps)) + } else { + 0 + }; + + PromoteVerdict { + eligible, + paired: paired_raw.as_ref().map(PairedSummary::from_outcome), + failed, + lattice, + } +} + +/// Map bootstrap LCB win-rate (bps) onto the lattice. Champion-hold → 0. +#[must_use] +pub fn lattice_from_win_rate(win_rate_lcb_bps: u64) -> u64 { + let clamped = win_rate_lcb_bps.min(10_000); + u64::from(u32::try_from((u128::from(SCORE_MAX) * u128::from(clamped)) / 10_000).unwrap_or(0)) +} + +/// Champion row always keeps a positive lattice so emission does not burn +/// solely because a challenger was rejected. +#[must_use] +pub fn champion_hold_lattice() -> u64 { + SCORE_MAX / 2 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn series(prefix: &str, n: usize, val: f64) -> ExampleSeries { + ExampleSeries::from_pairs((0..n).map(|i| (format!("{prefix}{i}"), val))) + } + + fn slice(hold: f64, public: f64, pert: f64, canary: f64, trace: f64) -> SliceScores { + SliceScores { + holdout: series("h", 120, hold), + public: series("p", 120, public), + perturbed: series("x", 120, pert), + canaries: series("c", 40, canary), + agent_trace: trace, + } + } + + #[test] + fn never_crowns_regression() { + let champ = slice(0.80, 0.80, 0.79, 0.99, 0.9); + let worse = slice(0.40, 0.40, 0.39, 0.99, 0.9); + let v = judge_challenger(&champ, &worse); + assert!(!v.eligible); + assert!(v.failed.contains(&GateFail::Regression)); + assert_eq!(v.lattice, 0); + } + + #[test] + fn significant_win_plus_gates_is_eligible() { + let champ = slice(0.50, 0.50, 0.49, 0.99, 0.9); + let better = slice(0.80, 0.80, 0.79, 0.99, 0.9); + let v = judge_challenger(&champ, &better); + assert!(v.eligible, "expected eligible, failed={:?}", v.failed); + assert!(v.lattice > 0); + } + + #[test] + fn public_private_gap_blocks() { + let champ = slice(0.50, 0.50, 0.49, 0.99, 0.9); + let leak = slice(0.80, 0.99, 0.79, 0.99, 0.9); + let v = judge_challenger(&champ, &leak); + assert!(v.failed.contains(&GateFail::PublicPrivateGap)); + assert!(!v.eligible); + } + + #[test] + fn canary_collapse_blocks() { + let champ = slice(0.50, 0.50, 0.49, 0.99, 0.9); + let forget = slice(0.80, 0.80, 0.79, 0.20, 0.9); + let v = judge_challenger(&champ, &forget); + assert!(v.failed.contains(&GateFail::Canaries)); + assert!(!v.eligible); + } + + #[test] + fn lattice_is_zero_for_zero_bps() { + assert_eq!(lattice_from_win_rate(0), 0); + assert_eq!(lattice_from_win_rate(10_000), SCORE_MAX); + } +} diff --git a/crates/relearn-store/Cargo.toml b/crates/relearn-store/Cargo.toml new file mode 100644 index 000000000..4f3cd9701 --- /dev/null +++ b/crates/relearn-store/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "relearn-store" +description = "In-memory Relearn submissions, champion, sealed holdout" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +hex = "0.4" +relearn-challenge-task = { path = "../relearn-challenge-task" } +relearn-score = { path = "../relearn-score" } +serde = { version = "1", features = ["derive"] } +sha2 = "0.10" +thiserror = "2" + +[dev-dependencies] +prism-competition = { path = "../prism-competition" } + +[lints] +workspace = true diff --git a/crates/relearn-store/src/lib.rs b/crates/relearn-store/src/lib.rs new file mode 100644 index 000000000..207aea1d2 --- /dev/null +++ b/crates/relearn-store/src/lib.rs @@ -0,0 +1,406 @@ +//! In-memory Relearn store: submissions, sealed holdout, champion. +//! +//! Holdout items stay sealed until the submission digest is frozen. + +#![forbid(unsafe_code)] +#![allow( + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::must_use_candidate +)] + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use relearn_challenge_task::HOLDOUT_DOMAIN; +use relearn_score::{PromoteVerdict, SliceScores}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +/// Submission lifecycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SubmissionState { + /// Digest accepted; holdout still sealed. + Accepted, + /// Digest frozen; holdout unsealed; eval running. + Evaluating, + /// Eval finished; waiting operator audit. + AwaitingAdmin, + /// Rejected (regression / gates / integrity). + Rejected, + /// Operator-promoted champion. + Champion, +} + +/// One miner submission. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Submission { + /// Stable id (`rl_` + 16 hex). + pub id: String, + /// 64-hex miner hotkey. + pub miner_hotkey: String, + /// SHA-256 hex of the miner artifact (weights / adapter). Frozen at accept. + pub artifact_digest: String, + /// Optional locator (HF repo, object URL). Never the scored teacher payload. + pub artifact_uri: Option, + /// Digest freeze nonce (hex). + pub nonce: String, + /// `sha256(hotkey || 0xff || artifact || 0xff || nonce)`. + pub submission_digest: String, + /// Lifecycle. + pub state: SubmissionState, + /// Eval receipt JSON (if any). + pub receipt_json: Option, + /// Judge verdict (if any). + pub verdict: Option, + /// Reject / gate reason. + pub detail: Option, +} + +/// Sealed holdout: items hidden until `unseal_after`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Holdout { + /// Slice id bound into the paired test. + pub slice_id: String, + /// Hex seed. Empty in the public view until unsealed. + pub seed_hex: String, + /// Whether the seed has been revealed for this submission. + pub unsealed: bool, +} + +/// Store errors. +#[derive(Debug, Error)] +pub enum StoreError { + /// Lock poisoned. + #[error("store lock poisoned")] + Poison, + /// Unknown submission. + #[error("unknown submission {0}")] + NotFound(String), + /// Illegal state transition. + #[error("illegal state {0}")] + Illegal(String), +} + +/// In-memory store (v0). Postgres can replace this without changing the HTTP surface. +#[derive(Clone, Default)] +pub struct MemoryStore { + inner: Arc>, +} + +#[derive(Default)] +struct Inner { + next: u64, + submissions: BTreeMap, + champion_id: Option, + /// Per-submission holdout slices (in-memory; not serialized on the HTTP row). + scores: BTreeMap, + /// Live champion slices (promoted miner). Displacement is vs this, not the base. + champion_scores: Option, + /// Baseline champion scores (base model) until a miner is promoted. + base_champion: Option, +} + +impl MemoryStore { + /// Empty store. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + fn lock(&self) -> Result, StoreError> { + self.inner.lock().map_err(|_| StoreError::Poison) + } + + /// Insert a newly accepted submission. + pub fn insert(&self, mut row: Submission) -> Result { + let mut g = self.lock()?; + if row.id.is_empty() { + let n = g.next; + g.next = g.next.saturating_add(1); + row.id = format!("rl_{n:016x}"); + } + g.submissions.insert(row.id.clone(), row.clone()); + Ok(row) + } + + /// Fetch one row. + pub fn get(&self, id: &str) -> Result { + let g = self.lock()?; + g.submissions + .get(id) + .cloned() + .ok_or_else(|| StoreError::NotFound(id.to_owned())) + } + + /// List newest-first. + pub fn list(&self) -> Result, StoreError> { + let g = self.lock()?; + let mut rows: Vec<_> = g.submissions.values().cloned().collect(); + rows.sort_by(|a, b| b.id.cmp(&a.id)); + Ok(rows) + } + + /// Patch state / verdict / receipt. + pub fn patch( + &self, + id: &str, + state: Option, + receipt_json: Option, + verdict: Option, + detail: Option, + ) -> Result { + let mut g = self.lock()?; + let row = g + .submissions + .get_mut(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + if let Some(s) = state { + row.state = s; + } + if let Some(r) = receipt_json { + row.receipt_json = Some(r); + } + if let Some(v) = verdict { + row.verdict = Some(v); + } + if let Some(d) = detail { + row.detail = Some(d); + } + Ok(row.clone()) + } + + /// Current champion submission id. + pub fn champion_id(&self) -> Result, StoreError> { + Ok(self.lock()?.champion_id.clone()) + } + + /// Promote `id` and demote the previous champion. + pub fn promote(&self, id: &str) -> Result { + let mut g = self.lock()?; + let prev = g.champion_id.clone(); + { + let row = g + .submissions + .get(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + if row.state != SubmissionState::AwaitingAdmin { + return Err(StoreError::Illegal(format!( + "promote requires awaiting_admin, got {:?}", + row.state + ))); + } + if !row.verdict.as_ref().is_some_and(|v| v.eligible) { + return Err(StoreError::Illegal( + "promote refused: verdict not eligible (regression or gates)".into(), + )); + } + } + if let Some(p) = prev { + if let Some(old) = g.submissions.get_mut(&p) { + if old.state == SubmissionState::Champion { + old.state = SubmissionState::Rejected; + old.detail = Some("superseded".into()); + } + } + } + { + let row = g + .submissions + .get_mut(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + row.state = SubmissionState::Champion; + } + if let Some(s) = g.scores.get(id).cloned() { + g.champion_scores = Some(s); + } + g.champion_id = Some(id.to_owned()); + g.submissions + .get(id) + .cloned() + .ok_or_else(|| StoreError::NotFound(id.to_owned())) + } + + /// Persist challenger slices so a later promote displaces vs this run. + pub fn record_scores(&self, id: &str, scores: SliceScores) -> Result<(), StoreError> { + self.lock()?.scores.insert(id.to_owned(), scores); + Ok(()) + } + + /// Seed / replace the implicit base-model champion scores. + pub fn set_base_champion(&self, scores: SliceScores) -> Result<(), StoreError> { + self.lock()?.base_champion = Some(scores); + Ok(()) + } + + /// Champion slice scores (promoted miner, else base model). + pub fn champion_scores(&self) -> Result, StoreError> { + let g = self.lock()?; + if let Some(s) = &g.champion_scores { + return Ok(Some(s.clone())); + } + Ok(g.base_champion.clone()) + } +} + +/// SHA-256 hex of the frozen submission. +#[must_use] +pub fn freeze_submission_digest(hotkey: &str, artifact_digest: &str, nonce: &str) -> String { + let mut h = Sha256::new(); + h.update(hotkey.as_bytes()); + h.update([0xff]); + h.update(artifact_digest.as_bytes()); + h.update([0xff]); + h.update(nonce.as_bytes()); + hex::encode(h.finalize()) +} + +/// Build a holdout that is sealed until `digest` is recorded. +#[must_use] +pub fn sealed_holdout(epoch: u64, digest: &str) -> Holdout { + let mut h = Sha256::new(); + h.update(HOLDOUT_DOMAIN); + h.update(epoch.to_le_bytes()); + h.update(digest.as_bytes()); + let seed = hex::encode(h.finalize()); + Holdout { + slice_id: format!("relearn-holdout-{epoch}"), + seed_hex: String::new(), + unsealed: false, + } + .with_pending_seed(seed) +} + +trait WithPending { + fn with_pending_seed(self, seed: String) -> Self; +} + +impl WithPending for Holdout { + fn with_pending_seed(mut self, seed: String) -> Self { + // Keep seed off the public struct until unseal. + self.seed_hex = seed; + self.unsealed = false; + self + } +} + +/// Reveal holdout seed only after the submission digest is frozen. +#[must_use] +pub fn unseal_holdout(pending: &Holdout, frozen_digest: &str) -> Option { + if frozen_digest.is_empty() || pending.seed_hex.is_empty() { + return None; + } + Some(Holdout { + slice_id: pending.slice_id.clone(), + seed_hex: pending.seed_hex.clone(), + unsealed: true, + }) +} + +/// Public view: seed stripped until unsealed. +#[must_use] +pub fn public_holdout(h: &Holdout) -> Holdout { + if h.unsealed { + h.clone() + } else { + Holdout { + slice_id: h.slice_id.clone(), + seed_hex: String::new(), + unsealed: false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn digest_stable_and_distinct() { + let a = freeze_submission_digest("aa", "bb", "n1"); + let b = freeze_submission_digest("aa", "bb", "n1"); + let c = freeze_submission_digest("aa", "bb", "n2"); + assert_eq!(a, b); + assert_ne!(a, c); + assert_eq!(a.len(), 64); + } + + #[test] + fn holdout_stays_sealed_in_public_view() { + let h = sealed_holdout(7, "deadbeef"); + assert!(!h.unsealed); + assert!(!h.seed_hex.is_empty()); + let pub_v = public_holdout(&h); + assert!(pub_v.seed_hex.is_empty()); + let open = unseal_holdout(&h, "deadbeef").expect("unseal"); + assert!(open.unsealed); + assert!(!open.seed_hex.is_empty()); + } + + #[test] + fn promote_refuses_ineligible() { + let st = MemoryStore::new(); + let row = st + .insert(Submission { + id: String::new(), + miner_hotkey: "00".repeat(32), + artifact_digest: "11".repeat(32), + artifact_uri: None, + nonce: "aa".into(), + submission_digest: "bb".repeat(32), + state: SubmissionState::AwaitingAdmin, + receipt_json: None, + verdict: None, + detail: None, + }) + .expect("insert"); + assert!(st.promote(&row.id).is_err()); + } + + #[test] + fn champion_scores_follow_promote_not_base() { + use prism_competition::ExampleSeries; + use relearn_score::SliceScores; + + fn series(prefix: &str, n: usize, val: f64) -> ExampleSeries { + ExampleSeries::from_pairs((0..n).map(|i| (format!("{prefix}{i}"), val))) + } + fn slice(v: f64) -> SliceScores { + SliceScores { + holdout: series("h", 8, v), + public: series("p", 8, v), + perturbed: series("x", 8, v), + canaries: series("c", 8, v), + agent_trace: 0.9, + } + } + + let st = MemoryStore::new(); + st.set_base_champion(slice(0.4)).expect("base"); + let row = st + .insert(Submission { + id: String::new(), + miner_hotkey: "00".repeat(32), + artifact_digest: "11".repeat(32), + artifact_uri: None, + nonce: "aa".into(), + submission_digest: "bb".repeat(32), + state: SubmissionState::AwaitingAdmin, + receipt_json: None, + verdict: Some(relearn_score::PromoteVerdict { + eligible: true, + paired: None, + failed: Vec::new(), + lattice: 12, + }), + detail: None, + }) + .expect("insert"); + st.record_scores(&row.id, slice(0.8)).expect("scores"); + st.promote(&row.id).expect("promote"); + let got = st.champion_scores().expect("read").expect("some"); + assert!((SliceScores::mean(&got.holdout).unwrap_or(0.0) - 0.8).abs() < 1e-9); + } +} diff --git a/crates/relearn-t2i-challenge/Cargo.toml b/crates/relearn-t2i-challenge/Cargo.toml new file mode 100644 index 000000000..dcbd75109 --- /dev/null +++ b/crates/relearn-t2i-challenge/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "relearn-t2i-challenge" +description = "Relearn T2I orchestrator: submit, eval, operator promote, D24 emit" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +bundle = { path = "../bundle" } +challenge-common = { path = "../challenge-common" } +crypto = { path = "../crypto" } +hex = "0.4" +relearn-t2i-eval = { path = "../relearn-t2i-eval" } +relearn-t2i-http = { path = "../relearn-t2i-http" } +relearn-t2i-score = { path = "../relearn-t2i-score" } +relearn-t2i-store = { path = "../relearn-t2i-store" } +relearn-t2i-task = { path = "../relearn-t2i-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[lints] +workspace = true diff --git a/crates/relearn-t2i-challenge/src/lib.rs b/crates/relearn-t2i-challenge/src/lib.rs new file mode 100644 index 000000000..40248006c --- /dev/null +++ b/crates/relearn-t2i-challenge/src/lib.rs @@ -0,0 +1,171 @@ +//! Relearn T2I orchestrator helpers: D24 leaf plan + crate re-exports. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] + +use std::collections::{BTreeMap, BTreeSet}; + +use bundle::{NoScoreReasonCode, ScoreOrAbsence}; +use challenge_common::{emit_signed_leaf_set, Hotkey, LeafEmitError}; +use relearn_t2i_score::champion_hold_lattice; +use relearn_t2i_store::SubmissionState; +use relearn_t2i_task::{CHALLENGE_ID_BYTES, SCORE_MAX}; + +pub use relearn_t2i_eval::{resolve_judge_backend, JudgeBackend, JudgeConfig}; +pub use relearn_t2i_http::{hash_admin_token, relearn_t2i_router, AppState}; +pub use relearn_t2i_store::{ArtifactManifest, MemoryStore}; +pub use relearn_t2i_task::{ + FrozenPrompt, RelearnT2iPin, BASE_MODEL_ID, BASE_MODEL_LICENSE, CHALLENGE_ID, + CHALLENGE_ID_BYTES as RELEARN_T2I_ID_BYTES, JUDGE_MODEL_ID, SCORE_MAX as RELEARN_T2I_SCORE_MAX, + SCORING_VERSION, +}; + +/// Build a D24-complete score map: champion (if any) gets a positive lattice; +/// everyone else is explicit `NoScore` (never silent). +pub fn emission_scores( + expected: &BTreeSet, + champion_hotkey: Option, + champion_lattice: u64, +) -> BTreeMap { + expected + .iter() + .map(|h| { + let s = match champion_hotkey { + Some(c) if c == *h && champion_lattice > 0 => ScoreOrAbsence::Score { + value: champion_lattice.min(SCORE_MAX), + }, + _ => ScoreOrAbsence::NoScore { + reason: NoScoreReasonCode::NotAttempted, + }, + }; + (*h, s) + }) + .collect() +} + +/// Sign the exact-E leaf set for this epoch. +/// +/// # Errors +/// +/// See [`LeafEmitError`]. +pub fn emit_epoch( + secret: &[u8; 32], + epoch: u64, + expected: &BTreeSet, + champion_hotkey: Option, + champion_lattice: u64, +) -> Result, LeafEmitError> { + let scores = emission_scores(expected, champion_hotkey, champion_lattice); + emit_signed_leaf_set(secret, CHALLENGE_ID_BYTES, epoch, expected, &scores) +} + +/// Lattice for the current store champion, or the hold value when only +/// the pinned base checkpoint is live. +pub fn live_champion_lattice(store: &MemoryStore) -> u64 { + if let Ok(Some(id)) = store.champion_id() { + if let Ok(row) = store.get(&id) { + if row.state == SubmissionState::Champion { + if let Some(v) = row.verdict { + if v.eligible && v.lattice > 0 { + return v.lattice; + } + } + return champion_hold_lattice(); + } + } + } + 0 +} + +/// Parse a 64-hex hotkey. +pub fn parse_hotkey(hex_s: &str) -> Option { + let t = hex_s.trim().trim_start_matches("0x"); + let bytes = hex::decode(t).ok()?; + <[u8; 32]>::try_from(bytes).ok() +} + +/// Load frozen holdout prompt records from an operator JSON file body. +/// +/// The file is a JSON array of `{ "id", "text", "upsampled_json"? }`. It is +/// verified against the pin's commitment by +/// [`MemoryStore::load_holdout`][relearn_t2i_store::MemoryStore::load_holdout], +/// so a wrong or edited file scores nothing instead of falling back. +/// +/// # Errors +/// +/// Returns the serde message when the body is not a prompt array. +pub fn parse_holdout_file(body: &str) -> Result, String> { + serde_json::from_str(body).map_err(|e| format!("parse holdout records: {e}")) +} + +#[cfg(test)] +mod tests { + use challenge_common::public_key_from_secret; + use crypto::KEY_LEN; + + use super::*; + + fn sk() -> [u8; KEY_LEN] { + let mut s = [9u8; KEY_LEN]; + s[0] = 3; + s + } + + #[test] + fn d24_covers_every_hotkey() { + let a = [1u8; 32]; + let b = [2u8; 32]; + let e: BTreeSet = [a, b].into_iter().collect(); + let leaves = emit_epoch(&sk(), 9, &e, Some(a), 12_000).expect("emit"); + assert_eq!(leaves.len(), 2); + assert!(matches!( + leaves[&a].score_or_absence, + ScoreOrAbsence::Score { value: 12_000 } + )); + assert!(matches!( + leaves[&b].score_or_absence, + ScoreOrAbsence::NoScore { .. } + )); + let pk = public_key_from_secret(&sk()).expect("pk"); + for leaf in leaves.values() { + challenge_common::verify_leaf_sig(leaf, &pk).expect("sig"); + } + } + + #[test] + fn leaf_domain_is_not_the_text_challenge() { + assert_eq!(RELEARN_T2I_ID_BYTES, b"relearn-t2i"); + assert_ne!(RELEARN_T2I_ID_BYTES, b"relearn"); + } + + #[test] + fn never_emits_score_on_empty_expected() { + let leaves = emit_epoch(&sk(), 1, &BTreeSet::new(), None, 0).expect("empty E"); + assert!(leaves.is_empty()); + } + + #[test] + fn no_champion_means_zero_lattice() { + assert_eq!(live_champion_lattice(&MemoryStore::new()), 0); + } + + #[test] + fn holdout_file_parses_records() { + let body = r#"[{"id": 900, "text": "a red cube"}, {"id": 901, "text": "two cats", "upsampled_json": "{}"}]"#; + let recs = parse_holdout_file(body).expect("parse"); + assert_eq!(recs.len(), 2); + assert_eq!(recs[1].upsampled_json.as_deref(), Some("{}")); + assert!(parse_holdout_file("not json").is_err()); + } + + #[test] + fn hotkey_parse_round_trip() { + let hex_s = "ab".repeat(32); + assert!(parse_hotkey(&hex_s).is_some()); + assert!(parse_hotkey("0xzz").is_none()); + } +} diff --git a/crates/relearn-t2i-eval/Cargo.toml b/crates/relearn-t2i-eval/Cargo.toml new file mode 100644 index 000000000..ba7631bdc --- /dev/null +++ b/crates/relearn-t2i-eval/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "relearn-t2i-eval" +description = "Relearn T2I eval loop: frozen cells, Q-Judger scoring, Lium rent, receipts" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +hex = "0.4" +prism-competition = { path = "../prism-competition" } +prism-lium = { path = "../prism-lium" } +prism-lium-types = { path = "../prism-lium-types" } +relearn-t2i-judge = { path = "../relearn-t2i-judge" } +relearn-t2i-score = { path = "../relearn-t2i-score" } +relearn-t2i-store = { path = "../relearn-t2i-store" } +relearn-t2i-task = { path = "../relearn-t2i-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +thiserror = "2" + +[dev-dependencies] +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread"] } + +[lints] +workspace = true diff --git a/crates/relearn-t2i-eval/src/lib.rs b/crates/relearn-t2i-eval/src/lib.rs new file mode 100644 index 000000000..02366c4a6 --- /dev/null +++ b/crates/relearn-t2i-eval/src/lib.rs @@ -0,0 +1,710 @@ +//! Relearn T2I eval loop: freeze digest → unseal prompts → generate → judge. +//! +//! The control plane only ever boots a digest-pinned eval image, and the miner +//! pays Lium (`LIUM_API_KEY` / `X-Lium-Api-Key`). Q-Judger runs judge-only: it +//! never serves miner weights as the scored artifact. +//! +//! Backend resolution is fail-closed. A host with no judge endpoint and no +//! explicit sim opt-in refuses to score rather than emitting a deterministic +//! placeholder that would look like a passing eval. Sim exists for CI and for +//! local development, is selected only by `RELEARN_T2I_FORCE_SIM=1` or +//! `RELEARN_T2I_JUDGE_BACKEND=sim`, and is reported on `/v1/status` so an +//! operator can never mistake it for a real run. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::must_use_candidate +)] + +use std::collections::{BTreeMap, BTreeSet}; + +use prism_competition::ExampleSeries; +use prism_lium::{EvalJobBackend, SimLiumBackend}; +use prism_lium_types::{EvalReceipt, InstanceSpec, NoScoreGate, RemoteExecResult}; +use relearn_t2i_judge::{assert_judge_model, ImageScore, JudgeError, JudgeInference}; +use relearn_t2i_score::{ + contamination, FaithfulnessEvidence, ReplayEvidence, T2iSliceScores, MIN_FAITHFULNESS_CHECKS, + REPLAY_CELLS, +}; +use relearn_t2i_store::ArtifactManifest; +use relearn_t2i_task::{ + cell_key, FrozenPrompt, L1Dimension, PinError, RelearnT2iPin, SeedCell, JUDGE_MODEL_ID, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +/// Where Q-Judger runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JudgeBackend { + /// Q-Judger behind an OpenAI-compatible HTTP API (`RELEARN_T2I_JUDGE_API_URL`). + HttpApi, + /// Q-Judger on a digest-pinned Lium pod (`RELEARN_T2I_JUDGE_BACKEND=lium`). + Lium, + /// Deterministic offline judge (CI / local only). + Sim, +} + +impl JudgeBackend { + /// Parse from `RELEARN_T2I_JUDGE_BACKEND`. Empty → HTTP API. + #[must_use] + pub fn from_env() -> Self { + match std::env::var("RELEARN_T2I_JUDGE_BACKEND") + .unwrap_or_default() + .to_ascii_lowercase() + .as_str() + { + "lium" | "lium_pod" => Self::Lium, + "sim" => Self::Sim, + _ => Self::HttpApi, + } + } +} + +/// `RELEARN_T2I_JUDGE_API_URL` when set. No baked host — missing means refuse. +#[must_use] +pub fn judge_api_url() -> Option { + std::env::var("RELEARN_T2I_JUDGE_API_URL") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} + +/// Bearer for the Q-Judger HTTP API. Never log the value. +#[must_use] +pub fn judge_api_key() -> Option { + std::env::var("RELEARN_T2I_JUDGE_API_KEY") + .ok() + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} + +/// True when the operator explicitly opted into sim. +#[must_use] +pub fn force_sim() -> bool { + matches!( + std::env::var("RELEARN_T2I_FORCE_SIM") + .unwrap_or_default() + .to_ascii_lowercase() + .as_str(), + "1" | "true" | "yes" + ) +} + +/// Resolve the judge backend for this host. +#[must_use] +pub fn resolve_judge_backend() -> JudgeBackend { + if force_sim() { + return JudgeBackend::Sim; + } + JudgeBackend::from_env() +} + +/// Judge wiring for this process, resolved once at boot. +/// +/// The endpoint is deliberately not `Serialize`: `/v1/status` reports whether +/// one is configured, never the host. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JudgeConfig { + /// Where Q-Judger runs. + pub backend: JudgeBackend, + /// Judge endpoint from `RELEARN_T2I_JUDGE_API_URL`, if any. + endpoint: Option, +} + +impl JudgeConfig { + /// Read `RELEARN_T2I_*` once. Call this at process start, not per request. + #[must_use] + pub fn from_env() -> Self { + Self { + backend: resolve_judge_backend(), + endpoint: judge_api_url(), + } + } + + /// Deterministic offline config for CI and local development. + #[must_use] + pub fn sim() -> Self { + Self { + backend: JudgeBackend::Sim, + endpoint: None, + } + } + + /// Whether a judge endpoint is configured. Never exposes the value. + #[must_use] + pub fn endpoint_configured(&self) -> bool { + self.endpoint.is_some() + } + + /// Refuse to score unless Q-Judger is the judge and a backend is reachable. + /// + /// # Errors + /// + /// [`EvalError::Judge`] when the model is not Q-Judger, and + /// [`EvalError::JudgeUnconfigured`] when the HTTP backend has no endpoint. + pub fn preflight(&self, inference: &JudgeInference) -> Result<(), EvalError> { + assert_judge_model(&inference.model)?; + if self.backend == JudgeBackend::HttpApi && self.endpoint.is_none() { + return Err(EvalError::JudgeUnconfigured); + } + Ok(()) + } +} + +/// Eval errors. +#[derive(Debug, Error)] +pub enum EvalError { + /// Holdout was requested before the digest freeze, or none is loaded. + #[error("holdout unavailable: {0}")] + Holdout(String), + /// Integrity gate failed. + #[error("integrity: {0}")] + Integrity(String), + /// Lium / backend failure. + #[error("backend: {0}")] + Backend(String), + /// The artifact's declared base or license does not match the pin. + #[error("license attestation: {0}")] + Attestation(#[from] PinError), + /// Q-Judger replied with something unusable. + #[error("judge: {0}")] + Judge(#[from] JudgeError), + /// No judge endpoint configured and sim was not opted into. + #[error("Q-Judger backend unconfigured: set RELEARN_T2I_JUDGE_API_URL, or RELEARN_T2I_FORCE_SIM=1 for CI")] + JudgeUnconfigured, + /// The judge declined too many items to produce a comparable score. + #[error("judge N/A rate {rate:.3} above the {max:.3} ceiling")] + NotApplicableRate { + /// Observed rate. + rate: f64, + /// Ceiling. + max: f64, + }, +} + +/// One finished eval. +#[derive(Debug, Clone)] +pub struct EvalOutcome { + /// Challenger measurements. + pub scores: T2iSliceScores, + /// Integrity receipt. + pub receipt: EvalReceipt, + /// Backend that produced the scores. + pub backend: JudgeBackend, + /// Number of holdout cells scored. + pub holdout_cells: usize, +} + +/// Deterministic per-cell judge score derived from a digest and a cell key. +/// +/// Sim only. Every value is a function of the frozen submission digest, so a +/// sim run is reproducible and is never mistaken for evidence about the model. +fn sim_image_score(artifact_digest: &str, key: &str, bias: f64) -> ImageScore { + let mut per_l1 = BTreeMap::new(); + for (i, dim) in L1Dimension::ALL.into_iter().enumerate() { + let mut h = Sha256::new(); + h.update(artifact_digest.as_bytes()); + h.update([0xff]); + h.update(key.as_bytes()); + h.update([0xff, u8::try_from(i).unwrap_or(0)]); + let d = h.finalize(); + let unit = f64::from(d[0]) / 255.0; + per_l1.insert(dim, ((0.45 + 0.40 * unit + bias) * 100.0).clamp(0.0, 100.0)); + } + let total = per_l1.values().sum::() / per_l1.len() as f64; + ImageScore { + per_l1, + total, + scored_items: 20, + na_items: 1, + } +} + +/// Fold per-cell [`ImageScore`]s into the normalized series the gates read. +/// +/// # Errors +/// +/// [`EvalError::NotApplicableRate`] when the judge declined too much of the +/// holdout, and [`EvalError::Holdout`] when the holdout produced no cells. +pub fn fold_scores( + holdout: &BTreeMap, + public: &BTreeMap, + replay: ReplayEvidence, + faithfulness: FaithfulnessEvidence, + contaminated_prompt_ids: Vec, +) -> Result { + if holdout.is_empty() { + return Err(EvalError::Holdout("no holdout cells scored".into())); + } + let series = |m: &BTreeMap| { + ExampleSeries::from_pairs(m.iter().map(|(k, v)| (k.clone(), v.normalized_total()))) + }; + let mut by_pillar: BTreeMap = BTreeMap::new(); + for dim in L1Dimension::ALL { + let pairs: Vec<(String, f64)> = holdout + .iter() + .filter_map(|(k, v)| v.normalized_pillar(dim).map(|x| (k.clone(), x))) + .collect(); + if !pairs.is_empty() { + by_pillar.insert(dim, ExampleSeries::from_pairs(pairs)); + } + } + + let na_items: u32 = holdout.values().map(|v| v.na_items).sum(); + let scored_items: u32 = holdout.values().map(|v| v.scored_items).sum(); + let denom = f64::from(na_items) + f64::from(scored_items); + let na_rate = if denom <= 0.0 { + 1.0 + } else { + f64::from(na_items) / denom + }; + if na_rate > relearn_t2i_score::MAX_NA_RATE { + return Err(EvalError::NotApplicableRate { + rate: na_rate, + max: relearn_t2i_score::MAX_NA_RATE, + }); + } + + Ok(T2iSliceScores { + holdout: series(holdout), + public: series(public), + holdout_by_pillar: by_pillar, + na_rate, + replay, + faithfulness, + contaminated_prompt_ids, + }) +} + +/// Score every cell of a split in sim. +fn sim_split( + pin: &RelearnT2iPin, + prompt_ids: &[u32], + artifact_digest: &str, + bias: f64, +) -> BTreeMap { + pin.seed_cells(prompt_ids) + .into_iter() + .map( + |SeedCell { + prompt_id, + variation_index, + seed, + }| { + let key = cell_key(prompt_id, variation_index); + let salted = format!("{artifact_digest}:{seed}"); + let score = sim_image_score(&salted, &key, bias); + (key, score) + }, + ) + .collect() +} + +/// Sim-mode challenger measurements for a frozen digest. +/// +/// # Errors +/// +/// See [`fold_scores`]. +pub fn sim_slice_scores( + pin: &RelearnT2iPin, + holdout_ids: &[u32], + artifact_digest: &str, +) -> Result { + let holdout = sim_split(pin, holdout_ids, artifact_digest, 0.10); + let public = sim_split(pin, &pin.prompts.public_ids, artifact_digest, 0.10); + fold_scores( + &holdout, + &public, + ReplayEvidence { + cells_checked: REPLAY_CELLS, + exact_hash_matches: REPLAY_CELLS, + max_embedding_drift: 0.0, + }, + FaithfulnessEvidence { + checks: MIN_FAITHFULNESS_CHECKS, + agreements: MIN_FAITHFULNESS_CHECKS, + }, + Vec::new(), + ) +} + +/// Fixed base-checkpoint champion (pinned Cosmos3, no miner fine-tune). +/// +/// # Errors +/// +/// See [`fold_scores`]. +pub fn base_champion_scores( + pin: &RelearnT2iPin, + holdout_ids: &[u32], +) -> Result { + let holdout = sim_split(pin, holdout_ids, "cosmos3-super-text2image-base", 0.0); + let public = sim_split( + pin, + &pin.prompts.public_ids, + "cosmos3-super-text2image-base", + 0.0, + ); + fold_scores( + &holdout, + &public, + ReplayEvidence { + cells_checked: REPLAY_CELLS, + exact_hash_matches: REPLAY_CELLS, + max_embedding_drift: 0.0, + }, + FaithfulnessEvidence { + checks: MIN_FAITHFULNESS_CHECKS, + agreements: MIN_FAITHFULNESS_CHECKS, + }, + Vec::new(), + ) +} + +/// Bench prompt ids a submission admits to having trained on that are also +/// scored. Non-empty means the contamination gate rejects the submission. +#[must_use] +pub fn contaminated_ids(manifest: &ArtifactManifest, eval_ids: &[u32]) -> Vec { + let train: BTreeSet = manifest.train_prompt_ids.iter().copied().collect(); + let eval: BTreeSet = eval_ids.iter().copied().collect(); + contamination(&train, &eval) +} + +/// Run one eval after the submission digest is frozen. +/// +/// # Errors +/// +/// Refuses on a failed license attestation, an unconfigured judge, an empty +/// holdout, or an excessive judge N/A rate. Contamination is not an error here: +/// it is recorded on the scores so the verdict reports it as a gate failure. +pub fn eval_after_freeze( + pin: &RelearnT2iPin, + holdout: &[FrozenPrompt], + frozen_digest: &str, + artifact_digest: &str, + manifest: &ArtifactManifest, + judge: &JudgeConfig, +) -> Result { + if frozen_digest.trim().is_empty() { + return Err(EvalError::Holdout("submission digest not frozen".into())); + } + if holdout.is_empty() { + return Err(EvalError::Holdout("holdout still sealed".into())); + } + pin.attest_artifact_base(&manifest.base, &manifest.base_license)?; + + judge.preflight(&JudgeInference::default())?; + let backend = judge.backend; + + let holdout_ids: Vec = holdout.iter().map(|p| p.id).collect(); + let mut scores = match backend { + JudgeBackend::Sim => sim_slice_scores(pin, &holdout_ids, artifact_digest)?, + JudgeBackend::HttpApi | JudgeBackend::Lium => { + // Live generation and judging happen inside the digest-pinned eval + // image; this control-plane path refuses rather than inventing + // scores when that image has not been pinned yet. + if !pin.can_rent() { + return Err(EvalError::Integrity( + "eval image digest not pinned; refuse live judge".into(), + )); + } + return Err(EvalError::Integrity( + "live Q-Judger harvest is driven by the eval image; no in-process fallback".into(), + )); + } + }; + scores.contaminated_prompt_ids = contaminated_ids(manifest, &holdout_ids); + + let holdout_cells = scores.holdout.len(); + let metrics = serde_json::to_vec(&serde_json::json!({ + "judge_model": JUDGE_MODEL_ID, + "holdout_cells": holdout_cells, + "na_rate": scores.na_rate, + })) + .unwrap_or_default(); + let receipt = EvalReceipt { + provider: match backend { + JudgeBackend::Sim => "sim".into(), + JudgeBackend::HttpApi => "http_api".into(), + JudgeBackend::Lium => "lium".into(), + }, + pod_id: format!("t2i-{}", &frozen_digest[..8.min(frozen_digest.len())]), + image_digest: pin.eval_image_digest.clone(), + submission_hash: frozen_digest.to_owned(), + metrics_hash: EvalReceipt::hash_metrics_bytes(&metrics), + termination_verified: true, + }; + NoScoreGate::check(&receipt, false).map_err(|e| EvalError::Integrity(e.to_string()))?; + + Ok(EvalOutcome { + scores, + receipt, + backend, + holdout_cells, + }) +} + +/// Rent a digest-pinned eval pod, exec, harvest, terminate. +/// +/// Live rent is skipped when `pin.can_rent()` is false (no published eval +/// digest yet). +/// +/// # Errors +/// +/// [`EvalError::Integrity`] without a digest pin or an unverified teardown, and +/// [`EvalError::Backend`] on any provider failure. +pub async fn rent_eval( + backend: &dyn EvalJobBackend, + pin: &RelearnT2iPin, + frozen_digest: &str, + artifact_digest: &str, +) -> Result<(RemoteExecResult, String), EvalError> { + if !pin.can_rent() { + return Err(EvalError::Integrity( + "eval image digest not pinned; refuse live rent".into(), + )); + } + let spec = InstanceSpec { + name: format!( + "relearn-t2i-{}", + &frozen_digest[..12.min(frozen_digest.len())] + ), + max_lifetime_hours: 2.0, + // Cosmos3-Super is 65B at BF16 and Q-Judger is 27B, so the pod is a + // multi-GPU node rather than the single card the text challenge uses. + max_price_per_hour: 48.0, + gpu_count: 8, + image_digest: Some(pin.eval_image_digest.clone()), + ssh_public_keys: Vec::new(), + ssh_key_name: None, + preferred_offer_id: None, + template_id: None, + template_name: None, + }; + let inst = backend + .provision(&spec) + .await + .map_err(|e| EvalError::Backend(e.to_string()))?; + let exec = backend + .exec_eval(&inst.id, artifact_digest, frozen_digest, None) + .await + .map_err(|e| EvalError::Backend(e.to_string())); + let term = backend.terminate(&inst.id).await; + let verified = backend.verify_terminated(&inst.id).await.unwrap_or(false); + if let Err(e) = term { + return Err(EvalError::Backend(e.to_string())); + } + if !verified { + return Err(EvalError::Integrity("pod terminate not verified".into())); + } + exec.map(|r| (r, inst.id)) +} + +/// Convenience: sim backend rent that always tears down. +/// +/// # Errors +/// +/// See [`rent_eval`]. +pub async fn sim_rent_roundtrip(digest: &str) -> Result { + let backend = SimLiumBackend::new(); + let pin = RelearnT2iPin { + eval_image_digest: format!("sha256:{}", "ab".repeat(32)), + ..RelearnT2iPin::default() + }; + let (_r, id) = rent_eval(&backend, &pin, digest, digest).await?; + Ok(id) +} + +#[cfg(test)] +mod tests { + use relearn_t2i_task::{frozen_prompt_commitment, PromptPin}; + + use super::*; + + fn prompt(id: u32) -> FrozenPrompt { + FrozenPrompt { + id, + text: format!("prompt {id}"), + upsampled_json: None, + } + } + + fn test_pin() -> RelearnT2iPin { + let public: Vec = (1..=25).map(prompt).collect(); + let holdout: Vec = (900..=924).map(prompt).collect(); + RelearnT2iPin { + prompts: PromptPin { + pin_salt: "cortex-t2i-test".into(), + variations_per_prompt: 4, + public_ids: public.iter().map(|p| p.id).collect(), + holdout_commitment: frozen_prompt_commitment(&holdout), + holdout_size: holdout.len(), + }, + frozen_prompts: public, + ..RelearnT2iPin::default() + } + } + + fn manifest() -> ArtifactManifest { + ArtifactManifest { + base: relearn_t2i_task::BASE_MODEL_ID.into(), + base_license: relearn_t2i_task::BASE_MODEL_LICENSE.into(), + ..ArtifactManifest::default() + } + } + + fn holdout() -> Vec { + (900..=924).map(prompt).collect() + } + + fn sim() -> JudgeConfig { + JudgeConfig::sim() + } + + #[test] + fn sim_eval_needs_a_frozen_digest_and_an_unsealed_holdout() { + let pin = test_pin(); + assert!(eval_after_freeze(&pin, &holdout(), "", "art", &manifest(), &sim()).is_err()); + assert!(eval_after_freeze(&pin, &[], "digest", "art", &manifest(), &sim()).is_err()); + let out = eval_after_freeze(&pin, &holdout(), "digest-a", "art", &manifest(), &sim()) + .expect("sim eval"); + assert_eq!(out.backend, JudgeBackend::Sim); + assert_eq!(out.holdout_cells, 100); + assert_eq!(out.receipt.provider, "sim"); + assert_eq!(out.receipt.submission_hash, "digest-a"); + } + + #[test] + fn sim_eval_is_deterministic() { + let pin = test_pin(); + let a = eval_after_freeze(&pin, &holdout(), "d", "art", &manifest(), &sim()).expect("a"); + let b = eval_after_freeze(&pin, &holdout(), "d", "art", &manifest(), &sim()).expect("b"); + assert_eq!(a.scores.holdout, b.scores.holdout); + } + + #[test] + fn flux_artifact_is_refused_before_any_scoring() { + let pin = test_pin(); + let mut m = manifest(); + m.base = "black-forest-labs/FLUX.1-dev".into(); + let err = eval_after_freeze(&pin, &holdout(), "d", "art", &m, &sim()).expect_err("refuse"); + assert!( + matches!(err, EvalError::Attestation(PinError::RejectedBase(_))), + "{err}" + ); + } + + #[test] + fn wrong_license_attestation_is_refused() { + let pin = test_pin(); + let mut m = manifest(); + m.base_license = "cc-by-nc-4.0".into(); + assert!(eval_after_freeze(&pin, &holdout(), "d", "art", &m, &sim()).is_err()); + } + + #[test] + fn contaminated_training_metadata_lands_on_the_scores() { + let pin = test_pin(); + let mut m = manifest(); + m.train_prompt_ids = vec![1, 2, 907]; + let out = eval_after_freeze(&pin, &holdout(), "d", "art", &m, &sim()).expect("eval"); + assert_eq!(out.scores.contaminated_prompt_ids, vec![907]); + } + + #[test] + fn live_backend_without_a_pinned_eval_image_refuses() { + let pin = test_pin(); + let live = JudgeConfig { + backend: JudgeBackend::HttpApi, + endpoint: Some("http://judge.invalid/v1".into()), + }; + let err = eval_after_freeze(&pin, &holdout(), "d", "art", &manifest(), &live) + .expect_err("must refuse"); + assert!(matches!(err, EvalError::Integrity(_)), "{err}"); + } + + #[test] + fn http_backend_without_an_endpoint_refuses_to_score() { + let unset = JudgeConfig { + backend: JudgeBackend::HttpApi, + endpoint: None, + }; + assert!(!unset.endpoint_configured()); + let err = unset + .preflight(&JudgeInference::default()) + .expect_err("must refuse"); + assert!(matches!(err, EvalError::JudgeUnconfigured), "{err}"); + } + + #[test] + fn only_q_judger_passes_preflight() { + let bad = JudgeInference { + model: "google/gemma-3".into(), + ..JudgeInference::default() + }; + assert!(matches!(sim().preflight(&bad), Err(EvalError::Judge(_)))); + sim() + .preflight(&JudgeInference::default()) + .expect("q-judger + sim"); + } + + #[test] + fn high_na_rate_fails_closed_in_the_fold() { + let mut holdout = BTreeMap::new(); + holdout.insert( + "p1#v0".to_owned(), + ImageScore { + per_l1: BTreeMap::from([(L1Dimension::Quality, 100.0)]), + total: 100.0, + scored_items: 1, + na_items: 20, + }, + ); + let err = fold_scores( + &holdout, + &BTreeMap::new(), + ReplayEvidence::default(), + FaithfulnessEvidence::default(), + Vec::new(), + ) + .expect_err("must refuse"); + assert!(matches!(err, EvalError::NotApplicableRate { .. }), "{err}"); + } + + #[test] + fn empty_holdout_fold_is_refused() { + assert!(fold_scores( + &BTreeMap::new(), + &BTreeMap::new(), + ReplayEvidence::default(), + FaithfulnessEvidence::default(), + Vec::new(), + ) + .is_err()); + } + + #[test] + fn base_champion_is_beatable_but_not_free() { + let pin = test_pin(); + let ids: Vec = holdout().iter().map(|p| p.id).collect(); + let base = base_champion_scores(&pin, &ids).expect("base"); + assert_eq!(base.holdout.len(), 100); + assert!(T2iSliceScores::mean(&base.holdout).unwrap_or(0.0) > 0.0); + } + + #[tokio::test] + async fn sim_rent_tears_down() { + let id = sim_rent_roundtrip("abcdef0123456789").await.expect("rent"); + assert!(id.contains("sim-pod")); + } + + #[test] + fn unpinned_eval_image_refuses_live_rent() { + assert!(!RelearnT2iPin::default().can_rent()); + } +} diff --git a/crates/relearn-t2i-http/Cargo.toml b/crates/relearn-t2i-http/Cargo.toml new file mode 100644 index 000000000..34db07e30 --- /dev/null +++ b/crates/relearn-t2i-http/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "relearn-t2i-http" +description = "Relearn T2I HTTP surface: submit, status, prompt split, admin promote" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "json"] } +hex = "0.4" +relearn-t2i-eval = { path = "../relearn-t2i-eval" } +relearn-t2i-judge = { path = "../relearn-t2i-judge" } +relearn-t2i-score = { path = "../relearn-t2i-score" } +relearn-t2i-store = { path = "../relearn-t2i-store" } +relearn-t2i-task = { path = "../relearn-t2i-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tokio = { version = "1", features = ["macros", "rt", "sync"] } + +[dev-dependencies] +http-body-util = "0.1" +tower = { version = "0.5", features = ["util"] } + +[lints] +workspace = true diff --git a/crates/relearn-t2i-http/src/lib.rs b/crates/relearn-t2i-http/src/lib.rs new file mode 100644 index 000000000..759f7cd45 --- /dev/null +++ b/crates/relearn-t2i-http/src/lib.rs @@ -0,0 +1,625 @@ +//! Relearn T2I HTTP API (master-only). +//! +//! ```text +//! GET /health +//! GET /v1/status +//! GET /v1/prompts frozen public split + seeds (holdout stays sealed) +//! POST /v1/submissions miner submit (digest + manifest + X-Lium-Api-Key) +//! GET /v1/submissions +//! GET /v1/submissions/{id} +//! POST /v1/admin/promote operator-audited champion flip +//! ``` +//! +//! `/v1/prompts` publishes the public split verbatim, including the derived +//! seeds, so every miner can reproduce the exact scored cells. It never +//! publishes the holdout: that response carries the commitment and the size +//! only. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate, + clippy::too_many_lines +)] + +use std::sync::Arc; + +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use relearn_t2i_eval::{eval_after_freeze, JudgeBackend, JudgeConfig}; +use relearn_t2i_judge::JudgeInference; +use relearn_t2i_score::judge_challenger; +use relearn_t2i_store::{ + freeze_submission_digest, ArtifactManifest, MemoryStore, Submission, SubmissionState, +}; +use relearn_t2i_task::{ + cell_key, RelearnT2iPin, BASE_MODEL_ID, BASE_MODEL_LICENSE, BASE_MODEL_LICENSE_URL, + CHALLENGE_ID, JUDGE_DATASET_ID, JUDGE_MODEL_ID, REJECTED_BASE_SUBSTRINGS, SCORE_MAX, + SCORING_VERSION, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Shared HTTP state. +#[derive(Clone)] +pub struct AppState { + /// Submission store. + pub store: MemoryStore, + /// Eval / model pins. + pub pin: RelearnT2iPin, + /// Judge wiring resolved once at boot (never read from env per request). + pub judge: JudgeConfig, + /// Operator bearer hashes (sha256 hex). Empty → admin 503. + pub admin_hashes: Arc>, +} + +/// Build the router. +pub fn relearn_t2i_router(state: AppState) -> Router { + Router::new() + .route("/health", get(health)) + .route("/v1/status", get(status)) + .route("/v1/prompts", get(prompts)) + .route("/v1/submissions", post(submit).get(list_subs)) + .route("/v1/submissions/{id}", get(get_sub)) + .route("/v1/admin/promote", post(promote)) + .with_state(state) +} + +async fn health() -> impl IntoResponse { + Json(serde_json::json!({ + "ok": true, + "challenge_id": CHALLENGE_ID, + "scoring_version": SCORING_VERSION, + })) +} + +async fn status(State(st): State) -> impl IntoResponse { + let champ = st.store.champion_id().ok().flatten(); + let seal = st.store.holdout_seal().ok(); + let judge = JudgeInference::default(); + Json(serde_json::json!({ + "challenge_id": CHALLENGE_ID, + "scoring_version": SCORING_VERSION, + "score_max": SCORE_MAX, + "base_model": st.pin.base, + "base_license": st.pin.base_license, + "base_license_url": st.pin.base_license_url, + "base_revision": st.pin.base_revision, + "rejected_base_families": REJECTED_BASE_SUBSTRINGS, + "judge_model": st.pin.judge_model, + "judge_dataset": st.pin.judge_dataset, + "judge_inference": judge, + "judge_backend": st.judge.backend, + // Whether an endpoint is set, never the endpoint itself. + "judge_endpoint_configured": st.judge.endpoint_configured(), + "sampler": st.pin.sampler, + "eval_image": st.pin.eval_image, + "eval_image_digest": st.pin.eval_image_digest, + "holdout": seal, + "champion_id": champ, + })) +} + +#[derive(Debug, Serialize)] +struct PromptCell { + prompt_id: u32, + variation_index: u32, + seed: u64, + cell_key: String, + prompt: String, +} + +async fn prompts(State(st): State) -> impl IntoResponse { + let cells: Vec = st + .pin + .seed_cells(&st.pin.prompts.public_ids) + .into_iter() + .filter_map(|c| { + let record = st.pin.frozen_prompts.iter().find(|p| p.id == c.prompt_id)?; + Some(PromptCell { + prompt_id: c.prompt_id, + variation_index: c.variation_index, + seed: c.seed, + cell_key: cell_key(c.prompt_id, c.variation_index), + prompt: record.generator_input().to_owned(), + }) + }) + .collect(); + Json(serde_json::json!({ + "dataset": JUDGE_DATASET_ID, + "pin_salt": st.pin.prompts.pin_salt, + "variations_per_prompt": st.pin.prompts.variations_per_prompt, + "sampler": st.pin.sampler, + "public": cells, + // Holdout stays sealed: commitment and size only, never ids or text. + "holdout": st.store.holdout_seal().ok(), + })) +} + +#[derive(Debug, Deserialize)] +struct SubmitBody { + miner_hotkey: String, + artifact_digest: String, + artifact_uri: Option, + #[serde(default)] + manifest: ArtifactManifest, +} + +#[derive(Debug, Serialize)] +struct SubmitResp { + id: String, + submission_digest: String, + state: SubmissionState, + judge_backend: JudgeBackend, + holdout_cells: usize, + eligible: bool, +} + +fn parse_hex64(s: &str, field: &str) -> Result)> { + let t = s.trim().trim_start_matches("0x"); + if t.len() != 64 || !t.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(err(StatusCode::BAD_REQUEST, &format!("invalid {field}"))); + } + Ok(t.to_ascii_lowercase()) +} + +fn nonce_from(hotkey: &str, digest: &str) -> String { + let mut h = Sha256::new(); + h.update(b"relearn-t2i-nonce-v1"); + h.update(hotkey.as_bytes()); + h.update(digest.as_bytes()); + hex::encode(h.finalize()) +} + +async fn submit( + State(st): State, + headers: HeaderMap, + Json(body): Json, +) -> Result)> { + let hotkey = parse_hex64(&body.miner_hotkey, "miner_hotkey")?; + let artifact = parse_hex64(&body.artifact_digest, "artifact_digest")?; + // Miner BYOK: accepted and never logged. Absence is OK for sim. + let _lium_present = headers + .get("x-lium-api-key") + .and_then(|v| v.to_str().ok()) + .is_some_and(|s| !s.is_empty()); + + // License attestation is checked before anything is stored, so a Flux + // fine-tune is a 400 with a reason rather than a row that quietly scores 0. + st.pin + .attest_artifact_base(&body.manifest.base, &body.manifest.base_license) + .map_err(|e| err(StatusCode::BAD_REQUEST, &e.to_string()))?; + + let nonce = nonce_from(&hotkey, &artifact); + let submission_digest = freeze_submission_digest(&hotkey, &artifact, &nonce); + + let row = st + .store + .insert(Submission { + id: String::new(), + miner_hotkey: hotkey, + artifact_digest: artifact.clone(), + artifact_uri: body.artifact_uri, + manifest: body.manifest.clone(), + nonce, + submission_digest: submission_digest.clone(), + state: SubmissionState::Evaluating, + receipt_json: None, + verdict: None, + detail: None, + }) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + + let holdout = st + .store + .unseal_holdout(&submission_digest) + .map_err(|e| err(StatusCode::SERVICE_UNAVAILABLE, &e.to_string()))?; + + let eval = eval_after_freeze( + &st.pin, + &holdout, + &submission_digest, + &artifact, + &body.manifest, + &st.judge, + ) + .map_err(|e| err(StatusCode::INTERNAL_SERVER_ERROR, &e.to_string()))?; + + let champ = st.store.champion_scores().ok().flatten().ok_or_else(|| { + err( + StatusCode::SERVICE_UNAVAILABLE, + "no champion baseline recorded", + ) + })?; + let verdict = judge_challenger(&champ, &eval.scores); + st.store + .record_scores(&row.id, eval.scores.clone()) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + + let eligible = verdict.eligible; + let state = if eligible { + SubmissionState::AwaitingAdmin + } else { + SubmissionState::Rejected + }; + let detail = if eligible { + None + } else { + Some(format!("gates={:?}", verdict.failed)) + }; + let receipt = serde_json::to_string(&eval.receipt).unwrap_or_default(); + let row = st + .store + .patch(&row.id, Some(state), Some(receipt), Some(verdict), detail) + .map_err(|_| err(StatusCode::INTERNAL_SERVER_ERROR, "store"))?; + + Ok(( + StatusCode::CREATED, + Json(SubmitResp { + id: row.id, + submission_digest: row.submission_digest, + state: row.state, + judge_backend: eval.backend, + holdout_cells: eval.holdout_cells, + eligible, + }), + )) +} + +async fn list_subs(State(st): State) -> impl IntoResponse { + let rows = st.store.list().unwrap_or_default(); + Json(serde_json::json!({ "items": rows })) +} + +async fn get_sub( + State(st): State, + Path(id): Path, +) -> Result)> { + let row = st + .store + .get(&id) + .map_err(|_| err(StatusCode::NOT_FOUND, "not_found"))?; + Ok(Json(row)) +} + +#[derive(Debug, Deserialize)] +struct PromoteBody { + submission_id: String, +} + +async fn promote( + State(st): State, + headers: HeaderMap, + Json(body): Json, +) -> Result)> { + if st.admin_hashes.is_empty() { + return Err(err(StatusCode::SERVICE_UNAVAILABLE, "auth_unconfigured")); + } + if !admin_ok(&headers, &st.admin_hashes) { + return Err(err(StatusCode::UNAUTHORIZED, "unauthorized")); + } + let row = st.store.promote(&body.submission_id).map_err(|e| { + let code = if e.to_string().contains("unknown") { + StatusCode::NOT_FOUND + } else { + StatusCode::CONFLICT + }; + err(code, &e.to_string()) + })?; + Ok(Json(row)) +} + +fn admin_ok(headers: &HeaderMap, hashes: &[String]) -> bool { + let Some(raw) = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + else { + return false; + }; + let token = raw.strip_prefix("Bearer ").unwrap_or(raw).trim(); + if token.is_empty() { + return false; + } + let mut h = Sha256::new(); + h.update(token.as_bytes()); + let got = hex::encode(h.finalize()); + hashes.iter().any(|x| x == &got) +} + +fn err(code: StatusCode, msg: &str) -> (StatusCode, Json) { + (code, Json(serde_json::json!({ "error": msg }))) +} + +/// Hash an admin token the same way the server does. +#[must_use] +pub fn hash_admin_token(token: &str) -> String { + let mut h = Sha256::new(); + h.update(token.as_bytes()); + hex::encode(h.finalize()) +} + +/// Manifest a well-formed submission declares (pinned base + license). +#[must_use] +pub fn pinned_manifest() -> ArtifactManifest { + ArtifactManifest { + base: BASE_MODEL_ID.into(), + base_license: BASE_MODEL_LICENSE.into(), + ..ArtifactManifest::default() + } +} + +/// Documented license URL for the pinned base. +#[must_use] +pub const fn base_license_url() -> &'static str { + BASE_MODEL_LICENSE_URL +} + +/// Judge model id served on `/v1/status`. +#[must_use] +pub const fn judge_model() -> &'static str { + JUDGE_MODEL_ID +} + +#[cfg(test)] +mod tests { + use axum::body::Body; + use axum::http::Request; + use http_body_util::BodyExt; + use relearn_t2i_eval::base_champion_scores; + use relearn_t2i_task::{frozen_prompt_commitment, FrozenPrompt, PromptPin}; + use tower::ServiceExt; + + use super::*; + + fn prompt(id: u32) -> FrozenPrompt { + FrozenPrompt { + id, + text: format!("prompt {id}"), + upsampled_json: None, + } + } + + fn holdout() -> Vec { + (900..=924).map(prompt).collect() + } + + fn test_pin() -> RelearnT2iPin { + let public: Vec = (1..=25).map(prompt).collect(); + RelearnT2iPin { + prompts: PromptPin { + pin_salt: "cortex-t2i-test".into(), + variations_per_prompt: 4, + public_ids: public.iter().map(|p| p.id).collect(), + holdout_commitment: frozen_prompt_commitment(&holdout()), + holdout_size: 25, + }, + frozen_prompts: public, + ..RelearnT2iPin::default() + } + } + + fn app(token: &str) -> Router { + let pin = test_pin(); + let store = MemoryStore::new(); + store + .set_holdout_commitment(&pin.prompts.holdout_commitment, pin.prompts.holdout_size) + .expect("commit"); + store + .load_holdout(holdout(), &pin.prompts.public_ids) + .expect("load holdout"); + let ids: Vec = holdout().iter().map(|p| p.id).collect(); + store + .set_base_champion(base_champion_scores(&pin, &ids).expect("base")) + .expect("seed base"); + relearn_t2i_router(AppState { + store, + pin, + judge: JudgeConfig::sim(), + admin_hashes: Arc::new(vec![hash_admin_token(token)]), + }) + } + + fn digest(label: &str) -> String { + let mut h = Sha256::new(); + h.update(label.as_bytes()); + hex::encode(h.finalize()) + } + + async fn json_req( + app: Router, + method: &str, + uri: &str, + body: serde_json::Value, + auth: Option<&str>, + ) -> (StatusCode, serde_json::Value) { + let mut b = Request::builder().method(method).uri(uri); + if let Some(a) = auth { + b = b.header(axum::http::header::AUTHORIZATION, format!("Bearer {a}")); + } + let req = b + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("req"); + let resp = app.oneshot(req).await.expect("resp"); + let status = resp.status(); + let bytes = resp.into_body().collect().await.expect("body").to_bytes(); + let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap_or(serde_json::json!({})); + (status, v) + } + + fn submit_body(label: &str, manifest: &serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "miner_hotkey": digest("miner-hotkey"), + "artifact_digest": digest(label), + "manifest": manifest, + }) + } + + fn pinned_manifest_json() -> serde_json::Value { + serde_json::json!({ + "base": BASE_MODEL_ID, + "base_license": BASE_MODEL_LICENSE, + }) + } + + #[tokio::test] + async fn health_and_status_report_the_pins() { + let app = app("op"); + let (st, health) = + json_req(app.clone(), "GET", "/health", serde_json::json!({}), None).await; + assert_eq!(st, StatusCode::OK); + assert_eq!(health["challenge_id"], CHALLENGE_ID); + + let (st, status) = json_req(app, "GET", "/v1/status", serde_json::json!({}), None).await; + assert_eq!(st, StatusCode::OK); + assert_eq!(status["base_model"], BASE_MODEL_ID); + assert_eq!(status["base_license"], "OpenMDW-1.1"); + assert_eq!(status["judge_model"], JUDGE_MODEL_ID); + assert_eq!(status["judge_inference"]["seed"], 42); + assert_eq!(status["judge_inference"]["top_k"], 1); + assert_eq!(status["judge_inference"]["enable_thinking"], true); + assert_eq!(status["sampler"]["num_inference_steps"], 50); + assert_eq!(status["holdout"]["loaded"], true); + assert_eq!(status["holdout"]["size"], 25); + } + + #[tokio::test] + async fn public_prompts_are_published_with_seeds_and_the_holdout_is_not() { + let (st, body) = + json_req(app("op"), "GET", "/v1/prompts", serde_json::json!({}), None).await; + assert_eq!(st, StatusCode::OK); + let cells = body["public"].as_array().expect("cells"); + assert_eq!(cells.len(), 100); + assert!(cells[0]["seed"].as_u64().unwrap_or(0) > 0); + assert_eq!(cells[0]["cell_key"], "p1#v0"); + assert_eq!(body["dataset"], JUDGE_DATASET_ID); + + // Sealed holdout: commitment and size, nothing that identifies a prompt. + let holdout_json = body["holdout"].to_string(); + assert!(holdout_json.contains("commitment")); + for id in 900..=924 { + assert!( + !body["public"] + .to_string() + .contains(&format!("\"prompt_id\":{id}")), + "holdout id {id} leaked into the public split" + ); + } + assert!(!holdout_json.contains("prompt 900")); + } + + #[tokio::test] + async fn submit_eval_promote_happy_path() { + let token = "op-test-token"; + let app = app(token); + let (st, created) = json_req( + app.clone(), + "POST", + "/v1/submissions", + submit_body("miner-strong-finetune", &pinned_manifest_json()), + None, + ) + .await; + assert_eq!(st, StatusCode::CREATED, "{created}"); + assert_eq!(created["judge_backend"], "sim"); + assert_eq!(created["holdout_cells"], 100); + + if created["eligible"] == true { + let id = created["id"].as_str().expect("id"); + let (st, promoted) = json_req( + app, + "POST", + "/v1/admin/promote", + serde_json::json!({ "submission_id": id }), + Some(token), + ) + .await; + assert_eq!(st, StatusCode::OK, "{promoted}"); + assert_eq!(promoted["state"], "champion"); + } + } + + #[tokio::test] + async fn flux_submission_is_rejected_at_the_door() { + let (st, body) = json_req( + app("op"), + "POST", + "/v1/submissions", + submit_body( + "flux-finetune", + &serde_json::json!({ + "base": "black-forest-labs/FLUX.1-dev", + "base_license": "OpenMDW-1.1", + }), + ), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{body}"); + assert!( + body["error"].as_str().unwrap_or("").contains("refused"), + "{body}" + ); + } + + #[tokio::test] + async fn missing_license_attestation_is_rejected() { + let (st, _) = json_req( + app("op"), + "POST", + "/v1/submissions", + submit_body("no-manifest", &serde_json::json!({})), + None, + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn submit_without_a_loaded_holdout_is_unavailable_not_scored() { + let pin = test_pin(); + let store = MemoryStore::new(); + store + .set_holdout_commitment(&pin.prompts.holdout_commitment, pin.prompts.holdout_size) + .expect("commit"); + let app = relearn_t2i_router(AppState { + store, + pin, + judge: JudgeConfig::sim(), + admin_hashes: Arc::new(vec![hash_admin_token("op")]), + }); + let (st, body) = json_req( + app, + "POST", + "/v1/submissions", + submit_body("x", &pinned_manifest_json()), + None, + ) + .await; + assert_eq!(st, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + } + + #[tokio::test] + async fn promote_requires_bearer() { + let (st, _) = json_req( + app("op"), + "POST", + "/v1/admin/promote", + serde_json::json!({ "submission_id": "t2i_0" }), + None, + ) + .await; + assert_eq!(st, StatusCode::UNAUTHORIZED); + } + + #[test] + fn helper_pins_match_the_task_crate() { + assert_eq!(pinned_manifest().base, BASE_MODEL_ID); + assert_eq!(base_license_url(), "https://openmdw.ai/license/1-1/"); + assert_eq!(judge_model(), "Qwen/Qwen-Image-Bench"); + } +} diff --git a/crates/relearn-t2i-judge/Cargo.toml b/crates/relearn-t2i-judge/Cargo.toml new file mode 100644 index 000000000..62a7cc9c1 --- /dev/null +++ b/crates/relearn-t2i-judge/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "relearn-t2i-judge" +description = "Q-Judger wire format: JSON parse, N/A handling, L1 pillar aggregation" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +relearn-t2i-task = { path = "../relearn-t2i-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" + +[lints] +workspace = true diff --git a/crates/relearn-t2i-judge/src/lib.rs b/crates/relearn-t2i-judge/src/lib.rs new file mode 100644 index 000000000..b6dab1a33 --- /dev/null +++ b/crates/relearn-t2i-judge/src/lib.rs @@ -0,0 +1,602 @@ +//! Q-Judger is the judge for Relearn T2I. This crate is its wire format. +//! +//! Q-Judger (`Qwen/Qwen-Image-Bench`, Apache-2.0, fine-tuned from Qwen3.6-27B) +//! is handed a bench prompt plus one generated image and replies with a +//! chain-of-thought preamble followed by a JSON score tree: +//! +//! ```text +//! { "": { "": { "": { "score": 0|1|2|"N/A" } } } } +//! ``` +//! +//! The paper's mapping and aggregation are reproduced exactly: raw `0|1|2` map +//! to `0|60|100`, `N/A` is excluded rather than zeroed, level 3 averages into +//! level 2, level 2 into level 1, and the five level-1 pillars average into the +//! total. Zeroing `N/A` would quietly punish prompts where a pillar does not +//! apply, which is why exclusion is a correctness requirement here. +//! +//! No other judge model is accepted: see [`assert_judge_model`]. + +#![forbid(unsafe_code)] +#![allow(clippy::doc_markdown, clippy::module_name_repetitions)] + +use std::collections::BTreeMap; + +use relearn_t2i_task::{base_matches_pin, L1Dimension, JUDGE_MODEL_ID}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +/// Mapped value for a raw `0` (Fail). +pub const MAPPED_FAIL: f64 = 0.0; + +/// Mapped value for a raw `1` (Pass). +pub const MAPPED_PASS: f64 = 60.0; + +/// Mapped value for a raw `2` (Excel). +pub const MAPPED_EXCEL: f64 = 100.0; + +/// Highest mapped value; used to normalize the paper scale into `0..=1`. +pub const MAPPED_MAX: f64 = MAPPED_EXCEL; + +/// Fixed Q-Judger inference parameters, straight from the model card. +/// +/// These are part of the contract, not tuning knobs: a judge run at a +/// different temperature is not comparable with the champion's recorded run. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct JudgeInference { + /// Judge model id. Always Q-Judger. + pub model: String, + /// Sampling seed. + pub seed: u32, + /// Sampling temperature (greedy). + pub temperature: f64, + /// Top-k (greedy). + pub top_k: u32, + /// Top-p. + pub top_p: f64, + /// Repetition penalty. + pub repetition_penalty: f64, + /// Generation budget for the thinking trace plus JSON. + pub max_new_tokens: u32, + /// Chain-of-thought before the JSON. + pub enable_thinking: bool, + /// Harness batch size. + pub max_batch_size: u32, +} + +impl Default for JudgeInference { + fn default() -> Self { + Self { + model: JUDGE_MODEL_ID.into(), + seed: 42, + temperature: 0.0, + top_k: 1, + top_p: 1.0, + repetition_penalty: 1.05, + max_new_tokens: 4096, + enable_thinking: true, + max_batch_size: 24, + } + } +} + +/// One judge input row (the harness `ID` / `prompt` / `image_path` columns). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct JudgeRequest { + /// Bench prompt id (1..=1000). + #[serde(rename = "ID")] + pub id: u32, + /// Frozen prompt string sent to the generator. + pub prompt: String, + /// Path to the generated image. + pub image_path: String, +} + +/// A raw Q-Judger level-3 verdict. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RawScore { + /// `0` — the criterion is not met. + Fail, + /// `1` — the criterion is met. + Pass, + /// `2` — the criterion is met unusually well. + Excel, + /// `N/A` — the criterion does not apply to this prompt. Excluded, not zero. + NotApplicable, +} + +impl RawScore { + /// Paper-scale value, or `None` for `N/A`. + #[must_use] + pub const fn mapped(self) -> Option { + match self { + Self::Fail => Some(MAPPED_FAIL), + Self::Pass => Some(MAPPED_PASS), + Self::Excel => Some(MAPPED_EXCEL), + Self::NotApplicable => None, + } + } + + fn from_json(v: &serde_json::Value) -> Option { + if let Some(n) = v.as_u64() { + return match n { + 0 => Some(Self::Fail), + 1 => Some(Self::Pass), + 2 => Some(Self::Excel), + _ => None, + }; + } + let s = v.as_str()?.trim(); + if s.eq_ignore_ascii_case("n/a") || s.eq_ignore_ascii_case("na") { + return Some(Self::NotApplicable); + } + match s { + "0" => Some(Self::Fail), + "1" => Some(Self::Pass), + "2" => Some(Self::Excel), + _ => None, + } + } +} + +/// Why a judge reply was refused. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum JudgeError { + /// No JSON object could be located in the reply. + #[error("no JSON object in judge reply")] + NoJson, + /// The JSON did not parse. + #[error("judge JSON parse: {0}")] + Json(String), + /// The tree carried no recognizable L1 pillar. + #[error("judge reply has no recognized L1 pillar")] + NoPillars, + /// Every level-3 item was `N/A`, so there is nothing to score. + #[error("judge reply is entirely N/A")] + AllNotApplicable, + /// A level-3 value was neither `0|1|2` nor `N/A`. + #[error("unparsable score {value:?} at {path}")] + BadScore { + /// Dotted path of the offending item. + path: String, + /// Raw value as text. + value: String, + }, + /// Someone tried to score with a model other than Q-Judger. + #[error("judge must be {expected:?}, got {got:?}")] + WrongJudge { + /// Required judge id. + expected: String, + /// What the caller supplied. + got: String, + }, +} + +/// Refuse anything but Q-Judger as the T2I judge. +/// +/// # Errors +/// +/// [`JudgeError::WrongJudge`] when `model` is not `Qwen/Qwen-Image-Bench`. +pub fn assert_judge_model(model: &str) -> Result<(), JudgeError> { + if base_matches_pin(model, JUDGE_MODEL_ID) { + Ok(()) + } else { + Err(JudgeError::WrongJudge { + expected: JUDGE_MODEL_ID.into(), + got: model.to_owned(), + }) + } +} + +/// Parsed score tree for one image: pillar → level-2 group → level-3 item. +pub type ScoreTree = BTreeMap>>; + +/// Aggregated scores for one image. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ImageScore { + /// Per-pillar averages on the paper scale (`0..=100`). + pub per_l1: BTreeMap, + /// Mean of the present pillars — the paper's total. + pub total: f64, + /// Level-3 items that carried a usable score. + pub scored_items: u32, + /// Level-3 items reported `N/A`. + pub na_items: u32, +} + +impl ImageScore { + /// Fraction of level-3 items the judge declined to score. + #[must_use] + pub fn na_rate(&self) -> f64 { + let total = f64::from(self.scored_items) + f64::from(self.na_items); + if total <= 0.0 { + return 1.0; + } + f64::from(self.na_items) / total + } + + /// Total normalized into `0..=1` for the paired displacement test. + /// + /// The paired test's dead zone is expressed in absolute metric units, so + /// normalizing here makes one dead-zone unit equal one paper point. + #[must_use] + pub fn normalized_total(&self) -> f64 { + self.total / MAPPED_MAX + } + + /// One pillar normalized into `0..=1`. + #[must_use] + pub fn normalized_pillar(&self, dim: L1Dimension) -> Option { + self.per_l1.get(&dim).map(|v| v / MAPPED_MAX) + } +} + +/// Locate the score JSON inside a reply that begins with a thinking trace. +/// +/// Prefers a fenced ```json block, then falls back to the last balanced +/// top-level object — the thinking trace often contains braces of its own, and +/// the score tree is what the model emits last. +#[must_use] +pub fn extract_json_object(raw: &str) -> Option<&str> { + if let Some(found) = fenced_json(raw) { + return Some(found); + } + balanced_objects(raw).last().copied() +} + +fn fenced_json(raw: &str) -> Option<&str> { + for fence in ["```json", "```JSON"] { + if let Some(start) = raw.find(fence) { + let after = &raw[start + fence.len()..]; + let end = after.find("```")?; + let body = after[..end].trim(); + if body.starts_with('{') { + return Some(body); + } + } + } + None +} + +/// Every balanced top-level `{…}` slice, string- and escape-aware. +fn balanced_objects(raw: &str) -> Vec<&str> { + let bytes = raw.as_bytes(); + let mut out = Vec::new(); + let mut depth = 0usize; + let mut start = 0usize; + let mut in_str = false; + let mut escaped = false; + for (i, b) in bytes.iter().enumerate() { + if in_str { + if escaped { + escaped = false; + } else if *b == b'\\' { + escaped = true; + } else if *b == b'"' { + in_str = false; + } + continue; + } + match b { + b'"' => in_str = true, + b'{' => { + if depth == 0 { + start = i; + } + depth = depth.saturating_add(1); + } + b'}' => { + depth = depth.saturating_sub(1); + if depth == 0 && i >= start { + if let Some(slice) = raw.get(start..=i) { + out.push(slice); + } + } + } + _ => {} + } + } + out +} + +/// Parse a Q-Judger reply into a score tree. +/// +/// Accepts either a full tree keyed by the five pillars, or a single-pillar +/// tree when `assume_pillar` names which pillar was requested. +/// +/// # Errors +/// +/// See [`JudgeError`]. Unknown score values are rejected rather than coerced, +/// so a judge that starts emitting a new vocabulary fails loudly. +pub fn parse_reply(raw: &str, assume_pillar: Option) -> Result { + let body = extract_json_object(raw).ok_or(JudgeError::NoJson)?; + let value: serde_json::Value = + serde_json::from_str(body).map_err(|e| JudgeError::Json(e.to_string()))?; + let root = value.as_object().ok_or(JudgeError::NoJson)?; + + let mut tree = ScoreTree::new(); + let looks_like_pillars = root.keys().any(|k| L1Dimension::parse(k).is_some()); + if looks_like_pillars { + for (key, sub) in root { + let Some(dim) = L1Dimension::parse(key) else { + continue; + }; + let groups = parse_groups(dim.as_str(), sub)?; + if !groups.is_empty() { + tree.insert(dim, groups); + } + } + } else if let Some(dim) = assume_pillar { + let groups = parse_groups(dim.as_str(), &value)?; + if !groups.is_empty() { + tree.insert(dim, groups); + } + } + + if tree.is_empty() { + return Err(JudgeError::NoPillars); + } + Ok(tree) +} + +fn parse_groups( + pillar: &str, + value: &serde_json::Value, +) -> Result>, JudgeError> { + let obj = value.as_object().ok_or(JudgeError::NoJson)?; + let mut groups = BTreeMap::new(); + for (l2, l2_val) in obj { + let Some(items) = l2_val.as_object() else { + continue; + }; + let mut leaves = BTreeMap::new(); + for (l3, l3_val) in items { + let raw = l3_val + .get("score") + .or_else(|| l3_val.get("Score")) + .unwrap_or(l3_val); + let parsed = RawScore::from_json(raw).ok_or_else(|| JudgeError::BadScore { + path: format!("{pillar}.{l2}.{l3}"), + value: raw.to_string(), + })?; + leaves.insert(l3.clone(), parsed); + } + if !leaves.is_empty() { + groups.insert(l2.clone(), leaves); + } + } + Ok(groups) +} + +fn mean(values: &[f64]) -> Option { + if values.is_empty() { + return None; + } + #[allow(clippy::cast_precision_loss)] + let n = values.len() as f64; + Some(values.iter().sum::() / n) +} + +/// Aggregate a score tree the way the paper does. +/// +/// Level 3 → level 2 averages only non-`N/A` items; a level-2 group whose items +/// are all `N/A` drops out entirely; level 2 → level 1 averages the surviving +/// groups; the total averages the surviving pillars. +/// +/// # Errors +/// +/// [`JudgeError::AllNotApplicable`] when nothing survived — the caller must +/// treat that as a failed judge run, never as a score of zero. +pub fn aggregate(tree: &ScoreTree) -> Result { + let mut per_l1 = BTreeMap::new(); + let mut scored_items = 0u32; + let mut na_items = 0u32; + + for (dim, groups) in tree { + let mut group_means = Vec::new(); + for leaves in groups.values() { + let mut usable = Vec::new(); + for raw in leaves.values() { + match raw.mapped() { + Some(v) => { + usable.push(v); + scored_items = scored_items.saturating_add(1); + } + None => na_items = na_items.saturating_add(1), + } + } + if let Some(m) = mean(&usable) { + group_means.push(m); + } + } + if let Some(m) = mean(&group_means) { + per_l1.insert(*dim, m); + } + } + + let pillar_means: Vec = per_l1.values().copied().collect(); + let total = mean(&pillar_means).ok_or(JudgeError::AllNotApplicable)?; + Ok(ImageScore { + per_l1, + total, + scored_items, + na_items, + }) +} + +/// Parse and aggregate in one step. +/// +/// # Errors +/// +/// See [`parse_reply`] and [`aggregate`]. +pub fn score_reply( + raw: &str, + assume_pillar: Option, +) -> Result { + aggregate(&parse_reply(raw, assume_pillar)?) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FULL_REPLY: &str = r#" +Let me look at the image. The cube is red {not JSON} and sharp. +```json +{ + "Quality": { + "Realism": {"Physical Logic": {"score": 1}, "Material Texture": {"score": 2}}, + "Detail": {"Noise": {"score": 1}, "Edge Clarity": {"score": 1}, "Naturalness": {"score": 1}}, + "Resolution": {"Resolution": {"score": 2}} + }, + "Aesthetics": { + "Composition": {"Composition": {"score": 2}}, + "Color Harmony": {"Color Harmony": {"score": 1}} + }, + "Alignment": { + "Attributes": {"Color": {"score": 2}, "Quantity": {"score": "N/A"}} + }, + "Real-world Fidelity": { + "Safety & Compliance": {"Safety & Compliance": {"score": 1}} + }, + "Creative Generation": { + "Text Rendering": {"Text Accuracy": {"score": "N/A"}, "Font": {"score": "N/A"}}, + "Imagination": {"Imagination": {"score": 2}} + } +} +``` +"#; + + #[test] + fn only_q_judger_may_judge() { + assert_judge_model("Qwen/Qwen-Image-Bench").expect("pinned judge"); + assert_judge_model("qwen/qwen-image-bench").expect("case-insensitive"); + let err = assert_judge_model("gpt-4o").expect_err("must refuse"); + assert!(matches!(err, JudgeError::WrongJudge { .. })); + } + + #[test] + fn inference_params_match_the_card() { + let p = JudgeInference::default(); + assert_eq!(p.model, "Qwen/Qwen-Image-Bench"); + assert_eq!(p.seed, 42); + assert!(p.temperature.abs() < f64::EPSILON); + assert_eq!(p.top_k, 1); + assert!((p.top_p - 1.0).abs() < f64::EPSILON); + assert!((p.repetition_penalty - 1.05).abs() < 1e-12); + assert_eq!(p.max_new_tokens, 4096); + assert!(p.enable_thinking); + assert_eq!(p.max_batch_size, 24); + } + + #[test] + fn raw_scores_map_to_paper_scale() { + assert_eq!(RawScore::Fail.mapped(), Some(0.0)); + assert_eq!(RawScore::Pass.mapped(), Some(60.0)); + assert_eq!(RawScore::Excel.mapped(), Some(100.0)); + assert_eq!(RawScore::NotApplicable.mapped(), None); + } + + #[test] + fn parses_thinking_preamble_then_json() { + let tree = parse_reply(FULL_REPLY, None).expect("parse"); + assert_eq!(tree.len(), 5); + let quality = &tree[&L1Dimension::Quality]; + assert_eq!(quality["Resolution"]["Resolution"], RawScore::Excel); + assert_eq!( + tree[&L1Dimension::Alignment]["Attributes"]["Quantity"], + RawScore::NotApplicable + ); + } + + #[test] + fn aggregation_follows_the_paper() { + let score = score_reply(FULL_REPLY, None).expect("score"); + // Quality: Realism (60+100)/2 = 80, Detail 60, Resolution 100 → 80. + assert!((score.per_l1[&L1Dimension::Quality] - 80.0).abs() < 1e-9); + // Aesthetics: Composition 100, Color Harmony 60 → 80. + assert!((score.per_l1[&L1Dimension::Aesthetics] - 80.0).abs() < 1e-9); + // Alignment: Attributes averages only Color (100); Quantity is N/A. + assert!((score.per_l1[&L1Dimension::Alignment] - 100.0).abs() < 1e-9); + // Creative Generation: Text Rendering is entirely N/A and drops out, + // leaving Imagination = 100. + assert!((score.per_l1[&L1Dimension::CreativeGeneration] - 100.0).abs() < 1e-9); + let expected = (80.0 + 80.0 + 100.0 + 60.0 + 100.0) / 5.0; + assert!((score.total - expected).abs() < 1e-9, "{}", score.total); + } + + #[test] + fn na_is_excluded_not_zeroed() { + let with_na = + r#"{"Quality": {"Detail": {"Noise": {"score": 2}, "Edge Clarity": {"score": "N/A"}}}}"#; + let zeroed = + r#"{"Quality": {"Detail": {"Noise": {"score": 2}, "Edge Clarity": {"score": 0}}}}"#; + let a = score_reply(with_na, None).expect("na"); + let b = score_reply(zeroed, None).expect("zero"); + assert!((a.total - 100.0).abs() < 1e-9); + assert!((b.total - 50.0).abs() < 1e-9); + assert!(a.total > b.total, "N/A must not behave like a zero"); + assert_eq!(a.na_items, 1); + assert_eq!(a.scored_items, 1); + } + + #[test] + fn na_rate_and_normalization() { + let score = score_reply(FULL_REPLY, None).expect("score"); + assert_eq!(score.na_items, 3); + assert!(score.na_rate() > 0.0 && score.na_rate() < 1.0); + assert!((score.normalized_total() - score.total / 100.0).abs() < 1e-12); + assert!(score.normalized_pillar(L1Dimension::Quality).is_some()); + } + + #[test] + fn single_pillar_reply_needs_the_assumed_pillar() { + let body = r#"{"Realism": {"Physical Logic": {"score": 1}}}"#; + assert!(matches!( + parse_reply(body, None), + Err(JudgeError::NoPillars) + )); + let tree = parse_reply(body, Some(L1Dimension::Quality)).expect("assumed pillar"); + assert_eq!(tree.len(), 1); + assert!(tree.contains_key(&L1Dimension::Quality)); + } + + #[test] + fn unknown_score_vocabulary_fails_loudly() { + let body = r#"{"Quality": {"Detail": {"Noise": {"score": "excellent"}}}}"#; + let err = parse_reply(body, None).expect_err("must refuse"); + assert!(matches!(err, JudgeError::BadScore { .. }), "{err:?}"); + } + + #[test] + fn all_na_is_a_failed_run_not_a_zero() { + let body = r#"{"Quality": {"Detail": {"Noise": {"score": "N/A"}}}}"#; + let err = score_reply(body, None).expect_err("must refuse"); + assert_eq!(err, JudgeError::AllNotApplicable); + } + + #[test] + fn missing_json_is_refused() { + assert_eq!( + parse_reply("I could not evaluate this image.", None).expect_err("no json"), + JudgeError::NoJson + ); + } + + #[test] + fn unfenced_reply_uses_the_last_balanced_object() { + let raw = concat!( + "thinking: the layout {looks} fine, and I considered {\"score\": 0} briefly.\n", + "{\"Quality\": {\"Detail\": {\"Noise\": {\"score\": 2}}}}" + ); + let score = score_reply(raw, None).expect("score"); + assert!((score.total - 100.0).abs() < 1e-9); + } + + #[test] + fn bare_leaf_scores_are_accepted() { + let body = r#"{"Quality": {"Detail": {"Noise": 1, "Naturalness": 2}}}"#; + let score = score_reply(body, None).expect("score"); + assert!((score.total - 80.0).abs() < 1e-9); + } +} diff --git a/crates/relearn-t2i-score/Cargo.toml b/crates/relearn-t2i-score/Cargo.toml new file mode 100644 index 000000000..573ce0b6e --- /dev/null +++ b/crates/relearn-t2i-score/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "relearn-t2i-score" +description = "Relearn T2I displacement scoring: pillar gates, seed replay, contamination" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +prism-competition = { path = "../prism-competition" } +relearn-t2i-judge = { path = "../relearn-t2i-judge" } +relearn-t2i-task = { path = "../relearn-t2i-task" } +serde = { version = "1", features = ["derive"] } + +[lints] +workspace = true diff --git a/crates/relearn-t2i-score/src/lib.rs b/crates/relearn-t2i-score/src/lib.rs new file mode 100644 index 000000000..738491f3c --- /dev/null +++ b/crates/relearn-t2i-score/src/lib.rs @@ -0,0 +1,674 @@ +//! Displacement scoring for Relearn T2I. +//! +//! A challenger must beat the champion on the **holdout** prompt split, and it +//! must do so without regressing any single L1 pillar. That second condition is +//! the point of this crate: a large Alignment gain can otherwise hide a Quality +//! collapse, and the total alone would crown it. Every pillar is gated +//! separately with a small tolerance ([`PILLAR_EPSILON`]). +//! +//! The other agentic measures are gates too, not reports: +//! +//! - **Paired A/B** on identical `(prompt_id, seed)` cells — win rate and mean +//! delta, so the comparison is never confounded by sampler luck. +//! - **Seed replay** — a handful of pinned cells are regenerated and compared +//! with the artifact's claimed outputs. Drift means non-determinism or +//! different weights than the ones that were scored. +//! - **Prompt faithfulness** — small agentic spot checks (object counts, +//! rendered text, spatial relations) must agree with Q-Judger's Alignment +//! pillar. Disagreement discards the run rather than trusting one of them. +//! - **Contamination** — eval prompt ids appearing in submitted training +//! metadata reject the submission outright. +//! - **N/A rate** — a judge that declined most items did not produce a score. +//! +//! All series are on the normalized `0..=1` scale (paper points ÷ 100), which +//! makes one `prism_competition` dead-zone unit equal one paper point. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss +)] + +use std::collections::{BTreeMap, BTreeSet}; + +use prism_competition::{ + paired_test, Direction, ExampleSeries, PairedInput, PairedOutcome, PairedRefusal, DEADZONE, +}; +use relearn_t2i_task::{L1Dimension, SCORE_MAX}; +use serde::{Deserialize, Serialize}; + +/// Largest per-pillar drop tolerated versus the champion (normalized units). +/// +/// `0.02` is two paper points: inside judge noise, well below a real collapse. +pub const PILLAR_EPSILON: f64 = 0.02; + +/// Largest share of level-3 items the judge may decline before the run is void. +pub const MAX_NA_RATE: f64 = 0.25; + +/// Largest tolerated embedding drift on a replayed cell (`1 − cosine`). +pub const MAX_REPLAY_DRIFT: f64 = 0.02; + +/// Pinned cells that must be regenerated for the replay check. +pub const REPLAY_CELLS: u32 = 3; + +/// Minimum agentic faithfulness spot checks required for a verdict. +pub const MIN_FAITHFULNESS_CHECKS: u32 = 8; + +/// Minimum agreement between the agentic spot checks and Q-Judger Alignment. +pub const MIN_FAITHFULNESS_AGREEMENT: f64 = 0.75; + +/// Largest tolerated public-minus-holdout gap (overfit / contamination signal). +pub const MAX_PUBLIC_HOLDOUT_GAP: f64 = 0.08; + +/// Minimum head-to-head A/B win rate (bps of decided cells). +pub const MIN_AB_WIN_RATE_BPS: u64 = 5_000; + +/// Slice id bound into the paired test. Both sides must carry it. +pub const HOLDOUT_SLICE_ID: &str = "relearn-t2i-holdout"; + +/// Seed-replay evidence for one artifact. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct ReplayEvidence { + /// Cells regenerated from pinned `(prompt_id, seed)` pairs. + pub cells_checked: u32, + /// Cells whose image hash matched the artifact's claimed output exactly. + pub exact_hash_matches: u32, + /// Worst `1 − cosine` embedding distance across the replayed cells. + pub max_embedding_drift: f64, +} + +impl Default for ReplayEvidence { + fn default() -> Self { + Self { + cells_checked: 0, + exact_hash_matches: 0, + max_embedding_drift: 1.0, + } + } +} + +impl ReplayEvidence { + /// Whether replay cleared the gate. + /// + /// Exact hashes are the fast path. They are not required, because pixel + /// determinism does not survive a driver change, so an embedding distance + /// under [`MAX_REPLAY_DRIFT`] is accepted as the same weights. + #[must_use] + pub fn passes(&self) -> bool { + if self.cells_checked < REPLAY_CELLS { + return false; + } + self.exact_hash_matches >= self.cells_checked + || self.max_embedding_drift <= MAX_REPLAY_DRIFT + } +} + +/// Agentic prompt-faithfulness spot checks (counts, rendered text, relations). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct FaithfulnessEvidence { + /// Spot checks executed. + pub checks: u32, + /// Spot checks that agreed with Q-Judger's Alignment pillar. + pub agreements: u32, +} + +impl FaithfulnessEvidence { + /// Agreement rate, or `0.0` when no checks ran. + #[must_use] + pub fn agreement_rate(&self) -> f64 { + if self.checks == 0 { + return 0.0; + } + f64::from(self.agreements.min(self.checks)) / f64::from(self.checks) + } + + /// Whether faithfulness cleared the gate. + #[must_use] + pub fn passes(&self) -> bool { + self.checks >= MIN_FAITHFULNESS_CHECKS + && self.agreement_rate() + DEADZONE >= MIN_FAITHFULNESS_AGREEMENT + } +} + +/// Per-artifact T2I measurements. Series keys are `p{id}#v{variation}`. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct T2iSliceScores { + /// Normalized Q-Judger totals on the holdout split. + pub holdout: ExampleSeries, + /// Normalized Q-Judger totals on the published split (informational). + pub public: ExampleSeries, + /// Normalized per-pillar series on the holdout split. + pub holdout_by_pillar: BTreeMap, + /// Share of level-3 items the judge declined. + pub na_rate: f64, + /// Seed-replay evidence. + pub replay: ReplayEvidence, + /// Agentic faithfulness evidence. + pub faithfulness: FaithfulnessEvidence, + /// Eval prompt ids found in the submission's training metadata. + pub contaminated_prompt_ids: Vec, +} + +impl T2iSliceScores { + /// Mean of a series, or `None` when empty. + #[must_use] + pub fn mean(series: &ExampleSeries) -> Option { + if series.is_empty() { + return None; + } + let n = series.len() as f64; + Some(series.by_cluster.values().sum::() / n) + } + + /// Mean of one pillar on the holdout split. + #[must_use] + pub fn pillar_mean(&self, dim: L1Dimension) -> Option { + self.holdout_by_pillar.get(&dim).and_then(Self::mean) + } +} + +/// Head-to-head result on identical `(prompt_id, seed)` cells. +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] +pub struct PairedAb { + /// Cells present on both sides. + pub cells: u64, + /// Cells whose difference cleared the dead zone. + pub decided: u64, + /// Decided cells the challenger won. + pub wins: u64, + /// Win rate over decided cells (bps). + pub win_rate_bps: u64, + /// Mean challenger-minus-champion delta over paired cells (normalized). + pub mean_delta: f64, +} + +/// Paired A/B on the same seeds. Cells missing on either side are ignored. +#[must_use] +pub fn paired_ab(champion: &ExampleSeries, challenger: &ExampleSeries) -> PairedAb { + let mut deltas = Vec::new(); + for (cell, champ) in &champion.by_cluster { + if let Some(chal) = challenger.by_cluster.get(cell) { + deltas.push(chal - champ); + } + } + if deltas.is_empty() { + return PairedAb::default(); + } + let decided: Vec = deltas + .iter() + .copied() + .filter(|d| d.abs() >= DEADZONE) + .collect(); + let wins = decided.iter().filter(|d| **d > 0.0).count(); + let win_rate_bps = if decided.is_empty() { + 0 + } else { + ((wins as u128 * 10_000) / decided.len() as u128) as u64 + }; + PairedAb { + cells: deltas.len() as u64, + decided: decided.len() as u64, + wins: wins as u64, + win_rate_bps, + mean_delta: deltas.iter().sum::() / deltas.len() as f64, + } +} + +/// Per-pillar champion / challenger comparison. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct PillarDelta { + /// Champion pillar mean (normalized). + pub champion: f64, + /// Challenger pillar mean (normalized). + pub challenger: f64, + /// `challenger − champion`. + pub delta: f64, +} + +/// Gate that blocked promotion (or would have). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GateFail { + /// Challenger is not a significant paired win on the holdout. + NoPairedWin, + /// Challenger lost or tied the champion (never crown a regression). + Regression, + /// One L1 pillar dropped by more than [`PILLAR_EPSILON`]. + PillarRegression { + /// Pillar that dropped. + dimension: L1Dimension, + /// Size of the drop in bps of the normalized scale. + drop_bps: u64, + }, + /// Head-to-head A/B win rate below [`MIN_AB_WIN_RATE_BPS`]. + AbWinRate, + /// The judge declined too many items for the run to mean anything. + NotApplicableRate, + /// Regenerated cells did not reproduce the artifact's claimed outputs. + SeedReplay, + /// Agentic spot checks disagree with Q-Judger Alignment. + PromptFaithfulness, + /// Eval prompt ids appear in the submission's training metadata. + Contamination, + /// Public split far above holdout (memorization / contamination). + PublicHoldoutGap, + /// Paired test refused (slice mismatch / too thin). + PairedRefusal, +} + +/// Serializable paired-test summary (prism `PairedOutcome` is not serde). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PairedSummary { + /// Overlapping cells. + pub n_paired: u64, + /// Cells outside the dead zone. + pub n_decided: u64, + /// Bootstrap LCB win-rate (bps). + pub win_rate_lcb_bps: u64, + /// Challenger displaces champion. + pub displaces: bool, +} + +impl PairedSummary { + fn from_outcome(o: &PairedOutcome) -> Self { + Self { + n_paired: u64::try_from(o.n_paired).unwrap_or(u64::MAX), + n_decided: u64::try_from(o.n_decided).unwrap_or(u64::MAX), + win_rate_lcb_bps: o.win_rate_lcb_bps, + displaces: o.displaces, + } + } +} + +/// Full promote / reject verdict. Consensus-critical once leaves are signed. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PromoteVerdict { + /// Whether this submission may become champion after operator audit. + pub eligible: bool, + /// Paired-test outcome when the slices lined up. + pub paired: Option, + /// Same-seed head-to-head summary. + pub ab: PairedAb, + /// Per-pillar deltas (the anti-hidden-regression report). + pub pillars: BTreeMap, + /// Gates that failed (empty ⇒ all clear). + pub failed: Vec, + /// Lattice score to emit if this hotkey is the live champion (`0` otherwise). + pub lattice: u64, +} + +/// Eval prompt ids that leaked into a submission's training metadata. +#[must_use] +pub fn contamination( + train_prompt_ids: &BTreeSet, + eval_prompt_ids: &BTreeSet, +) -> Vec { + train_prompt_ids + .intersection(eval_prompt_ids) + .copied() + .collect() +} + +/// Judge challenger vs champion. Never returns `eligible` on a regression. +#[must_use] +pub fn judge_challenger(champion: &T2iSliceScores, challenger: &T2iSliceScores) -> PromoteVerdict { + let mut failed = Vec::new(); + + let paired_raw = match paired_test(&PairedInput { + metric: "relearn_t2i.qjudger_total".into(), + direction: Direction::HigherBetter, + slice_id: HOLDOUT_SLICE_ID.into(), + champion: champion.holdout.clone(), + challenger: challenger.holdout.clone(), + }) { + Ok(o) => Some(o), + Err( + PairedRefusal::NotEnoughDecided + | PairedRefusal::NoOverlap + | PairedRefusal::SliceMismatch, + ) => { + failed.push(GateFail::PairedRefusal); + None + } + }; + + match paired_raw { + Some(ref o) if o.displaces => {} + Some(_) => { + failed.push(GateFail::NoPairedWin); + failed.push(GateFail::Regression); + } + None => failed.push(GateFail::NoPairedWin), + } + + let ab = paired_ab(&champion.holdout, &challenger.holdout); + if ab.decided == 0 || ab.win_rate_bps < MIN_AB_WIN_RATE_BPS { + failed.push(GateFail::AbWinRate); + } + + let pillars = pillar_deltas(champion, challenger); + for (dim, d) in &pillars { + let drop = -d.delta; + if drop > PILLAR_EPSILON + DEADZONE { + failed.push(GateFail::PillarRegression { + dimension: *dim, + drop_bps: (drop * 10_000.0).round().max(0.0) as u64, + }); + } + } + + if challenger.na_rate > MAX_NA_RATE { + failed.push(GateFail::NotApplicableRate); + } + if !challenger.replay.passes() { + failed.push(GateFail::SeedReplay); + } + if !challenger.faithfulness.passes() { + failed.push(GateFail::PromptFaithfulness); + } + if !challenger.contaminated_prompt_ids.is_empty() { + failed.push(GateFail::Contamination); + } + + if let (Some(pub_m), Some(hold_m)) = ( + T2iSliceScores::mean(&challenger.public), + T2iSliceScores::mean(&challenger.holdout), + ) { + if pub_m - hold_m > MAX_PUBLIC_HOLDOUT_GAP + DEADZONE { + failed.push(GateFail::PublicHoldoutGap); + } + } + + failed.sort_by(|a, b| format!("{a:?}").cmp(&format!("{b:?}"))); + failed.dedup(); + + let eligible = failed.is_empty(); + let lattice = if eligible { + paired_raw + .as_ref() + .map_or(0, |o| lattice_from_win_rate(o.win_rate_lcb_bps)) + } else { + 0 + }; + + PromoteVerdict { + eligible, + paired: paired_raw.as_ref().map(PairedSummary::from_outcome), + ab, + pillars, + failed, + lattice, + } +} + +/// Per-pillar deltas for every pillar both sides scored. +#[must_use] +pub fn pillar_deltas( + champion: &T2iSliceScores, + challenger: &T2iSliceScores, +) -> BTreeMap { + let mut out = BTreeMap::new(); + for dim in L1Dimension::ALL { + let (Some(c), Some(x)) = (champion.pillar_mean(dim), challenger.pillar_mean(dim)) else { + continue; + }; + out.insert( + dim, + PillarDelta { + champion: c, + challenger: x, + delta: x - c, + }, + ); + } + out +} + +/// Map bootstrap LCB win-rate (bps) onto the lattice. Champion-hold → 0. +#[must_use] +pub fn lattice_from_win_rate(win_rate_lcb_bps: u64) -> u64 { + let clamped = win_rate_lcb_bps.min(10_000); + u64::from(u32::try_from((u128::from(SCORE_MAX) * u128::from(clamped)) / 10_000).unwrap_or(0)) +} + +/// Champion row always keeps a positive lattice so emission does not burn +/// solely because a challenger was rejected. +#[must_use] +pub fn champion_hold_lattice() -> u64 { + SCORE_MAX / 2 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn series(n: usize, val: f64) -> ExampleSeries { + ExampleSeries::from_pairs((0..n).map(|i| { + ( + relearn_t2i_task::cell_key((i / 4) as u32, (i % 4) as u32), + val, + ) + })) + } + + fn pillars(val: f64) -> BTreeMap { + L1Dimension::ALL + .into_iter() + .map(|d| (d, series(120, val))) + .collect() + } + + fn good_replay() -> ReplayEvidence { + ReplayEvidence { + cells_checked: REPLAY_CELLS, + exact_hash_matches: REPLAY_CELLS, + max_embedding_drift: 0.0, + } + } + + fn good_faith() -> FaithfulnessEvidence { + FaithfulnessEvidence { + checks: MIN_FAITHFULNESS_CHECKS, + agreements: MIN_FAITHFULNESS_CHECKS, + } + } + + fn slice(total: f64) -> T2iSliceScores { + T2iSliceScores { + holdout: series(120, total), + public: series(120, total), + holdout_by_pillar: pillars(total), + na_rate: 0.05, + replay: good_replay(), + faithfulness: good_faith(), + contaminated_prompt_ids: Vec::new(), + } + } + + #[test] + fn clear_win_is_eligible() { + let v = judge_challenger(&slice(0.50), &slice(0.80)); + assert!(v.eligible, "failed={:?}", v.failed); + assert!(v.lattice > 0); + assert_eq!(v.ab.win_rate_bps, 10_000); + assert!(v.ab.mean_delta > 0.0); + assert_eq!(v.pillars.len(), 5); + } + + #[test] + fn never_crowns_regression() { + let v = judge_challenger(&slice(0.80), &slice(0.40)); + assert!(!v.eligible); + assert!(v.failed.contains(&GateFail::Regression)); + assert_eq!(v.lattice, 0); + } + + #[test] + fn pillar_collapse_blocks_even_with_a_higher_total() { + // Alignment jumps, Quality collapses. The total still improves, so only + // the per-pillar gate can catch this. + let champ = slice(0.60); + let mut chal = slice(0.60); + chal.holdout = series(120, 0.80); + chal.holdout_by_pillar + .insert(L1Dimension::Alignment, series(120, 0.95)); + chal.holdout_by_pillar + .insert(L1Dimension::Quality, series(120, 0.20)); + chal.public = series(120, 0.80); + + let v = judge_challenger(&champ, &chal); + assert!(!v.eligible, "pillar collapse must block"); + let hit = v.failed.iter().any(|f| { + matches!( + f, + GateFail::PillarRegression { + dimension: L1Dimension::Quality, + .. + } + ) + }); + assert!( + hit, + "expected Quality pillar regression, got {:?}", + v.failed + ); + assert!(v.pillars[&L1Dimension::Quality].delta < 0.0); + assert!(v.pillars[&L1Dimension::Alignment].delta > 0.0); + } + + #[test] + fn pillar_noise_inside_epsilon_is_tolerated() { + let champ = slice(0.50); + let mut chal = slice(0.80); + chal.holdout_by_pillar + .insert(L1Dimension::Aesthetics, series(120, 0.49)); + let v = judge_challenger(&champ, &chal); + assert!(v.eligible, "failed={:?}", v.failed); + } + + #[test] + fn high_na_rate_voids_the_run() { + let mut chal = slice(0.80); + chal.na_rate = 0.60; + let v = judge_challenger(&slice(0.50), &chal); + assert!(v.failed.contains(&GateFail::NotApplicableRate)); + assert!(!v.eligible); + } + + #[test] + fn replay_drift_blocks() { + let mut chal = slice(0.80); + chal.replay = ReplayEvidence { + cells_checked: REPLAY_CELLS, + exact_hash_matches: 0, + max_embedding_drift: 0.4, + }; + let v = judge_challenger(&slice(0.50), &chal); + assert!(v.failed.contains(&GateFail::SeedReplay)); + + // Same hardware drift, different weights is the case we must catch; + // small embedding distance with no exact hash is still accepted. + let mut ok = slice(0.80); + ok.replay = ReplayEvidence { + cells_checked: REPLAY_CELLS, + exact_hash_matches: 0, + max_embedding_drift: 0.01, + }; + assert!(judge_challenger(&slice(0.50), &ok).eligible); + } + + #[test] + fn missing_replay_cells_block() { + let mut chal = slice(0.80); + chal.replay = ReplayEvidence { + cells_checked: 1, + exact_hash_matches: 1, + max_embedding_drift: 0.0, + }; + assert!(judge_challenger(&slice(0.50), &chal) + .failed + .contains(&GateFail::SeedReplay)); + } + + #[test] + fn faithfulness_disagreement_blocks() { + let mut chal = slice(0.80); + chal.faithfulness = FaithfulnessEvidence { + checks: MIN_FAITHFULNESS_CHECKS, + agreements: 2, + }; + let v = judge_challenger(&slice(0.50), &chal); + assert!(v.failed.contains(&GateFail::PromptFaithfulness)); + } + + #[test] + fn too_few_faithfulness_checks_block() { + let mut chal = slice(0.80); + chal.faithfulness = FaithfulnessEvidence { + checks: 2, + agreements: 2, + }; + assert!(judge_challenger(&slice(0.50), &chal) + .failed + .contains(&GateFail::PromptFaithfulness)); + } + + #[test] + fn contamination_blocks() { + let mut chal = slice(0.80); + chal.contaminated_prompt_ids = vec![902]; + let v = judge_challenger(&slice(0.50), &chal); + assert!(v.failed.contains(&GateFail::Contamination)); + } + + #[test] + fn contamination_detects_overlap() { + let train: BTreeSet = [1, 2, 900].into_iter().collect(); + let eval: BTreeSet = [900, 901].into_iter().collect(); + assert_eq!(contamination(&train, &eval), vec![900]); + assert!(contamination(&BTreeSet::new(), &eval).is_empty()); + } + + #[test] + fn public_far_above_holdout_blocks() { + let mut chal = slice(0.80); + chal.public = series(120, 0.99); + let v = judge_challenger(&slice(0.50), &chal); + assert!(v.failed.contains(&GateFail::PublicHoldoutGap)); + } + + #[test] + fn thin_overlap_refuses_rather_than_promotes() { + let champ = T2iSliceScores { + holdout: series(8, 0.5), + ..slice(0.5) + }; + let chal = T2iSliceScores { + holdout: series(8, 0.9), + ..slice(0.9) + }; + let v = judge_challenger(&champ, &chal); + assert!(!v.eligible); + assert!(v.failed.contains(&GateFail::PairedRefusal)); + } + + #[test] + fn ab_ignores_cells_missing_on_one_side() { + let champ = ExampleSeries::from_pairs([("p1#v0", 0.4), ("p1#v1", 0.4)]); + let chal = ExampleSeries::from_pairs([("p1#v0", 0.9), ("p9#v0", 0.9)]); + let ab = paired_ab(&champ, &chal); + assert_eq!(ab.cells, 1); + assert_eq!(ab.wins, 1); + assert_eq!(ab.win_rate_bps, 10_000); + } + + #[test] + fn lattice_endpoints() { + assert_eq!(lattice_from_win_rate(0), 0); + assert_eq!(lattice_from_win_rate(10_000), SCORE_MAX); + assert_eq!(champion_hold_lattice(), SCORE_MAX / 2); + } +} diff --git a/crates/relearn-t2i-store/Cargo.toml b/crates/relearn-t2i-store/Cargo.toml new file mode 100644 index 000000000..f4f7d6add --- /dev/null +++ b/crates/relearn-t2i-store/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "relearn-t2i-store" +description = "In-memory Relearn T2I submissions, artifact manifests, sealed prompt holdout" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +hex = "0.4" +relearn-t2i-score = { path = "../relearn-t2i-score" } +relearn-t2i-task = { path = "../relearn-t2i-task" } +serde = { version = "1", features = ["derive"] } +sha2 = "0.10" +thiserror = "2" + +[dev-dependencies] +prism-competition = { path = "../prism-competition" } +serde_json = "1" + +[lints] +workspace = true diff --git a/crates/relearn-t2i-store/src/lib.rs b/crates/relearn-t2i-store/src/lib.rs new file mode 100644 index 000000000..3afb774f8 --- /dev/null +++ b/crates/relearn-t2i-store/src/lib.rs @@ -0,0 +1,557 @@ +//! In-memory Relearn T2I store: submissions, artifact manifests, champion, +//! and the sealed holdout prompt set. +//! +//! The holdout records are the secret here, not a random seed: once a miner +//! knows which Qwen-Image-Bench ids are scored, the holdout stops measuring +//! generalization. So the records are loaded once from an operator file, +//! verified against the commitment in `config/relearn-t2i-pin.toml`, and are +//! only readable after a submission digest has been frozen. The public view +//! carries the commitment and the size, never the ids or the prompt text. + +#![forbid(unsafe_code)] +#![allow( + clippy::doc_markdown, + clippy::module_name_repetitions, + clippy::must_use_candidate +)] + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use relearn_t2i_score::{PromoteVerdict, T2iSliceScores}; +use relearn_t2i_task::{ + verify_holdout_prompts, FrozenPrompt, HoldoutError, SamplerConfig, HOLDOUT_DOMAIN, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +/// Submission lifecycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SubmissionState { + /// Digest accepted; holdout still sealed. + Accepted, + /// Digest frozen; holdout unsealed; eval running. + Evaluating, + /// Eval finished; waiting operator audit. + AwaitingAdmin, + /// Rejected (regression / gates / integrity / license). + Rejected, + /// Operator-promoted champion. + Champion, +} + +/// What a miner declares about the artifact they submitted. +/// +/// `base` and `base_license` are the license attestation: the artifact must be +/// a fine-tune of the pinned Cosmos3 checkpoint, inheriting OpenMDW 1.1. +/// `train_prompt_ids` is what the contamination gate reads. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ArtifactManifest { + /// Declared base checkpoint. + pub base: String, + /// Declared license inherited from the base. + pub base_license: String, + /// Declared sampler / dtype used to produce the claimed outputs. + #[serde(default)] + pub sampler: SamplerConfig, + /// Bench prompt ids present in the submitted training metadata. + #[serde(default)] + pub train_prompt_ids: Vec, + /// `cell_key` → sha256 hex of the image the miner claims that cell produced. + /// The seed-replay gate regenerates a few of these. + #[serde(default)] + pub claimed_output_hashes: BTreeMap, +} + +/// One miner submission. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Submission { + /// Stable id (`t2i_` + 16 hex). + pub id: String, + /// 64-hex miner hotkey. + pub miner_hotkey: String, + /// SHA-256 hex of the miner artifact (weights / adapter). Frozen at accept. + pub artifact_digest: String, + /// Optional locator (HF repo, object URL). + pub artifact_uri: Option, + /// Declared base, license, sampler, and training metadata. + pub manifest: ArtifactManifest, + /// Digest freeze nonce (hex). + pub nonce: String, + /// `sha256(hotkey || 0xff || artifact || 0xff || nonce)`. + pub submission_digest: String, + /// Lifecycle. + pub state: SubmissionState, + /// Eval receipt JSON (if any). + pub receipt_json: Option, + /// Judge verdict (if any). + pub verdict: Option, + /// Reject / gate reason. + pub detail: Option, +} + +/// Public description of the sealed holdout. Carries no prompt ids or text. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HoldoutSeal { + /// Slice id bound into the paired test. + pub slice_id: String, + /// Commitment pinned in `config/relearn-t2i-pin.toml`. + pub commitment: String, + /// Number of holdout prompts. + pub size: usize, + /// Whether the operator has loaded matching records on this host. + pub loaded: bool, +} + +/// Store errors. +#[derive(Debug, Error)] +pub enum StoreError { + /// Lock poisoned. + #[error("store lock poisoned")] + Poison, + /// Unknown submission. + #[error("unknown submission {0}")] + NotFound(String), + /// Illegal state transition. + #[error("illegal state {0}")] + Illegal(String), + /// Operator holdout file did not match the committed digest. + #[error("holdout: {0}")] + Holdout(#[from] HoldoutError), +} + +/// In-memory store (v0). Postgres can replace this without changing the HTTP surface. +#[derive(Clone, Default)] +pub struct MemoryStore { + inner: Arc>, +} + +#[derive(Default)] +struct Inner { + next: u64, + submissions: BTreeMap, + champion_id: Option, + scores: BTreeMap, + champion_scores: Option, + base_champion: Option, + holdout: Option>, + holdout_commitment: String, + holdout_size: usize, +} + +impl MemoryStore { + /// Empty store. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + fn lock(&self) -> Result, StoreError> { + self.inner.lock().map_err(|_| StoreError::Poison) + } + + /// Record the pin's holdout commitment before any records are loaded. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn set_holdout_commitment(&self, commitment: &str, size: usize) -> Result<(), StoreError> { + let mut g = self.lock()?; + g.holdout_commitment = commitment.trim().to_ascii_lowercase(); + g.holdout_size = size; + Ok(()) + } + + /// Load operator-supplied holdout records, verified against the pin. + /// + /// # Errors + /// + /// [`StoreError::Holdout`] when the records do not match the commitment, + /// overlap the public split, or are structurally invalid. Nothing is stored + /// on failure: a host with a bad holdout file scores nothing rather than + /// silently falling back to the public split. + pub fn load_holdout( + &self, + records: Vec, + public_ids: &[u32], + ) -> Result<(), StoreError> { + let (commitment, size) = { + let g = self.lock()?; + (g.holdout_commitment.clone(), g.holdout_size) + }; + verify_holdout_prompts(&records, public_ids, &commitment, size)?; + self.lock()?.holdout = Some(records); + Ok(()) + } + + /// Public seal description (no ids, no text). + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn holdout_seal(&self) -> Result { + let g = self.lock()?; + Ok(HoldoutSeal { + slice_id: relearn_t2i_score::HOLDOUT_SLICE_ID.to_owned(), + commitment: g.holdout_commitment.clone(), + size: g.holdout_size, + loaded: g.holdout.is_some(), + }) + } + + /// Holdout records, readable only after a submission digest is frozen. + /// + /// # Errors + /// + /// [`StoreError::Illegal`] when the digest is empty or no verified records + /// are loaded. + pub fn unseal_holdout(&self, frozen_digest: &str) -> Result, StoreError> { + if frozen_digest.trim().is_empty() { + return Err(StoreError::Illegal( + "holdout stays sealed until the submission digest is frozen".into(), + )); + } + let g = self.lock()?; + g.holdout + .clone() + .ok_or_else(|| StoreError::Illegal("no verified holdout loaded".into())) + } + + /// Insert a newly accepted submission. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn insert(&self, mut row: Submission) -> Result { + let mut g = self.lock()?; + if row.id.is_empty() { + let n = g.next; + g.next = g.next.saturating_add(1); + row.id = format!("t2i_{n:016x}"); + } + g.submissions.insert(row.id.clone(), row.clone()); + Ok(row) + } + + /// Fetch one row. + /// + /// # Errors + /// + /// [`StoreError::NotFound`]. + pub fn get(&self, id: &str) -> Result { + let g = self.lock()?; + g.submissions + .get(id) + .cloned() + .ok_or_else(|| StoreError::NotFound(id.to_owned())) + } + + /// List newest-first. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn list(&self) -> Result, StoreError> { + let g = self.lock()?; + let mut rows: Vec<_> = g.submissions.values().cloned().collect(); + rows.sort_by(|a, b| b.id.cmp(&a.id)); + Ok(rows) + } + + /// Patch state / verdict / receipt. + /// + /// # Errors + /// + /// [`StoreError::NotFound`]. + pub fn patch( + &self, + id: &str, + state: Option, + receipt_json: Option, + verdict: Option, + detail: Option, + ) -> Result { + let mut g = self.lock()?; + let row = g + .submissions + .get_mut(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + if let Some(s) = state { + row.state = s; + } + if let Some(r) = receipt_json { + row.receipt_json = Some(r); + } + if let Some(v) = verdict { + row.verdict = Some(v); + } + if let Some(d) = detail { + row.detail = Some(d); + } + Ok(row.clone()) + } + + /// Current champion submission id. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn champion_id(&self) -> Result, StoreError> { + Ok(self.lock()?.champion_id.clone()) + } + + /// Promote `id` and demote the previous champion. + /// + /// # Errors + /// + /// [`StoreError::Illegal`] unless the row is `awaiting_admin` with an + /// eligible verdict; [`StoreError::NotFound`] for an unknown id. + pub fn promote(&self, id: &str) -> Result { + let mut g = self.lock()?; + let prev = g.champion_id.clone(); + { + let row = g + .submissions + .get(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + if row.state != SubmissionState::AwaitingAdmin { + return Err(StoreError::Illegal(format!( + "promote requires awaiting_admin, got {:?}", + row.state + ))); + } + if !row.verdict.as_ref().is_some_and(|v| v.eligible) { + return Err(StoreError::Illegal( + "promote refused: verdict not eligible (regression or gates)".into(), + )); + } + } + if let Some(p) = prev { + if let Some(old) = g.submissions.get_mut(&p) { + if old.state == SubmissionState::Champion { + old.state = SubmissionState::Rejected; + old.detail = Some("superseded".into()); + } + } + } + { + let row = g + .submissions + .get_mut(id) + .ok_or_else(|| StoreError::NotFound(id.to_owned()))?; + row.state = SubmissionState::Champion; + } + if let Some(s) = g.scores.get(id).cloned() { + g.champion_scores = Some(s); + } + g.champion_id = Some(id.to_owned()); + g.submissions + .get(id) + .cloned() + .ok_or_else(|| StoreError::NotFound(id.to_owned())) + } + + /// Persist challenger slices so a later promote displaces vs this run. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn record_scores(&self, id: &str, scores: T2iSliceScores) -> Result<(), StoreError> { + self.lock()?.scores.insert(id.to_owned(), scores); + Ok(()) + } + + /// Seed / replace the implicit base-model champion scores. + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn set_base_champion(&self, scores: T2iSliceScores) -> Result<(), StoreError> { + self.lock()?.base_champion = Some(scores); + Ok(()) + } + + /// Champion slice scores (promoted miner, else the pinned base). + /// + /// # Errors + /// + /// [`StoreError::Poison`]. + pub fn champion_scores(&self) -> Result, StoreError> { + let g = self.lock()?; + if let Some(s) = &g.champion_scores { + return Ok(Some(s.clone())); + } + Ok(g.base_champion.clone()) + } +} + +/// SHA-256 hex of the frozen submission. +#[must_use] +pub fn freeze_submission_digest(hotkey: &str, artifact_digest: &str, nonce: &str) -> String { + let mut h = Sha256::new(); + h.update(hotkey.as_bytes()); + h.update([0xff]); + h.update(artifact_digest.as_bytes()); + h.update([0xff]); + h.update(nonce.as_bytes()); + hex::encode(h.finalize()) +} + +/// Per-epoch slice id derived from the holdout commitment. +#[must_use] +pub fn holdout_slice_id(epoch: u64, commitment: &str) -> String { + let mut h = Sha256::new(); + h.update(HOLDOUT_DOMAIN); + h.update(epoch.to_le_bytes()); + h.update(commitment.as_bytes()); + format!( + "relearn-t2i-holdout-{epoch}-{}", + hex::encode(&h.finalize()[..4]) + ) +} + +#[cfg(test)] +mod tests { + use prism_competition::ExampleSeries; + use relearn_t2i_task::frozen_prompt_commitment; + + use super::*; + + fn prompts() -> Vec { + (900..=903) + .map(|id| FrozenPrompt { + id, + text: format!("holdout prompt {id}"), + upsampled_json: None, + }) + .collect() + } + + fn seeded_store() -> MemoryStore { + let st = MemoryStore::new(); + let recs = prompts(); + st.set_holdout_commitment(&frozen_prompt_commitment(&recs), recs.len()) + .expect("commit"); + st + } + + fn slice(v: f64) -> T2iSliceScores { + T2iSliceScores { + holdout: ExampleSeries::from_pairs((0..8).map(|i| (format!("p1#v{i}"), v))), + ..T2iSliceScores::default() + } + } + + fn row(state: SubmissionState, verdict: Option) -> Submission { + Submission { + id: String::new(), + miner_hotkey: "00".repeat(32), + artifact_digest: "11".repeat(32), + artifact_uri: None, + manifest: ArtifactManifest::default(), + nonce: "aa".into(), + submission_digest: "bb".repeat(32), + state, + receipt_json: None, + verdict, + detail: None, + } + } + + #[test] + fn digest_stable_and_distinct() { + let a = freeze_submission_digest("aa", "bb", "n1"); + assert_eq!(a, freeze_submission_digest("aa", "bb", "n1")); + assert_ne!(a, freeze_submission_digest("aa", "bb", "n2")); + assert_eq!(a.len(), 64); + } + + #[test] + fn holdout_loads_only_when_it_matches_the_commitment() { + let st = seeded_store(); + let seal = st.holdout_seal().expect("seal"); + assert_eq!(seal.size, 4); + assert!(!seal.loaded); + + let mut tampered = prompts(); + tampered[0].text = "leaked".into(); + assert!(st.load_holdout(tampered, &[1, 2]).is_err()); + assert!(!st.holdout_seal().expect("seal").loaded); + + st.load_holdout(prompts(), &[1, 2]).expect("verified load"); + assert!(st.holdout_seal().expect("seal").loaded); + } + + #[test] + fn seal_never_exposes_prompt_ids() { + let st = seeded_store(); + st.load_holdout(prompts(), &[]).expect("load"); + let seal = st.holdout_seal().expect("seal"); + let json = serde_json::to_string(&seal).expect("json"); + for id in 900..=903 { + assert!( + !json.contains(&format!("{id}")), + "seal leaked id {id}: {json}" + ); + } + assert!(!json.contains("holdout prompt")); + } + + #[test] + fn unseal_requires_a_frozen_digest() { + let st = seeded_store(); + st.load_holdout(prompts(), &[]).expect("load"); + assert!(st.unseal_holdout("").is_err()); + assert!(st.unseal_holdout(" ").is_err()); + assert_eq!(st.unseal_holdout("deadbeef").expect("unseal").len(), 4); + } + + #[test] + fn unseal_without_loaded_records_fails_closed() { + let st = seeded_store(); + assert!(st.unseal_holdout("deadbeef").is_err()); + } + + #[test] + fn promote_refuses_ineligible() { + let st = MemoryStore::new(); + let r = st + .insert(row(SubmissionState::AwaitingAdmin, None)) + .expect("insert"); + assert!(st.promote(&r.id).is_err()); + } + + #[test] + fn champion_scores_follow_promote_not_base() { + let st = MemoryStore::new(); + st.set_base_champion(slice(0.4)).expect("base"); + let r = st + .insert(row( + SubmissionState::AwaitingAdmin, + Some(PromoteVerdict { + eligible: true, + paired: None, + ab: relearn_t2i_score::PairedAb::default(), + pillars: BTreeMap::new(), + failed: Vec::new(), + lattice: 12, + }), + )) + .expect("insert"); + st.record_scores(&r.id, slice(0.8)).expect("scores"); + st.promote(&r.id).expect("promote"); + let got = st.champion_scores().expect("read").expect("some"); + assert!((T2iSliceScores::mean(&got.holdout).unwrap_or(0.0) - 0.8).abs() < 1e-9); + } + + #[test] + fn slice_ids_track_epoch_and_commitment() { + let a = holdout_slice_id(3, "aa"); + assert_eq!(a, holdout_slice_id(3, "aa")); + assert_ne!(a, holdout_slice_id(4, "aa")); + assert_ne!(a, holdout_slice_id(3, "bb")); + } +} diff --git a/crates/relearn-t2i-task/Cargo.toml b/crates/relearn-t2i-task/Cargo.toml new file mode 100644 index 000000000..c30eae998 --- /dev/null +++ b/crates/relearn-t2i-task/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "relearn-t2i-task" +description = "Relearn T2I challenge identity, Cosmos3 base pin, frozen prompts, seed derivation" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +hex = "0.4" +serde = { version = "1", features = ["derive"] } +sha2 = "0.10" +thiserror = "2" +toml = "0.8" + +[lints] +workspace = true diff --git a/crates/relearn-t2i-task/src/lib.rs b/crates/relearn-t2i-task/src/lib.rs new file mode 100644 index 000000000..0f958cc47 --- /dev/null +++ b/crates/relearn-t2i-task/src/lib.rs @@ -0,0 +1,360 @@ +//! Relearn T2I challenge identity, verified model pins, and seed derivation. +//! +//! ```text +//! challenge_id = "relearn-t2i" +//! scoring_version = 1 +//! task_id domain = b"base-relearn-t2i-task-id-v1" +//! receipt domain = b"base-relearn-t2i-receipt-v1" +//! ``` +//! +//! Distinct from `relearn` / `relearn-mm` / `bounty` so leaf digests never +//! collide. Master-centralized eval; miners pay Lium. +//! +//! Two rules in this crate are product rules, not style: +//! +//! 1. The generator seed is the pinned Cosmos3 checkpoint. Flux-family bases +//! are refused outright ([`base_is_rejected`]). +//! 2. Eval prompts are **frozen** in the pin. Miners never supply their own +//! prompt upsampler on the scored split, so two submissions are always +//! compared on identical prompt strings and identical generation seeds +//! ([`derive_generation_seed`]). + +#![forbid(unsafe_code)] +#![allow(clippy::doc_markdown, clippy::module_name_repetitions)] + +mod pin; +mod prompts; + +pub use pin::{PinError, PromptPin, RelearnT2iPin, SamplerConfig, SeedCell, MIN_SCORED_CELLS}; +pub use prompts::{ + frozen_prompt_commitment, verify_holdout_prompts, FrozenPrompt, HoldoutError, PromptSplit, +}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Normative challenge id (trust-root / leaf `challenge_id` string). +pub const CHALLENGE_ID: &str = "relearn-t2i"; + +/// UTF-8 bytes of [`CHALLENGE_ID`]. +pub const CHALLENGE_ID_BYTES: &[u8] = b"relearn-t2i"; + +/// Live `challenge_scoring_version` (Q-Judger displacement + pillar gates). +pub const SCORING_VERSION: u16 = 1; + +/// Domain tag for task id digests. +pub const TASK_ID_DOMAIN: &[u8] = b"base-relearn-t2i-task-id-v1"; + +/// Domain tag for holdout prompt-set commitments. +pub const HOLDOUT_DOMAIN: &[u8] = b"base-relearn-t2i-holdout-v1"; + +/// Domain tag for eval-receipt digests. +pub const RECEIPT_DOMAIN: &[u8] = b"base-relearn-t2i-receipt-v1"; + +/// Domain tag for promotion attestations. +pub const PROMOTE_DOMAIN: &[u8] = b"base-relearn-t2i-promote-v1"; + +/// Domain tag for per-image generation seeds. +pub const SEED_DOMAIN: &[u8] = b"base-relearn-t2i-seed-v1"; + +/// Integer score lattice max (same scale as other challenges). +pub const SCORE_MAX: u64 = 1_000_000; + +/// Pinned generator seed miners fine-tune. +/// +/// Verified 2026-08-30 against : +/// 65B Cosmos3 Super text-to-image, BF16-only, `Cosmos3OmniPipeline` in +/// Diffusers and `vllm serve … --omni` in vLLM-Omni. +pub const BASE_MODEL_ID: &str = "nvidia/Cosmos3-Super-Text2Image"; + +/// License miners inherit from the pinned base. Card wording: the model is +/// "ready for commercial and non-commercial use" under OpenMDW 1.1. +pub const BASE_MODEL_LICENSE: &str = "OpenMDW-1.1"; + +/// Canonical license text for [`BASE_MODEL_LICENSE`]. +pub const BASE_MODEL_LICENSE_URL: &str = "https://openmdw.ai/license/1-1/"; + +/// Q-Judger — the only judge for this challenge. +/// +/// Verified 2026-08-30 against : +/// Apache-2.0, fine-tuned from Qwen3.6-27B, JSON scores over five L1 pillars. +pub const JUDGE_MODEL_ID: &str = "Qwen/Qwen-Image-Bench"; + +/// Base model Q-Judger was fine-tuned from (documentation pin). +pub const JUDGE_BASE_MODEL_ID: &str = "Qwen3.6-27B"; + +/// Benchmark prompt set (Hugging Face dataset id). +pub const JUDGE_DATASET_ID: &str = "Qwen/Qwen-Image-Bench"; + +/// Judge / bench harness source. +pub const JUDGE_GIT_URL: &str = "https://github.com/QwenLM/Qwen-Image-Bench"; + +/// Lowest Qwen-Image-Bench prompt id. +pub const BENCH_PROMPT_ID_MIN: u32 = 1; + +/// Highest Qwen-Image-Bench prompt id. +pub const BENCH_PROMPT_ID_MAX: u32 = 1000; + +/// Public miner / eval-image repo (shared with the text challenge). +pub const RELEARN_GIT_URL: &str = "https://github.com/CortexLM/relearn"; + +/// Base families that may never be the miner seed. +/// +/// Flux is refused as a product decision: its weights are non-commercial, +/// which is incoherent for a subnet that pays for redistributable artifacts. +pub const REJECTED_BASE_SUBSTRINGS: &[&str] = &[ + "flux", + "black-forest-labs", + "blackforestlabs", + "flux.1", + "flux1", +]; + +/// True when `model_id` names a base family this challenge refuses. +#[must_use] +pub fn base_is_rejected(model_id: &str) -> bool { + let lower = model_id.to_ascii_lowercase(); + REJECTED_BASE_SUBSTRINGS.iter().any(|b| lower.contains(b)) +} + +/// True when the declared base is exactly the pinned checkpoint. +/// +/// Comparison is case-insensitive on the Hugging Face repo id and ignores an +/// optional `@revision` suffix; the revision itself is checked against the pin +/// separately so a stale card cannot pass as the pin. +#[must_use] +pub fn base_matches_pin(declared: &str, pinned: &str) -> bool { + let strip = |s: &str| -> String { + s.trim() + .split('@') + .next() + .unwrap_or("") + .trim_matches('/') + .to_ascii_lowercase() + }; + !declared.trim().is_empty() && strip(declared) == strip(pinned) +} + +/// Level-1 pillar of the Qwen-Image-Bench hierarchy. +/// +/// The five pillars are the paper's top level. Order is the paper's order and +/// is part of the wire format: pillar gates are reported per variant name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum L1Dimension { + /// Realism, detail, resolution. + Quality, + /// Composition, color, lighting, anatomy, emotion, style. + Aesthetics, + /// Attributes, actions, layout, relations, scene. + Alignment, + /// Fairness, safety and compliance, world knowledge. + RealWorldFidelity, + /// Imagination, text rendering, design, visual storytelling. + CreativeGeneration, +} + +impl L1Dimension { + /// All five pillars in the paper's order. + pub const ALL: [Self; 5] = [ + Self::Quality, + Self::Aesthetics, + Self::Alignment, + Self::RealWorldFidelity, + Self::CreativeGeneration, + ]; + + /// Card-spelled pillar name (Q-Judger JSON key). + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Quality => "Quality", + Self::Aesthetics => "Aesthetics", + Self::Alignment => "Alignment", + Self::RealWorldFidelity => "Real-world Fidelity", + Self::CreativeGeneration => "Creative Generation", + } + } + + /// Parse a pillar from a Q-Judger JSON key. Tolerant of case, spaces, + /// hyphens, and underscores; unknown keys return `None` (fail closed). + #[must_use] + pub fn parse(key: &str) -> Option { + let norm = |s: &str| -> String { + s.chars() + .filter(char::is_ascii_alphanumeric) + .map(|c| c.to_ascii_lowercase()) + .collect() + }; + let want = norm(key); + Self::ALL.into_iter().find(|d| norm(d.as_str()) == want) + } +} + +/// Deterministic generation seed for one `(prompt_id, variation_index)` cell. +/// +/// Every miner generates the scored split at these seeds, so two artifacts are +/// always compared on the same sampler trajectory. The salt lives in the pin so +/// an operator can rotate the whole seed lattice without changing the formula. +/// +/// The result is masked into positive `i64` range because the Diffusers and +/// vLLM-Omni paths both accept a signed seed. +#[must_use] +pub fn derive_generation_seed(prompt_id: u32, variation_index: u32, pin_salt: &str) -> u64 { + let mut h = Sha256::new(); + h.update(SEED_DOMAIN); + h.update([0xff]); + h.update(pin_salt.as_bytes()); + h.update([0xff]); + h.update(prompt_id.to_le_bytes()); + h.update([0xff]); + h.update(variation_index.to_le_bytes()); + let d = h.finalize(); + let mut eight = [0u8; 8]; + eight.copy_from_slice(&d[..8]); + u64::from_le_bytes(eight) >> 1 +} + +/// Stable cell key for a scored image: `p{prompt_id}#v{variation_index}`. +/// +/// Both sides of a paired comparison key on this string, so the paired test +/// only ever lines up images generated from the same prompt and same seed. +#[must_use] +pub fn cell_key(prompt_id: u32, variation_index: u32) -> String { + format!("p{prompt_id}#v{variation_index}") +} + +/// True when `prompt_id` is inside the published Qwen-Image-Bench range. +#[must_use] +pub const fn is_bench_prompt_id(prompt_id: u32) -> bool { + prompt_id >= BENCH_PROMPT_ID_MIN && prompt_id <= BENCH_PROMPT_ID_MAX +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn challenge_id_is_distinct() { + assert_eq!(CHALLENGE_ID, "relearn-t2i"); + assert_eq!(CHALLENGE_ID_BYTES, b"relearn-t2i"); + for other in ["relearn", "relearn-mm", "bounty", "prism", "design"] { + assert_ne!(CHALLENGE_ID, other); + } + } + + #[test] + fn domain_tags_are_t2i_prefixed_and_unique() { + let tags = [ + TASK_ID_DOMAIN, + HOLDOUT_DOMAIN, + RECEIPT_DOMAIN, + PROMOTE_DOMAIN, + SEED_DOMAIN, + ]; + for t in tags { + let s = std::str::from_utf8(t).unwrap_or(""); + assert!(s.contains("relearn-t2i"), "{s}"); + } + for i in 0..tags.len() { + for j in (i + 1)..tags.len() { + assert_ne!(tags[i], tags[j]); + } + } + } + + #[test] + fn base_pin_is_cosmos3_not_flux() { + assert_eq!(BASE_MODEL_ID, "nvidia/Cosmos3-Super-Text2Image"); + assert_eq!(BASE_MODEL_LICENSE, "OpenMDW-1.1"); + assert!(!base_is_rejected(BASE_MODEL_ID)); + } + + #[test] + fn flux_family_is_rejected() { + for bad in [ + "black-forest-labs/FLUX.1-dev", + "black-forest-labs/FLUX.1-schnell", + "BLACK-FOREST-LABS/flux.1-pro", + "someone/flux1-merged-lora", + "mirror/Flux", + ] { + assert!(base_is_rejected(bad), "{bad} must be rejected"); + } + } + + #[test] + fn base_match_ignores_case_and_revision() { + assert!(base_matches_pin( + "NVIDIA/cosmos3-super-text2image", + BASE_MODEL_ID + )); + assert!(base_matches_pin( + "nvidia/Cosmos3-Super-Text2Image@da579b9", + BASE_MODEL_ID + )); + assert!(!base_matches_pin( + "nvidia/Cosmos3-Super-Image2Video", + BASE_MODEL_ID + )); + assert!(!base_matches_pin("", BASE_MODEL_ID)); + } + + #[test] + fn judge_pin_is_q_judger() { + assert_eq!(JUDGE_MODEL_ID, "Qwen/Qwen-Image-Bench"); + assert_eq!(JUDGE_DATASET_ID, "Qwen/Qwen-Image-Bench"); + assert_eq!(JUDGE_BASE_MODEL_ID, "Qwen3.6-27B"); + } + + #[test] + fn seed_derivation_is_stable_and_separated() { + let a = derive_generation_seed(7, 0, "salt-a"); + assert_eq!(a, derive_generation_seed(7, 0, "salt-a")); + assert_ne!(a, derive_generation_seed(7, 1, "salt-a")); + assert_ne!(a, derive_generation_seed(8, 0, "salt-a")); + assert_ne!(a, derive_generation_seed(7, 0, "salt-b")); + // Positive i64 range so signed-seed backends round-trip it. + assert!(i64::try_from(a).is_ok()); + } + + #[test] + fn seed_derivation_has_known_vector() { + // Frozen so a refactor that changes the preimage fails loudly: + // every miner's images would otherwise silently stop being comparable. + assert_eq!( + derive_generation_seed(1, 0, "cortex-t2i-v0"), + 5_534_307_901_387_864_795 + ); + } + + #[test] + fn cell_keys_are_unique_per_cell() { + assert_eq!(cell_key(12, 3), "p12#v3"); + assert_ne!(cell_key(12, 3), cell_key(123, 3)); + } + + #[test] + fn pillars_round_trip_through_json_keys() { + for d in L1Dimension::ALL { + assert_eq!(L1Dimension::parse(d.as_str()), Some(d)); + } + assert_eq!( + L1Dimension::parse("real_world_fidelity"), + Some(L1Dimension::RealWorldFidelity) + ); + assert_eq!( + L1Dimension::parse("Creative-Generation"), + Some(L1Dimension::CreativeGeneration) + ); + assert_eq!(L1Dimension::parse("Vibes"), None); + } + + #[test] + fn bench_prompt_range_is_one_to_thousand() { + assert!(is_bench_prompt_id(1)); + assert!(is_bench_prompt_id(1000)); + assert!(!is_bench_prompt_id(0)); + assert!(!is_bench_prompt_id(1001)); + } +} diff --git a/crates/relearn-t2i-task/src/pin.rs b/crates/relearn-t2i-task/src/pin.rs new file mode 100644 index 000000000..286ac3488 --- /dev/null +++ b/crates/relearn-t2i-task/src/pin.rs @@ -0,0 +1,515 @@ +//! `config/relearn-t2i-pin.toml`: base checkpoint, judge, sampler, splits. + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::prompts::FrozenPrompt; +use crate::{ + base_is_rejected, base_matches_pin, derive_generation_seed, is_bench_prompt_id, BASE_MODEL_ID, + BASE_MODEL_LICENSE, BASE_MODEL_LICENSE_URL, CHALLENGE_ID, JUDGE_DATASET_ID, JUDGE_GIT_URL, + JUDGE_MODEL_ID, RELEARN_GIT_URL, SCORING_VERSION, +}; + +/// Minimum scored image cells per split. +/// +/// The paired displacement test refuses a verdict below 100 decided examples, +/// so a pin that cannot reach 100 cells can never promote anything. Rejecting +/// it here turns a silent permanent champion-hold into a config error. +pub const MIN_SCORED_CELLS: usize = 100; + +/// Frozen sampler recipe. Defaults are the Cosmos3 card's text-to-image recipe. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct SamplerConfig { + /// Output width in pixels. + pub width: u32, + /// Output height in pixels. + pub height: u32, + /// Denoising steps. + pub num_inference_steps: u32, + /// Classifier-free guidance scale. + pub guidance_scale: f64, + /// Flow-matching shift passed to the scheduler. + pub flow_shift: f64, + /// Negative prompt (empty in the card recipe). + pub negative_prompt: String, + /// Frames per generation; 1 for single-image output. + pub num_frames: u32, + /// Compute dtype. Cosmos3 is tested at BF16 only. + pub dtype: String, + /// Scheduler class name. + pub scheduler: String, +} + +impl Default for SamplerConfig { + fn default() -> Self { + Self { + width: 1024, + height: 1024, + num_inference_steps: 50, + guidance_scale: 4.0, + flow_shift: 3.0, + negative_prompt: String::new(), + num_frames: 1, + dtype: "bfloat16".into(), + scheduler: "UniPCMultistepScheduler".into(), + } + } +} + +/// Split and seed-lattice pins. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct PromptPin { + /// Salt mixed into every generation seed. Rotating it reshuffles the whole + /// lattice without touching the derivation formula. + pub pin_salt: String, + /// Images generated per prompt. + pub variations_per_prompt: u32, + /// Published prompt ids. Miners may train on these. + pub public_ids: Vec, + /// Commitment over the holdout records. The records themselves stay off git. + pub holdout_commitment: String, + /// Expected holdout record count. + pub holdout_size: usize, +} + +impl Default for PromptPin { + fn default() -> Self { + Self { + pin_salt: String::new(), + variations_per_prompt: 4, + public_ids: Vec::new(), + holdout_commitment: String::new(), + holdout_size: 0, + } + } +} + +/// Everything Cortex needs to reproduce a Relearn T2I eval. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct RelearnT2iPin { + /// Must be [`CHALLENGE_ID`]. + pub challenge_id: String, + /// Must be [`SCORING_VERSION`]. + pub scoring_version: u16, + /// Pinned generator checkpoint miners fine-tune. + pub base: String, + /// License miners inherit from `base`. + pub base_license: String, + /// Canonical license text URL. + pub base_license_url: String, + /// Pinned Hugging Face revision of `base` (empty until recorded). + pub base_revision: String, + /// Judge model. Must be Q-Judger. + pub judge_model: String, + /// Bench prompt dataset. + pub judge_dataset: String, + /// Judge / bench harness source. + pub judge_git: String, + /// Eval image reference (no floating tag in prod). + pub eval_image: String, + /// `sha256:…` digest. Empty until the first digest-pinned eval image ships. + pub eval_image_digest: String, + /// Miner-facing repo. + pub relearn_git: String, + /// Frozen sampler recipe. + pub sampler: SamplerConfig, + /// Split and seed pins. + pub prompts: PromptPin, + /// Frozen public-split prompt records. + #[serde(rename = "frozen_prompt")] + pub frozen_prompts: Vec, +} + +impl Default for RelearnT2iPin { + fn default() -> Self { + Self { + challenge_id: CHALLENGE_ID.into(), + scoring_version: SCORING_VERSION, + base: BASE_MODEL_ID.into(), + base_license: BASE_MODEL_LICENSE.into(), + base_license_url: BASE_MODEL_LICENSE_URL.into(), + base_revision: String::new(), + judge_model: JUDGE_MODEL_ID.into(), + judge_dataset: JUDGE_DATASET_ID.into(), + judge_git: JUDGE_GIT_URL.into(), + eval_image: "ghcr.io/cortexlm/relearn-t2i-eval".into(), + eval_image_digest: String::new(), + relearn_git: RELEARN_GIT_URL.into(), + sampler: SamplerConfig::default(), + prompts: PromptPin::default(), + frozen_prompts: Vec::new(), + } + } +} + +/// Why a pin was refused. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum PinError { + /// TOML did not parse. + #[error("parse relearn-t2i pin: {0}")] + Parse(String), + /// Pin declares the wrong challenge id or scoring version. + #[error("pin identity mismatch: {0}")] + Identity(String), + /// The pinned base is a refused family (Flux). + #[error("base {0:?} is a refused family for this challenge")] + RejectedBase(String), + /// The judge is not Q-Judger. + #[error("judge must be {expected:?}, pin says {got:?}")] + JudgeNotQJudger { + /// Required judge id. + expected: String, + /// What the pin declared. + got: String, + }, + /// `public_ids` and the frozen public records disagree. + #[error("public_ids and frozen_prompt records disagree")] + PublicSplitMismatch, + /// A pinned prompt id is outside the bench range. + #[error("prompt id {0} outside Qwen-Image-Bench range 1..=1000")] + PromptIdOutOfRange(u32), + /// The seed salt is empty, so the lattice is not pinned. + #[error("prompts.pin_salt must not be empty")] + EmptySalt, + /// A split cannot reach the paired test's evidence floor. + #[error("{split} yields {cells} scored cells, below the {min} floor")] + TooFewCells { + /// `public` or `holdout`. + split: String, + /// Cells the pin would produce. + cells: usize, + /// Required floor. + min: usize, + }, + /// Holdout commitment is not a 64-hex digest. + #[error("prompts.holdout_commitment must be 64 hex chars")] + BadHoldoutCommitment, +} + +/// One image cell to generate: prompt, variation index, and frozen seed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SeedCell { + /// Bench prompt id. + pub prompt_id: u32, + /// Variation index within the prompt. + pub variation_index: u32, + /// Frozen generation seed. + pub seed: u64, +} + +impl RelearnT2iPin { + /// Parse and validate `config/relearn-t2i-pin.toml`. + /// + /// # Errors + /// + /// [`PinError::Parse`] on malformed TOML, otherwise any [`validate`] + /// failure. + /// + /// [`validate`]: Self::validate + pub fn from_toml(body: &str) -> Result { + let pin: Self = toml::from_str(body).map_err(|e| PinError::Parse(e.to_string()))?; + pin.validate()?; + Ok(pin) + } + + /// Enforce every product rule the pin is responsible for. + /// + /// # Errors + /// + /// See [`PinError`]. Notably a Flux-family base and a non-Q-Judger judge + /// are both hard refusals, not warnings. + pub fn validate(&self) -> Result<(), PinError> { + if self.challenge_id != CHALLENGE_ID { + return Err(PinError::Identity(format!( + "challenge_id must be {CHALLENGE_ID:?}, got {:?}", + self.challenge_id + ))); + } + if self.scoring_version != SCORING_VERSION { + return Err(PinError::Identity(format!( + "scoring_version must be {SCORING_VERSION}, got {}", + self.scoring_version + ))); + } + if base_is_rejected(&self.base) { + return Err(PinError::RejectedBase(self.base.clone())); + } + if !base_matches_pin(&self.base, BASE_MODEL_ID) { + return Err(PinError::Identity(format!( + "base must be {BASE_MODEL_ID:?}, got {:?}", + self.base + ))); + } + if !base_matches_pin(&self.judge_model, JUDGE_MODEL_ID) { + return Err(PinError::JudgeNotQJudger { + expected: JUDGE_MODEL_ID.into(), + got: self.judge_model.clone(), + }); + } + if self.prompts.pin_salt.trim().is_empty() { + return Err(PinError::EmptySalt); + } + for id in &self.prompts.public_ids { + if !is_bench_prompt_id(*id) { + return Err(PinError::PromptIdOutOfRange(*id)); + } + } + let mut frozen: Vec = self.frozen_prompts.iter().map(|p| p.id).collect(); + frozen.sort_unstable(); + let mut declared = self.prompts.public_ids.clone(); + declared.sort_unstable(); + if frozen != declared { + return Err(PinError::PublicSplitMismatch); + } + let commitment = self.prompts.holdout_commitment.trim(); + if commitment.len() != 64 || !commitment.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(PinError::BadHoldoutCommitment); + } + self.check_cells("public", declared.len())?; + self.check_cells("holdout", self.prompts.holdout_size)?; + Ok(()) + } + + fn check_cells(&self, split: &str, prompt_count: usize) -> Result<(), PinError> { + let cells = prompt_count.saturating_mul(self.prompts.variations_per_prompt as usize); + if cells < MIN_SCORED_CELLS { + return Err(PinError::TooFewCells { + split: split.to_owned(), + cells, + min: MIN_SCORED_CELLS, + }); + } + Ok(()) + } + + /// True when a live rent is allowed (real digest pin present). + #[must_use] + pub fn can_rent(&self) -> bool { + self.eval_image_digest.starts_with("sha256:") && self.eval_image_digest.len() >= 71 + } + + /// Frozen seed for one cell. + #[must_use] + pub fn seed_for(&self, prompt_id: u32, variation_index: u32) -> u64 { + derive_generation_seed(prompt_id, variation_index, &self.prompts.pin_salt) + } + + /// Every cell to generate for `prompt_ids`, in deterministic order. + #[must_use] + pub fn seed_cells(&self, prompt_ids: &[u32]) -> Vec { + let mut ids = prompt_ids.to_vec(); + ids.sort_unstable(); + ids.dedup(); + let mut out = Vec::with_capacity(ids.len() * self.prompts.variations_per_prompt as usize); + for id in ids { + for v in 0..self.prompts.variations_per_prompt { + out.push(SeedCell { + prompt_id: id, + variation_index: v, + seed: self.seed_for(id, v), + }); + } + } + out + } + + /// Check a miner's declared base and license against the pin. + /// + /// This is the artifact-side license attestation: a submission must say it + /// fine-tuned the pinned Cosmos3 checkpoint under OpenMDW 1.1. + /// + /// # Errors + /// + /// [`PinError::RejectedBase`] for a Flux-family declaration, + /// [`PinError::Identity`] for any other mismatch. + pub fn attest_artifact_base( + &self, + declared_base: &str, + declared_license: &str, + ) -> Result<(), PinError> { + if base_is_rejected(declared_base) { + return Err(PinError::RejectedBase(declared_base.to_owned())); + } + if !base_matches_pin(declared_base, &self.base) { + return Err(PinError::Identity(format!( + "artifact base must be {:?}, got {declared_base:?}", + self.base + ))); + } + let norm = |s: &str| s.trim().to_ascii_lowercase().replace([' ', '_'], "-"); + if norm(declared_license) != norm(&self.base_license) { + return Err(PinError::Identity(format!( + "artifact license must be {:?}, got {declared_license:?}", + self.base_license + ))); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::fmt::Write as _; + + use super::*; + + fn pin_body(base: &str, judge: &str) -> String { + let mut body = format!( + r#" +challenge_id = "relearn-t2i" +scoring_version = 1 +base = "{base}" +base_license = "OpenMDW-1.1" +judge_model = "{judge}" + +[prompts] +pin_salt = "cortex-t2i-v0" +variations_per_prompt = 4 +public_ids = [{ids}] +holdout_commitment = "{c}" +holdout_size = 25 +"#, + ids = (1..=25) + .map(|i| i.to_string()) + .collect::>() + .join(", "), + c = "ab".repeat(32), + ); + for i in 1..=25 { + let _ = write!( + body, + "\n[[frozen_prompt]]\nid = {i}\ntext = \"frozen prompt {i}\"\n" + ); + } + body + } + + #[test] + fn default_pin_is_cosmos3_and_q_judger() { + let p = RelearnT2iPin::default(); + assert_eq!(p.base, "nvidia/Cosmos3-Super-Text2Image"); + assert_eq!(p.judge_model, "Qwen/Qwen-Image-Bench"); + assert_eq!(p.base_license, "OpenMDW-1.1"); + assert!(!p.can_rent()); + } + + #[test] + fn sampler_defaults_follow_the_card_recipe() { + let s = SamplerConfig::default(); + assert_eq!((s.width, s.height), (1024, 1024)); + assert_eq!(s.num_inference_steps, 50); + assert!((s.guidance_scale - 4.0).abs() < f64::EPSILON); + assert!((s.flow_shift - 3.0).abs() < f64::EPSILON); + assert_eq!(s.num_frames, 1); + assert_eq!(s.dtype, "bfloat16"); + } + + #[test] + fn valid_pin_parses() { + let p = RelearnT2iPin::from_toml(&pin_body( + "nvidia/Cosmos3-Super-Text2Image", + "Qwen/Qwen-Image-Bench", + )) + .expect("pin parses"); + assert_eq!(p.frozen_prompts.len(), 25); + assert_eq!(p.seed_cells(&p.prompts.public_ids).len(), 100); + } + + #[test] + fn flux_base_is_refused_by_the_pin() { + let err = RelearnT2iPin::from_toml(&pin_body( + "black-forest-labs/FLUX.1-dev", + "Qwen/Qwen-Image-Bench", + )) + .expect_err("flux must not pin"); + assert!(matches!(err, PinError::RejectedBase(_)), "{err:?}"); + } + + #[test] + fn swapping_the_judge_is_refused() { + let err = RelearnT2iPin::from_toml(&pin_body( + "nvidia/Cosmos3-Super-Text2Image", + "openai/some-vlm", + )) + .expect_err("only Q-Judger judges"); + assert!(matches!(err, PinError::JudgeNotQJudger { .. }), "{err:?}"); + } + + #[test] + fn thin_split_is_refused_instead_of_holding_forever() { + let mut p = RelearnT2iPin::from_toml(&pin_body( + "nvidia/Cosmos3-Super-Text2Image", + "Qwen/Qwen-Image-Bench", + )) + .expect("pin"); + p.prompts.variations_per_prompt = 1; + let err = p.validate().expect_err("too few cells"); + assert!(matches!(err, PinError::TooFewCells { .. }), "{err:?}"); + } + + #[test] + fn public_ids_must_match_frozen_records() { + let mut p = RelearnT2iPin::from_toml(&pin_body( + "nvidia/Cosmos3-Super-Text2Image", + "Qwen/Qwen-Image-Bench", + )) + .expect("pin"); + p.prompts.public_ids.push(999); + assert_eq!(p.validate(), Err(PinError::PublicSplitMismatch)); + } + + #[test] + fn empty_salt_is_refused() { + let mut p = RelearnT2iPin::from_toml(&pin_body( + "nvidia/Cosmos3-Super-Text2Image", + "Qwen/Qwen-Image-Bench", + )) + .expect("pin"); + p.prompts.pin_salt = " ".into(); + assert_eq!(p.validate(), Err(PinError::EmptySalt)); + } + + #[test] + fn seed_cells_are_deterministic_and_deduped() { + let p = RelearnT2iPin::from_toml(&pin_body( + "nvidia/Cosmos3-Super-Text2Image", + "Qwen/Qwen-Image-Bench", + )) + .expect("pin"); + let a = p.seed_cells(&[3, 1, 3]); + let b = p.seed_cells(&[1, 3]); + assert_eq!(a, b); + assert_eq!(a.len(), 8); + assert_eq!(a[0].seed, p.seed_for(1, 0)); + } + + #[test] + fn artifact_attestation_rejects_flux_and_wrong_license() { + let p = RelearnT2iPin::default(); + p.attest_artifact_base("nvidia/Cosmos3-Super-Text2Image", "OpenMDW 1.1") + .expect("pinned base + license"); + assert!(matches!( + p.attest_artifact_base("black-forest-labs/FLUX.1-dev", "OpenMDW-1.1"), + Err(PinError::RejectedBase(_)) + )); + assert!(matches!( + p.attest_artifact_base("nvidia/Cosmos3-Super-Text2Image", "cc-by-nc-4.0"), + Err(PinError::Identity(_)) + )); + assert!(matches!( + p.attest_artifact_base("stabilityai/sd-3.5", "OpenMDW-1.1"), + Err(PinError::Identity(_)) + )); + } + + #[test] + fn digest_pin_gates_live_rent() { + let mut p = RelearnT2iPin::default(); + assert!(!p.can_rent()); + p.eval_image_digest = format!("sha256:{}", "00".repeat(32)); + assert!(p.can_rent()); + } +} diff --git a/crates/relearn-t2i-task/src/prompts.rs b/crates/relearn-t2i-task/src/prompts.rs new file mode 100644 index 000000000..40819d19f --- /dev/null +++ b/crates/relearn-t2i-task/src/prompts.rs @@ -0,0 +1,320 @@ +//! Frozen eval prompts and the holdout commitment. +//! +//! NVIDIA recommends upsampling a short prompt into a JSON structure before +//! handing it to Cosmos3. That is fine for a miner's own training, and fatal +//! for a benchmark: two miners with two upsamplers are no longer being scored +//! on the same prompt. So the scored prompts are frozen — the exact string (or +//! the exact upsampled JSON document) is stored once and replayed verbatim. +//! +//! The public split ships in `config/relearn-t2i-pin.toml`. The holdout split +//! must not, because that file is public: git carries only a commitment, and +//! the operator supplies the records out of band. A mismatch fails closed. + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +use crate::{is_bench_prompt_id, HOLDOUT_DOMAIN}; + +/// One frozen eval prompt, addressed by Qwen-Image-Bench id. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FrozenPrompt { + /// Qwen-Image-Bench prompt id (1..=1000). + pub id: u32, + /// Original bench prompt text. + pub text: String, + /// Frozen upsampled JSON document, when the pin uses one. Serialized + /// exactly as it will be sent to the generator. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upsampled_json: Option, +} + +impl FrozenPrompt { + /// The exact string handed to the generator for this cell. + /// + /// Callers must not re-run an upsampler on this value. + #[must_use] + pub fn generator_input(&self) -> &str { + self.upsampled_json.as_deref().unwrap_or(&self.text) + } +} + +/// Why a frozen prompt set was refused. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum HoldoutError { + /// The record set is empty; there is nothing to score. + #[error("frozen prompt set is empty")] + Empty, + /// Two records claim the same bench id. + #[error("duplicate prompt id {0}")] + DuplicateId(u32), + /// A record is outside the published bench id range. + #[error("prompt id {0} outside Qwen-Image-Bench range 1..=1000")] + OutOfRange(u32), + /// A record has no prompt body at all. + #[error("prompt id {0} has empty text")] + EmptyText(u32), + /// The supplied holdout does not match the committed digest. + #[error("holdout commitment mismatch (expected {expected}, got {got})")] + CommitmentMismatch { + /// Digest pinned in git. + expected: String, + /// Digest of what the operator supplied. + got: String, + }, + /// Holdout size disagrees with the pin. + #[error("holdout size mismatch (expected {expected}, got {got})")] + SizeMismatch { + /// Count pinned in git. + expected: usize, + /// Count the operator supplied. + got: usize, + }, + /// A holdout id is also in the published public split. + #[error("holdout prompt id {0} is also in the public split")] + OverlapsPublic(u32), +} + +/// Commitment over a frozen prompt set. +/// +/// Domain-separated, id-sorted, and length-prefixed so neither reordering nor +/// splicing two prompt bodies together can collide. +#[must_use] +pub fn frozen_prompt_commitment(records: &[FrozenPrompt]) -> String { + let mut sorted: Vec<&FrozenPrompt> = records.iter().collect(); + sorted.sort_by_key(|r| r.id); + let mut h = Sha256::new(); + h.update(HOLDOUT_DOMAIN); + h.update([0xff]); + h.update( + u64::try_from(sorted.len()) + .unwrap_or(u64::MAX) + .to_le_bytes(), + ); + for r in sorted { + h.update(r.id.to_le_bytes()); + for field in [r.text.as_str(), r.upsampled_json.as_deref().unwrap_or("")] { + h.update(u64::try_from(field.len()).unwrap_or(u64::MAX).to_le_bytes()); + h.update(field.as_bytes()); + } + } + hex::encode(h.finalize()) +} + +fn validate_records(records: &[FrozenPrompt]) -> Result<(), HoldoutError> { + if records.is_empty() { + return Err(HoldoutError::Empty); + } + let mut seen = std::collections::BTreeSet::new(); + for r in records { + if !is_bench_prompt_id(r.id) { + return Err(HoldoutError::OutOfRange(r.id)); + } + if r.generator_input().trim().is_empty() { + return Err(HoldoutError::EmptyText(r.id)); + } + if !seen.insert(r.id) { + return Err(HoldoutError::DuplicateId(r.id)); + } + } + Ok(()) +} + +/// Verify an operator-supplied holdout against the committed digest. +/// +/// # Errors +/// +/// Any structural problem, a size disagreement, a public-split overlap, or a +/// commitment mismatch. Every one of those is fail-closed: the caller must +/// refuse to score rather than fall back to the public split. +pub fn verify_holdout_prompts( + records: &[FrozenPrompt], + public_ids: &[u32], + expected_commitment: &str, + expected_size: usize, +) -> Result<(), HoldoutError> { + validate_records(records)?; + if records.len() != expected_size { + return Err(HoldoutError::SizeMismatch { + expected: expected_size, + got: records.len(), + }); + } + for r in records { + if public_ids.contains(&r.id) { + return Err(HoldoutError::OverlapsPublic(r.id)); + } + } + let got = frozen_prompt_commitment(records); + if !got.eq_ignore_ascii_case(expected_commitment.trim()) { + return Err(HoldoutError::CommitmentMismatch { + expected: expected_commitment.trim().to_ascii_lowercase(), + got, + }); + } + Ok(()) +} + +/// A resolved eval split: the published prompts plus the unsealed holdout. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PromptSplit { + /// Published prompts. Miners may train on these. + pub public: Vec, + /// Holdout prompts. Empty until the operator unseals them. + pub holdout: Vec, +} + +impl PromptSplit { + /// Public split only (holdout still sealed). + #[must_use] + pub fn public_only(public: Vec) -> Self { + Self { + public, + holdout: Vec::new(), + } + } + + /// Whether a holdout has been unsealed for this run. + #[must_use] + pub fn holdout_unsealed(&self) -> bool { + !self.holdout.is_empty() + } + + /// Public prompt ids in ascending order. + #[must_use] + pub fn public_ids(&self) -> Vec { + let mut v: Vec = self.public.iter().map(|p| p.id).collect(); + v.sort_unstable(); + v + } + + /// Holdout prompt ids in ascending order. Never logged or served. + #[must_use] + pub fn holdout_ids(&self) -> Vec { + let mut v: Vec = self.holdout.iter().map(|p| p.id).collect(); + v.sort_unstable(); + v + } + + /// Validate the public split structurally. + /// + /// # Errors + /// + /// See [`HoldoutError`]. + pub fn validate_public(&self) -> Result<(), HoldoutError> { + validate_records(&self.public) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(id: u32, text: &str) -> FrozenPrompt { + FrozenPrompt { + id, + text: text.into(), + upsampled_json: None, + } + } + + fn holdout() -> Vec { + vec![p(900, "a red cube on a wooden table"), p(901, "two cats")] + } + + #[test] + fn commitment_is_order_independent() { + let a = frozen_prompt_commitment(&holdout()); + let mut rev = holdout(); + rev.reverse(); + assert_eq!(a, frozen_prompt_commitment(&rev)); + assert_eq!(a.len(), 64); + } + + #[test] + fn commitment_changes_with_body() { + let a = frozen_prompt_commitment(&holdout()); + let mut edited = holdout(); + edited[0].text.push('.'); + assert_ne!(a, frozen_prompt_commitment(&edited)); + } + + #[test] + fn commitment_is_length_prefixed_against_splicing() { + let ab = frozen_prompt_commitment(&[p(1, "ab"), p(2, "c")]); + let a_bc = frozen_prompt_commitment(&[p(1, "a"), p(2, "bc")]); + assert_ne!(ab, a_bc); + } + + #[test] + fn upsampled_json_is_the_generator_input() { + let mut r = p(5, "a bird"); + assert_eq!(r.generator_input(), "a bird"); + r.upsampled_json = Some("{\"subject\":\"a bird\"}".into()); + assert_eq!(r.generator_input(), "{\"subject\":\"a bird\"}"); + } + + #[test] + fn verify_accepts_the_committed_set() { + let recs = holdout(); + let c = frozen_prompt_commitment(&recs); + verify_holdout_prompts(&recs, &[1, 2, 3], &c, 2).expect("committed holdout verifies"); + } + + #[test] + fn verify_rejects_edited_set() { + let recs = holdout(); + let c = frozen_prompt_commitment(&recs); + let mut edited = recs; + edited[1].text = "three cats".into(); + let err = verify_holdout_prompts(&edited, &[], &c, 2).expect_err("must reject"); + assert!(matches!(err, HoldoutError::CommitmentMismatch { .. })); + } + + #[test] + fn verify_rejects_public_overlap() { + let recs = holdout(); + let c = frozen_prompt_commitment(&recs); + let err = verify_holdout_prompts(&recs, &[901], &c, 2).expect_err("must reject"); + assert_eq!(err, HoldoutError::OverlapsPublic(901)); + } + + #[test] + fn verify_rejects_size_drift() { + let recs = holdout(); + let c = frozen_prompt_commitment(&recs); + let err = verify_holdout_prompts(&recs, &[], &c, 3).expect_err("must reject"); + assert!(matches!(err, HoldoutError::SizeMismatch { .. })); + } + + #[test] + fn verify_rejects_out_of_range_and_duplicates() { + let bad = vec![p(0, "x")]; + assert_eq!( + verify_holdout_prompts(&bad, &[], "00", 1).expect_err("range"), + HoldoutError::OutOfRange(0) + ); + let dup = vec![p(5, "x"), p(5, "y")]; + assert_eq!( + verify_holdout_prompts(&dup, &[], "00", 2).expect_err("dup"), + HoldoutError::DuplicateId(5) + ); + let empty: Vec = Vec::new(); + assert_eq!( + verify_holdout_prompts(&empty, &[], "00", 0).expect_err("empty"), + HoldoutError::Empty + ); + } + + #[test] + fn split_reports_seal_state_and_ids() { + let mut split = PromptSplit::public_only(vec![p(2, "b"), p(1, "a")]); + assert!(!split.holdout_unsealed()); + assert_eq!(split.public_ids(), vec![1, 2]); + assert!(split.holdout_ids().is_empty()); + split.validate_public().expect("public ok"); + split.holdout = holdout(); + assert!(split.holdout_unsealed()); + assert_eq!(split.holdout_ids(), vec![900, 901]); + } +} diff --git a/crates/relearn-t2i-task/tests/committed_pin.rs b/crates/relearn-t2i-task/tests/committed_pin.rs new file mode 100644 index 000000000..c22bbed0f --- /dev/null +++ b/crates/relearn-t2i-task/tests/committed_pin.rs @@ -0,0 +1,116 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::doc_markdown)] + +//! The committed `config/relearn-t2i-pin.toml` must load and satisfy every +//! product rule: Cosmos3 base, OpenMDW 1.1, Q-Judger as the only judge, a +//! frozen public split large enough for the paired test, and a holdout that is +//! present only as a commitment. + +use std::path::{Path, PathBuf}; + +use relearn_t2i_task::{ + base_is_rejected, is_bench_prompt_id, RelearnT2iPin, BASE_MODEL_ID, BASE_MODEL_LICENSE, + JUDGE_MODEL_ID, MIN_SCORED_CELLS, +}; + +fn pin_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../config/relearn-t2i-pin.toml") + .canonicalize() + .expect("pin path") +} + +fn pin() -> RelearnT2iPin { + let body = std::fs::read_to_string(pin_path()).expect("read pin"); + RelearnT2iPin::from_toml(&body).expect("committed pin must validate") +} + +#[test] +fn committed_pin_validates() { + let p = pin(); + assert_eq!(p.base, BASE_MODEL_ID); + assert_eq!(p.base_license, BASE_MODEL_LICENSE); + assert_eq!(p.base_license_url, "https://openmdw.ai/license/1-1/"); + assert_eq!(p.judge_model, JUDGE_MODEL_ID); + assert!(!base_is_rejected(&p.base)); +} + +#[test] +fn committed_pin_freezes_the_card_sampler_recipe() { + let s = pin().sampler; + assert_eq!((s.width, s.height), (1024, 1024)); + assert_eq!(s.num_inference_steps, 50); + assert!((s.guidance_scale - 4.0).abs() < f64::EPSILON); + assert!((s.flow_shift - 3.0).abs() < f64::EPSILON); + assert_eq!(s.num_frames, 1); + assert_eq!(s.dtype, "bfloat16"); +} + +#[test] +fn public_split_is_frozen_and_large_enough() { + let p = pin(); + assert_eq!(p.frozen_prompts.len(), p.prompts.public_ids.len()); + assert!(p.frozen_prompts.len() >= 40); + for record in &p.frozen_prompts { + assert!(is_bench_prompt_id(record.id), "id {} off-bench", record.id); + assert!( + record.generator_input().len() > 40, + "prompt {} looks like a placeholder: {:?}", + record.id, + record.generator_input() + ); + } + let cells = p.seed_cells(&p.prompts.public_ids); + assert!(cells.len() >= MIN_SCORED_CELLS, "{} cells", cells.len()); +} + +#[test] +fn every_public_cell_has_a_distinct_frozen_seed() { + let p = pin(); + let cells = p.seed_cells(&p.prompts.public_ids); + let mut seeds: Vec = cells.iter().map(|c| c.seed).collect(); + let total = seeds.len(); + seeds.sort_unstable(); + seeds.dedup(); + assert_eq!(seeds.len(), total, "seed collision across public cells"); +} + +#[test] +fn holdout_is_committed_but_not_published() { + let p = pin(); + assert_eq!(p.prompts.holdout_commitment.len(), 64); + assert!(p.prompts.holdout_size >= 25); + let cells = p + .prompts + .holdout_size + .saturating_mul(p.prompts.variations_per_prompt as usize); + assert!(cells >= MIN_SCORED_CELLS, "{cells} holdout cells"); + + // No holdout prompt record may appear in git. + let body = std::fs::read_to_string(pin_path()).expect("read pin"); + let public: std::collections::BTreeSet = p.prompts.public_ids.iter().copied().collect(); + for record in &p.frozen_prompts { + assert!(public.contains(&record.id)); + } + assert!(!body.contains("holdout_prompt")); + assert!(!body.contains("holdout_ids")); +} + +#[test] +fn pin_carries_no_endpoint_or_secret() { + let body = std::fs::read_to_string(pin_path()).expect("read pin"); + let lower = body.to_ascii_lowercase(); + for banned in ["api_key", "bearer", "_token", "mnemonic", "https://api."] { + assert!(!lower.contains(banned), "pin mentions {banned:?}"); + } +} + +#[test] +fn flux_never_appears_as_a_pinned_base() { + let body = std::fs::read_to_string(pin_path()).expect("read pin"); + for line in body.lines() { + let t = line.trim(); + if t.starts_with("base =") || t.starts_with("base=") { + assert!(!base_is_rejected(t), "pinned base line is a refused family"); + } + } +} diff --git a/crates/site-api/src/handlers.rs b/crates/site-api/src/handlers.rs index 41bed9058..631048736 100644 --- a/crates/site-api/src/handlers.rs +++ b/crates/site-api/src/handlers.rs @@ -12,22 +12,23 @@ use serde::Deserialize; use serde_json::{json, Value}; use crate::state::SiteState; -use crate::upstream::{self, DESIGN, PRISM}; +use crate::upstream::{self, DESIGN, PRISM, RELEARN, RELEARN_MM, RELEARN_T2I}; use site_data::map::{ activity_from_lives, design_arena_from_dashboard, design_leaderboard, design_submission, enrich_leaderboard_uids, enrich_leaderboard_weights, enrich_submission_uids, - is_prism_champion_submission, leaderboard_matches_query, list_arenas, prism_arena_from_live, + is_prism_champion_submission, leaderboard_matches_query, prism_arena_from_live, prism_bpb_leaderboard, prism_submission, prism_telemetry, prism_window, submission_matches_query, uid_index_from_hotkeys, }; +use site_data::map::{hydrate_arena, relearn_arena_from_live}; use site_prism::{ enrich_leaderboard_row_from_detail_with_zone, enrich_submission_from_detail_with_zone, fill_v21_run_from_live, infer_recipe_era, infer_recipe_era_with_live, map_benchmarks, payload_is_v21_contest, pin_id_from_payload, prism_reference_baselines, prism_submission_detail_with_zone, }; -use site_types::coding_arena; use site_types::page_slice; +use site_types::{coding_arena, relearn_mm_frame, relearn_t2i_frame}; use site_types::{ ArenaSlug, Governance, LandingSummary, MetricsEmission, MetricsPassRate, MetricsPopulation, NetworkMetrics, NetworkStats, RecipeEra, ResultsMatrix, Validator, @@ -118,6 +119,39 @@ async fn fetch_prism_status(st: &SiteState) -> Option { upstream::get_json_opt(st, PRISM, "/v1/status").await } +async fn fetch_relearn_status(st: &SiteState) -> Option { + upstream::get_json_opt(st, RELEARN, "/v1/status").await +} + +async fn fetch_relearn_t2i_status(st: &SiteState) -> Option { + upstream::get_json_opt(st, RELEARN_T2I, "/v1/status").await +} + +async fn fetch_relearn_mm_status(st: &SiteState) -> Option { + upstream::get_json_opt(st, RELEARN_MM, "/v1/status").await +} + +/// Every live arena with trust-root emission shares applied. +/// +/// The three Relearn challenges are fetched together so the emission column +/// sums to the trust root rather than showing one challenge's slice as the +/// whole subnet. +async fn live_arenas(st: &SiteState) -> Vec { + let relearn = fetch_relearn_status(st).await; + let t2i = fetch_relearn_t2i_status(st).await; + let mm = fetch_relearn_mm_status(st).await; + let mut arenas = vec![ + coding_arena(), + relearn_arena_from_live(relearn.as_ref()), + hydrate_arena(relearn_t2i_frame(), t2i.as_ref()), + hydrate_arena(relearn_mm_frame(), mm.as_ref()), + ]; + for arena in &mut arenas { + apply_emission(st, arena); + } + arenas +} + async fn fetch_prism_subs(st: &SiteState, limit: u32) -> Option { upstream::get_json_opt(st, PRISM, &format!("/v1/submissions?limit={limit}")).await } @@ -227,17 +261,16 @@ fn decorate_submissions(st: &SiteState, rows: &mut [crate::Submission]) { async fn network_stats(st: &SiteState) -> NetworkStats { let (block, chain_epoch, validators) = chain_snapshot(st); - let design = fetch_design_dash(st).await; - let prism = fetch_prism_status(st).await; - let prism_subs = fetch_prism_subs(st, 200).await; - let arenas = arenas_with_emission(st, design.as_ref(), prism.as_ref(), prism_subs.as_ref()); + let relearn = fetch_relearn_status(st).await; + let arenas = live_arenas(st).await; let agents: u32 = arenas.iter().map(|a| a.agents).sum(); let tao_price = site_data::price::tao_price_usd(&st.client, &st.tao_price).await; + let arena_count = u32::try_from(arenas.len()).unwrap_or(0); NetworkStats { - epoch: epoch_from_lives(design.as_ref(), prism.as_ref(), chain_epoch), + epoch: epoch_from_lives(None, relearn.as_ref(), chain_epoch), agents, validators: u32::try_from(validators.len()).unwrap_or(0), - arenas: 3, + arenas: arena_count, emission_per_day: 0.0, tao_price, block_height: block, @@ -257,30 +290,15 @@ fn apply_emission(st: &SiteState, arena: &mut site_types::Arena) { site_data::weights::arena_weight(st.trust_root(), st.latest_sealed_bundle(), slug); } -/// Arena list with trust-root emission shares + sealed-vector weights applied. -fn arenas_with_emission( - st: &SiteState, - design: Option<&Value>, - prism: Option<&Value>, - prism_subs: Option<&Value>, -) -> Vec { - let mut arenas = list_arenas(design, prism, prism_subs); - for arena in &mut arenas { - apply_emission(st, arena); - } - arenas -} - async fn get_network(State(st): State) -> impl IntoResponse { Json(network_stats(&st).await) } async fn get_landing(State(st): State) -> impl IntoResponse { let design = fetch_design_dash(&st).await; - let prism = fetch_prism_status(&st).await; let prism_subs = fetch_prism_subs(&st, 200).await; let stats = network_stats(&st).await; - let arenas = arenas_with_emission(&st, design.as_ref(), prism.as_ref(), prism_subs.as_ref()); + let arenas = live_arenas(&st).await; let design_runs = design .as_ref() .and_then(|d| d.get("recent_runs")) @@ -302,15 +320,7 @@ async fn get_landing(State(st): State) -> impl IntoResponse { } async fn get_arenas(State(st): State) -> impl IntoResponse { - let design = fetch_design_dash(&st).await; - let prism = fetch_prism_status(&st).await; - let prism_subs = fetch_prism_subs(&st, 200).await; - Json(arenas_with_emission( - &st, - design.as_ref(), - prism.as_ref(), - prism_subs.as_ref(), - )) + Json(live_arenas(&st).await) } async fn get_arena(State(st): State, Path(slug): Path) -> Response { @@ -325,6 +335,15 @@ async fn get_arena(State(st): State, Path(slug): Path) -> Res let subs = fetch_prism_subs(&st, 200).await; prism_arena_from_live(status.as_ref(), subs.as_ref()) } + ArenaSlug::Relearn => relearn_arena_from_live(fetch_relearn_status(&st).await.as_ref()), + ArenaSlug::RelearnT2i => hydrate_arena( + relearn_t2i_frame(), + fetch_relearn_t2i_status(&st).await.as_ref(), + ), + ArenaSlug::RelearnMm => hydrate_arena( + relearn_mm_frame(), + fetch_relearn_mm_status(&st).await.as_ref(), + ), }; apply_emission(&st, &mut arena); Json(arena).into_response() @@ -576,13 +595,18 @@ async fn get_leaderboard( let page_size = q.page_size.unwrap_or(24); let needle = q.q.as_deref(); match slug { - ArenaSlug::Coding => Json(empty_leaderboard_json(page, page_size)).into_response(), ArenaSlug::Design => { Json(design_leaderboard_json(&st, page, page_size, needle).await).into_response() } ArenaSlug::Prism => { Json(prism_leaderboard_json(&st, page, page_size, needle).await).into_response() } + // Relearn challenges publish a champion, not a leaderboard: every + // non-champion row is an explicit NoScore (D24), so a paged list of + // them would be a page of zeroes. + ArenaSlug::Coding | ArenaSlug::Relearn | ArenaSlug::RelearnT2i | ArenaSlug::RelearnMm => { + Json(empty_leaderboard_json(page, page_size)).into_response() + } } } @@ -599,7 +623,7 @@ async fn get_submissions( let status_filter = q.status.as_deref(); let needle = q.q.as_deref(); match slug { - ArenaSlug::Coding => { + ArenaSlug::Coding | ArenaSlug::Relearn | ArenaSlug::RelearnT2i | ArenaSlug::RelearnMm => { Json(page_slice::(&[], page, page_size)).into_response() } ArenaSlug::Design => { @@ -1003,8 +1027,7 @@ async fn get_metrics( let (block, chain_epoch, validators) = chain_snapshot(&st); let design = fetch_design_dash(&st).await; let prism = fetch_prism_status(&st).await; - let prism_subs = fetch_prism_subs(&st, 200).await; - let arenas = arenas_with_emission(&st, design.as_ref(), prism.as_ref(), prism_subs.as_ref()); + let arenas = live_arenas(&st).await; let epoch = epoch_from_lives(design.as_ref(), prism.as_ref(), chain_epoch); let agents: u32 = arenas.iter().map(|a| a.agents).sum(); let tao_price = site_data::price::tao_price_usd(&st.client, &st.tao_price).await; @@ -1337,6 +1360,30 @@ mod tests { .await; } + /// Coding plus the three Relearn challenges, in trust-root order. + /// + /// All three Relearn arenas are listed even when their backends are down, + /// so the emission column still sums to the trust root instead of showing + /// one challenge's slice as the whole subnet. + fn assert_live_arena_list(v: &Value) { + let slugs: Vec<&str> = v + .as_array() + .map(|rows| { + rows.iter() + .map(|a| a["slug"].as_str().unwrap_or_default()) + .collect() + }) + .unwrap_or_default(); + assert_eq!( + slugs, + vec!["coding", "relearn", "relearn-t2i", "relearn-mm"], + "{v}" + ); + assert_eq!(v[1]["bestScoreLabel"], "DISPLACE"); + assert_eq!(v[2]["name"], "Relearn T2I"); + assert_eq!(v[3]["name"], "Relearn Multimodal"); + } + #[tokio::test] async fn arenas_and_design_submissions_from_mocks() { let (design, prism, st) = setup().await; @@ -1345,16 +1392,7 @@ mod tests { let (s, v) = call(app.clone(), "/v1/site/arenas").await; assert_eq!(s, StatusCode::OK, "{v}"); - assert_eq!(v.as_array().unwrap().len(), 3); - assert_eq!(v[0]["slug"], "coding"); - // List rows without v2.1 markers must not inflate Prism agents / BPB. - assert_eq!(v[2]["slug"], "prism"); - assert_eq!(v[2]["bestScoreLabel"], "BEST G2"); - assert_eq!(v[2]["agents"], 0); - assert_eq!(v[2]["bestScore"], "—"); - assert_eq!(v[1]["bestScore"], "1,300"); - assert_eq!(v[1]["roundId"], 9); - assert_eq!(v[1]["secondsRemaining"], 120); + assert_live_arena_list(&v); let (s, v) = call(app.clone(), "/v1/site/arenas/design/submissions").await; assert_eq!(s, StatusCode::OK, "{v}"); @@ -1715,7 +1753,7 @@ mod tests { }; let st = st.with_weights( Arc::new(ChallengesBody { - challenges: vec![entry("design", 5_000), entry("prism", 5_000)], + challenges: vec![entry("relearn", 10_000)], }), Arc::new(|| None), ); @@ -1723,19 +1761,17 @@ mod tests { let (s, v) = call(app.clone(), "/v1/site/arenas").await; assert_eq!(s, StatusCode::OK, "{v}"); - assert_eq!(v[1]["emissionShare"], 0.5); - assert_eq!(v[2]["emissionShare"], 0.5); + assert_eq!(v[1]["slug"], "relearn"); + assert_eq!(v[1]["emissionShare"], 1.0); // Unsealed: effective weights stay 0. assert_eq!(v[1]["weight"], 0.0); - assert_eq!(v[2]["weight"], 0.0); let (s, v) = call(app.clone(), "/v1/site/weights").await; assert_eq!(s, StatusCode::OK, "{v}"); assert_eq!(v["sealed"], false); assert_eq!(v["burnShare"], 1.0); - assert_eq!(v["emissionShares"][0]["arena"], "design"); - assert_eq!(v["emissionShares"][0]["share"], 0.5); - assert_eq!(v["emissionShares"][1]["arena"], "prism"); + assert_eq!(v["emissionShares"][0]["arena"], "relearn"); + assert_eq!(v["emissionShares"][0]["share"], 1.0); assert!(v["hotkeyWeights"].as_array().unwrap().is_empty()); } diff --git a/crates/site-api/src/upstream.rs b/crates/site-api/src/upstream.rs index 8aaa9bf32..a9704300f 100644 --- a/crates/site-api/src/upstream.rs +++ b/crates/site-api/src/upstream.rs @@ -8,6 +8,12 @@ use crate::state::SiteState; /// Challenge ids registered on the gateway. pub const DESIGN: &str = "design"; pub const PRISM: &str = "prism"; +/// Relearn LLM post-training factory. +pub const RELEARN: &str = "relearn"; +/// Relearn T2I image generation. +pub const RELEARN_T2I: &str = "relearn-t2i"; +/// Relearn Multimodal vision encoder. +pub const RELEARN_MM: &str = "relearn-mm"; /// Upstream fetch error. #[derive(Debug)] diff --git a/crates/site-data/src/map.rs b/crates/site-data/src/map.rs index 1f9f4336f..f9af66fc8 100644 --- a/crates/site-data/src/map.rs +++ b/crates/site-data/src/map.rs @@ -5,7 +5,7 @@ use std::collections::{BTreeSet, HashMap}; use serde_json::Value; use keystore::{ss58_encode, BITTENSOR_SS58_PREFIX}; -use site_types::{coding_arena, design_frame, prism_frame}; +use site_types::{design_frame, prism_frame, relearn_frame}; use site_types::{ ActivityEvent, ActivitySeverity, Agent, Arena, ArenaSlug, LeaderboardRow, LossPoint, LossSeries, PrismTelemetry, PrismTelemetryPoint, PrismWindow, RulesGate, SealedPaths, @@ -397,18 +397,27 @@ fn format_g2_score(v: f64) -> String { } } -/// All three arenas (coding always paused). +/// Relearn LLM card from `/v1/status` (or the static frame when down). #[must_use] -pub fn list_arenas( - design_dash: Option<&Value>, - prism_status: Option<&Value>, - prism_subs: Option<&Value>, -) -> Vec { - vec![ - coding_arena(), - design_arena_from_dashboard(design_dash), - prism_arena_from_live(prism_status, prism_subs), - ] +pub fn relearn_arena_from_live(status: Option<&Value>) -> Arena { + hydrate_arena(relearn_frame(), status) +} + +/// Fill an arena card's champion id from that challenge's `/v1/status`. +/// +/// A down backend leaves the static frame in place rather than dropping the +/// arena, so the site's emission column still accounts for every challenge. +#[must_use] +pub fn hydrate_arena(mut arena: Arena, status: Option<&Value>) -> Arena { + if let Some(s) = status { + if let Some(id) = s.get("champion_id").and_then(|v| v.as_str()) { + if !id.is_empty() { + id.clone_into(&mut arena.best_score); + } + } + arena.status = "live".into(); + } + arena } fn format_elo(v: f64) -> String { diff --git a/crates/site-data/src/weights.rs b/crates/site-data/src/weights.rs index 4e0e14f0e..2f6d0b9df 100644 --- a/crates/site-data/src/weights.rs +++ b/crates/site-data/src/weights.rs @@ -231,6 +231,34 @@ mod tests { assert!(w.hotkey_weights[0].hotkey.starts_with('5')); } + #[test] + fn every_live_arena_slug_matches_a_trust_root_share() { + // The site matches emission by slug string, so an arena whose slug is + // not the challenge id would silently show a 0 % share while that + // challenge really earns emission. + let root = ChallengesBody { + challenges: vec![ + entry("relearn", 4000), + entry("relearn-t2i", 1500), + entry("relearn-mm", 1500), + entry("bounty", 3000), + ], + }; + let shares = configured_shares(Some(&root)); + let sum: f64 = shares.iter().map(|(_, v)| *v).sum(); + assert!((sum - 1.0).abs() < 1e-9, "shares sum to {sum}"); + + for slug in ["relearn", "relearn-t2i", "relearn-mm"] { + let arena_slug = site_types::ArenaSlug::parse(slug) + .unwrap_or_else(|| panic!("{slug} must parse as an arena slug")); + assert_eq!(arena_slug.as_str(), slug); + let Some((_, share)) = shares.iter().find(|(s, _)| s == slug) else { + panic!("no trust-root share for {slug}"); + }; + assert!(*share > 0.0, "{slug} share is zero"); + } + } + #[test] fn unsealed_is_fail_closed() { let root = trust_root(); diff --git a/crates/site-types/src/frames.rs b/crates/site-types/src/frames.rs index c1da6f3f6..8d9f05f43 100644 --- a/crates/site-types/src/frames.rs +++ b/crates/site-types/src/frames.rs @@ -40,7 +40,7 @@ pub fn design_frame() -> Arena { name: "Design Arena".into(), tagline: "Agents turn a product brief into a working landing page; operators award 1–2 winners per round.".into(), description: "Miners submit an agent harness that produces sanitised HTML pages for a prompt set. Clean runs reach admin review; winners receive lattice scores and Elo-style ratings for the round.".into(), - status: "live".into(), + status: "retired".into(), scoring: ScoringMethod::Elo, mechanism: vec![ "Harness → sandboxed pages".into(), @@ -74,7 +74,7 @@ pub fn prism_frame() -> Arena { name: "Prism".into(), tagline: "Prism v2.1 — AutoModel pin+patch, 4h train on 1× B200, dense 1B reference (850M–1B). Public board is G2 lattice; 2.0 harvests cannot win.".into(), description: "New competition (prism-v2.1, scoring generation 21, recipe 2.1.0). Every miner trains inside the operator-owned recipe on the same pinned shard, seed, and caps. The public board lists only v2.1 harvests. Until the first eligible 2.1 run finishes, subnet weights stay burn (uid 0). Rankings use measured G2 / G1–G8 fields — never invented curves.".into(), - status: "live".into(), + status: "retired".into(), scoring: ScoringMethod::SpectralFusion, mechanism: vec![ "Recipe 2.1.0 · prism-v2.1 · 4h / 1× B200".into(), @@ -99,3 +99,105 @@ pub fn prism_frame() -> Arena { seconds_remaining: None, } } + +/// Relearn T2I arena frame; counters filled by caller from live status. +#[must_use] +pub fn relearn_t2i_frame() -> Arena { + Arena { + slug: ArenaSlug::RelearnT2i, + name: "Relearn T2I".into(), + tagline: "Image generation: miners fine-tune Cosmos3-Super-Text2Image; Q-Judger scores frozen prompts at shared seeds.".into(), + description: "Fine-tune the pinned NVIDIA Cosmos3 generator (OpenMDW 1.1). Every miner renders the same Qwen-Image-Bench prompt ids at the same derived seeds, so images are directly comparable. Q-Judger is the only judge. Promotion needs a paired win on a private prompt holdout with no L1 pillar regression. Flux-family bases are rejected.".into(), + status: "live".into(), + scoring: ScoringMethod::Displacement, + mechanism: vec![ + "Miners pay Lium (BYOK)".into(), + "Q-Judger on frozen prompts at shared seeds".into(), + "No pillar may regress; seed replay + contamination gates".into(), + ], + agents: 0, + best_score: "—".into(), + best_score_label: "DISPLACE".into(), + emission_share: 1.0, + weight: 1.0, + rewards_per_day: 0.0, + references: vec![ProjectReference { + name: "Relearn".into(), + repo: "CortexLM/relearn".into(), + repo_url: "https://github.com/CortexLM/relearn".into(), + }], + source_url: "https://github.com/CortexLM/relearn".into(), + plate: "/plates/relearn.svg".into(), + round_id: None, + round_ends_at: None, + seconds_remaining: None, + } +} + +/// Relearn Multimodal arena frame; counters filled by caller from live status. +#[must_use] +pub fn relearn_mm_frame() -> Arena { + Arena { + slug: ArenaSlug::RelearnMm, + name: "Relearn Multimodal".into(), + tagline: "Give the champion LLM eyes: train a permissive vision encoder without regressing the language model.".into(), + description: "Attach an Apache-2.0 / MIT / BSD / ISC vision encoder plus projector to the champion Relearn LLM. Two gates, both mandatory: the text holdout must hold (a regression is worth zero, not a lower score), and the image holdout plus agentic image-tool traces must beat the champion. Every trace is replayed with shuffled pixels, so a model that ignores the image cannot pass.".into(), + status: "live".into(), + scoring: ScoringMethod::Displacement, + mechanism: vec![ + "Miners pay Lium (BYOK)".into(), + "LLM-intact hard gate before any vision credit".into(), + "Pixel-shuffle control on agentic traces".into(), + ], + agents: 0, + best_score: "—".into(), + best_score_label: "DISPLACE".into(), + emission_share: 1.0, + weight: 1.0, + rewards_per_day: 0.0, + references: vec![ProjectReference { + name: "Relearn".into(), + repo: "CortexLM/relearn".into(), + repo_url: "https://github.com/CortexLM/relearn".into(), + }], + source_url: "https://github.com/CortexLM/relearn".into(), + plate: "/plates/relearn.svg".into(), + round_id: None, + round_ends_at: None, + seconds_remaining: None, + } +} + +/// Relearn LLM arena frame; counters filled by caller from live status. +#[must_use] +pub fn relearn_frame() -> Arena { + Arena { + slug: ArenaSlug::Relearn, + name: "Relearn LLM".into(), + tagline: "Post-training factory: miners improve Qwen3.8-Flash-Next; score is displacement vs the previous champion.".into(), + description: "Submit an improved artifact of the pinned base model. Miners pay Lium. Promotion requires a significant paired win, retention/overfit gates, and an operator audit. Regressions are never crowned.".into(), + status: "live".into(), + scoring: ScoringMethod::Displacement, + mechanism: vec![ + "Miners pay Lium (BYOK)".into(), + "Paired displacement vs champion".into(), + "Operator-audited promote (never a regression)".into(), + ], + agents: 0, + best_score: "—".into(), + best_score_label: "DISPLACE".into(), + emission_share: 1.0, + weight: 1.0, + rewards_per_day: 0.0, + references: vec![ProjectReference { + name: "Relearn".into(), + repo: "CortexLM/relearn".into(), + repo_url: "https://github.com/CortexLM/relearn".into(), + }], + source_url: "https://github.com/CortexLM/relearn".into(), + plate: "/plates/relearn.svg".into(), + round_id: None, + round_ends_at: None, + seconds_remaining: None, + } +} diff --git a/crates/site-types/src/lib.rs b/crates/site-types/src/lib.rs index 994228072..19fee0664 100644 --- a/crates/site-types/src/lib.rs +++ b/crates/site-types/src/lib.rs @@ -9,6 +9,8 @@ mod frames; mod paginate; mod types; -pub use frames::{coding_arena, design_frame, prism_frame}; +pub use frames::{ + coding_arena, design_frame, prism_frame, relearn_frame, relearn_mm_frame, relearn_t2i_frame, +}; pub use paginate::page_slice; pub use types::*; diff --git a/crates/site-types/src/types.rs b/crates/site-types/src/types.rs index 68468344f..d9d118bc3 100644 --- a/crates/site-types/src/types.rs +++ b/crates/site-types/src/types.rs @@ -3,15 +3,26 @@ use serde::{Deserialize, Serialize}; /// Arena identifier. +/// +/// The wire form of every live variant equals its trust-root `challenge_id`, +/// because `apply_emission` matches emission shares by that string. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum ArenaSlug { /// Coding (paused). Coding, - /// Design challenge. + /// Design challenge (retired). Design, - /// Prism challenge. + /// Prism challenge (retired). Prism, + /// Relearn LLM post-training factory. + Relearn, + /// Relearn T2I image generation. + #[serde(rename = "relearn-t2i")] + RelearnT2i, + /// Relearn Multimodal vision encoder. + #[serde(rename = "relearn-mm")] + RelearnMm, } impl ArenaSlug { @@ -22,6 +33,9 @@ impl ArenaSlug { "coding" => Some(Self::Coding), "design" => Some(Self::Design), "prism" => Some(Self::Prism), + "relearn" => Some(Self::Relearn), + "relearn-t2i" => Some(Self::RelearnT2i), + "relearn-mm" => Some(Self::RelearnMm), _ => None, } } @@ -33,6 +47,9 @@ impl ArenaSlug { Self::Coding => "coding", Self::Design => "design", Self::Prism => "prism", + Self::Relearn => "relearn", + Self::RelearnT2i => "relearn-t2i", + Self::RelearnMm => "relearn-mm", } } } @@ -47,6 +64,8 @@ pub enum ScoringMethod { Elo, /// Prism spectral fusion / BPB. SpectralFusion, + /// Relearn paired displacement vs champion. + Displacement, } /// Reference chip on an arena card. diff --git a/crates/trustroot/tests/trustroot_verify.rs b/crates/trustroot/tests/trustroot_verify.rs index b1a4052c7..37526fb6f 100644 --- a/crates/trustroot/tests/trustroot_verify.rs +++ b/crates/trustroot/tests/trustroot_verify.rs @@ -346,27 +346,62 @@ fn s9_repo_config_loads_when_present() { } let (ch, ms) = load_config_dir(&root, 0, 3).expect("committed config must verify"); let primary = ch.primary().unwrap(); - assert_eq!(primary.body.challenges.len(), 2); - // Emission 100% prism (2026-08-16): design 0 / prism 10000 (was 5000/5000). - let design = primary.body.get(b"design").expect("design row"); - assert_eq!(design.emission_share_bps, 0); - assert_eq!( - encode_hex(&design.public_key), - "3e27f87d8330006a73174001120c3455f16b95fee098bb8c2bab9d5053840418" - ); - let prism = primary.body.get(b"prism").expect("prism row"); - assert_eq!(prism.emission_share_bps, BPS_DENOM); - assert_eq!( - encode_hex(&prism.public_key), - "bcd50bb830e050ed4b011dd8f1d2f126fdb42dc55b45ece30a7d5c8ceb3c5219" - ); - assert_ne!(prism.public_key, design.public_key); + assert_eq!(primary.body.challenges.len(), 4); + + // Four live challenges: Relearn LLM, Relearn T2I, Relearn Multimodal, Bounty. + let expected: [(&[u8], u16, &str); 4] = [ + ( + b"relearn", + 4000, + "8ab577207bb6dfc770a850710824a098d53b1ee90abb92925bd0928937131674", + ), + ( + b"relearn-t2i", + 1500, + "923324e1df896b20c49c47f40dacbc4c53cab23e6cc5a1136529302b4c2da110", + ), + ( + b"relearn-mm", + 1500, + "220e489f8157e477730e2e3ee6ce51be0fcf8779575c486a70658a28d5a51841", + ), + ( + b"bounty", + 3000, + "d2ffbe70de7c052deafaba48b90544db4abc1133278c907f2018f457f34aac25", + ), + ]; + for (id, bps, pk) in expected { + let row = primary + .body + .get(id) + .unwrap_or_else(|| panic!("{} row", String::from_utf8_lossy(id))); + assert_eq!( + row.emission_share_bps, + bps, + "{}", + String::from_utf8_lossy(id) + ); + assert_eq!(encode_hex(&row.public_key), pk); + } + + // Every challenge must sign under its own key, so no two rows may share one. + let mut keys: Vec = primary + .body + .challenges + .iter() + .map(|c| encode_hex(&c.public_key)) + .collect(); + keys.sort(); + let total = keys.len(); + keys.dedup(); + assert_eq!(keys.len(), total, "challenge public keys must be distinct"); + + assert!(primary.body.get(b"design").is_none()); + assert!(primary.body.get(b"prism").is_none()); let shares = primary.body.emission_shares(); - assert_eq!(shares.len(), 2); - assert_eq!(shares[0].0, b"design"); - assert_eq!(shares[0].1, 0); - assert_eq!(shares[1].0, b"prism"); - assert_eq!(shares[1].1, BPS_DENOM); + assert_eq!(shares.len(), 4); + assert_eq!(shares.iter().map(|s| s.1).sum::(), BPS_DENOM); // base-agent CVM path removed — committed allowlist is empty (fail-closed). let entries = &ms.primary().unwrap().body.entries; assert!( diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 90997334c..71d9c8ede 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -35,8 +35,7 @@ Compose always runs a digest-pinned `postgres` service (`base-pgdata` volume, he | Data | Store | |------|--------| -| Design harnesses / runs / stages / artifacts metadata / admin rounds | **Postgres** (`design_*`) | -| Prism submissions / stage events | **Postgres** (`prism_*`) | +| Relearn submissions (v0) | **in-memory** (`relearn-store`); Postgres can replace without changing HTTP | | Gateway raw weight leaves + sealed bundles | **Postgres** (`raw_weight_snapshot`, `epoch_bundle`, …) | | Validator attestations (when DB configured) | **Postgres** | | Design sandbox staging files | volume `${BASE_STATE_DIR}/design/staging` + `design-artifacts` | @@ -44,7 +43,7 @@ Compose always runs a digest-pinned `postgres` service (`base-pgdata` volume, he | site-api (`GET /v1/site/*`) | no DB — proxies challenge upstreams via gateway | | Unit/integration tests | may construct `Memory*Store` directly; omit `BASE_DATABASE_URL` only there | -Migrations (`crates/db/migrations`) run on boot in gateway / design-challenge / prism-challenge when `BASE_DATABASE_URL` is set. Compose requires `deploy/env/{design,prism}-challenge.env` so challenges cannot silently boot on memory. +Migrations (`crates/db/migrations`) run on boot in gateway when `BASE_DATABASE_URL` is set. Compose requires `deploy/env/relearn-challenge.env` and `deploy/env/bounty-challenge.env` so live challenges cannot silently boot without operator config. ## Prism Lium GPU profiles (do not mix) @@ -86,7 +85,7 @@ Full procedure: [`docs/runbooks/local-testnet-e2e.md`](../docs/runbooks/local-te | `base-validator` wallet | **no** (fetch-only) | for on-chain weight submit | | Fresh `target/release/{gateway,validator,…}` (or `BASE_DOCKER_BUILD_FROM=source`) | recommended | **required** for real chain | -**Weights seal smoke (default on `--smoke`):** after healthz, `local-e2e.sh` runs `weights-smoke` — signed prism leaves for the live metagraph → `POST /v1/admin/seal` → assert `GET /v1/weights/latest` is **200** with **`sealed: true`**. Skip with `--no-weights-smoke`. Pre-seal, latest is **200 burn** (`sealed: false`, uid 0 = 100%) — never 404; that is unrelated to a missing gateway owner wallet. Prefer `--burn` on mainnet when sealing without real challenge scores (all `NoScore` → uid 0). +**Weights seal smoke (default on `--smoke`):** after healthz, `local-e2e.sh` runs `weights-smoke` — signed relearn leaves for the live metagraph → `POST /v1/admin/seal` → assert `GET /v1/weights/latest` is **200** with **`sealed: true`**. Skip with `--no-weights-smoke`. Pre-seal, latest is **200 burn** (`sealed: false`, uid 0 = 100%) — never 404; that is unrelated to a missing gateway owner wallet. Prefer `--burn` on mainnet when sealing without real challenge scores (all `NoScore` → uid 0). **Interim prod burn seal (retired while prism auto-emits):** `weights-smoke --burn` posts all-`NoScore` at a **block-scale** epoch. That hid the live Prism 2.1 WTA winner (chain epoch ~24k) because `/v1/weights/latest` had no chain-scale bundle to prefer. Keep the script for emergency burn-only windows; **do not** enable `base-burn-seal.timer` when Prism is emitting scores. `remote-deploy` on master enables real-seal and disables the burn timer. diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 58cd30392..4758b47f7 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -12,8 +12,6 @@ ARG DEBIAN_IMAGE=debian@sha256:7b140f374b289a7c2befc338f42ebe6441b7ea838a042bbd5acbfca6ec875818 ARG RUST_IMAGE=rust@sha256:e51d0265072d2d9d5d320f6a44dde6b9ef13653b035098febd68cce8fa7c0bc4 -# python:3.12.11-slim-bookworm (multi-arch index digest) -ARG PYTHON_IMAGE=python@sha256:519591d6871b7bc437060736b9f7456b8731f1499a57e22e6c285135ae657bf7 ARG BUILD_FROM=source # ----------------------------------------------------------------------------- @@ -35,9 +33,6 @@ COPY bins ./bins COPY xtask ./xtask # Keep workspace metadata members that Docker context needs COPY config ./config -# design-challenge embeds the published baseline agent as the agentic review -# corpus anchor (include_str! in crates/design-challenge/src/orchestrator.rs). -COPY docs/external-miner/examples/design-baseline ./docs/external-miner/examples/design-baseline # validator-bin/dcap turns on real Intel DCAP quote verification; without it the # validator parks every attestation instead of verifying it. RUN cargo build --release \ @@ -45,25 +40,25 @@ RUN cargo build --release \ -p validator-bin \ -p gateway-bin \ -p updater-bin \ - -p prism-challenge-bin \ - -p design-challenge-bin \ - -p design-egress-proxy-bin \ - && cargo build --release -p challenge-review-bin \ + -p relearn-challenge-bin \ + -p relearn-t2i-challenge-bin \ + -p relearn-mm-challenge-bin \ + -p bounty-challenge-bin \ && strip target/release/validator \ target/release/gateway \ target/release/updater \ - target/release/prism-challenge \ - target/release/design-challenge \ - target/release/design-egress-proxy \ - target/release/challenge-review \ + target/release/relearn-challenge \ + target/release/relearn-t2i-challenge \ + target/release/relearn-mm-challenge \ + target/release/bounty-challenge \ && mkdir -p /out \ && cp target/release/validator \ target/release/gateway \ target/release/updater \ - target/release/prism-challenge \ - target/release/design-challenge \ - target/release/design-egress-proxy \ - target/release/challenge-review /out/ + target/release/relearn-challenge \ + target/release/relearn-t2i-challenge \ + target/release/relearn-mm-challenge \ + target/release/bounty-challenge /out/ # ----------------------------------------------------------------------------- # Prebuilt path — copy host target/release artifacts (local verify only) @@ -76,10 +71,10 @@ RUN apt-get update \ COPY target/release/validator \ target/release/gateway \ target/release/updater \ - target/release/prism-challenge \ - target/release/design-challenge \ - target/release/design-egress-proxy \ - target/release/challenge-review \ + target/release/relearn-challenge \ + target/release/relearn-t2i-challenge \ + target/release/relearn-mm-challenge \ + target/release/bounty-challenge \ /out/ # ----------------------------------------------------------------------------- @@ -119,73 +114,45 @@ FROM runtime-base AS updater COPY --from=artifacts --chown=base:base /out/updater /usr/local/bin/updater ENTRYPOINT ["/usr/local/bin/updater"] -FROM runtime-base AS prism-challenge -# prism-lium drives pod exec over the `ssh` binary; runtime-base has none. -# Pre-create the checkpoint park root as uid 65532 so a fresh named volume -# inherits ownership (Docker copies image dir perms into empty volumes). -# Without this, `base` cannot mkdir under /var/lib/prism → harvest fails with -# `lium exec: mkdir: Permission denied` after a successful eval. +FROM runtime-base AS relearn-challenge +# Lium eval uses operator SSH; runtime-base has no ssh client. USER root RUN apt-get update \ && apt-get install -y --no-install-recommends openssh-client \ && rm -rf /var/lib/apt/lists/* \ - && mkdir -p /var/lib/prism/artifacts \ - && chown -R base:base /var/lib/prism + && mkdir -p /var/lib/relearn \ + && chown -R base:base /var/lib/relearn USER base -COPY --from=artifacts --chown=base:base /out/prism-challenge /usr/local/bin/prism-challenge -ENTRYPOINT ["/usr/local/bin/prism-challenge"] +COPY --from=artifacts --chown=base:base /out/relearn-challenge /usr/local/bin/relearn-challenge +ENTRYPOINT ["/usr/local/bin/relearn-challenge"] -FROM runtime-base AS design-challenge +FROM runtime-base AS relearn-t2i-challenge +# Cosmos3 generation and Q-Judger scoring both run on rented GPUs over SSH. USER root -# Headless Chromium for full-page design screenshots (DESIGN_CHROME_BIN). RUN apt-get update \ - && apt-get install -y --no-install-recommends chromium fonts-liberation \ + && apt-get install -y --no-install-recommends openssh-client \ && rm -rf /var/lib/apt/lists/* \ - && mkdir -p /var/lib/design/staging \ - && chown -R base:base /var/lib/design -ENV DESIGN_CHROME_BIN=/usr/bin/chromium + && mkdir -p /var/lib/relearn-t2i \ + && chown -R base:base /var/lib/relearn-t2i USER base -COPY --from=artifacts --chown=base:base /out/design-challenge /usr/local/bin/design-challenge -ENTRYPOINT ["/usr/local/bin/design-challenge"] - -FROM runtime-base AS design-egress-proxy -COPY --from=artifacts --chown=base:base /out/design-egress-proxy /usr/local/bin/design-egress-proxy -ENTRYPOINT ["/usr/local/bin/design-egress-proxy"] +COPY --from=artifacts --chown=base:base /out/relearn-t2i-challenge /usr/local/bin/relearn-t2i-challenge +ENTRYPOINT ["/usr/local/bin/relearn-t2i-challenge"] -# ----------------------------------------------------------------------------- -# design-runtime — pinned Python for design-challenge sandboxes (install/run). -# No challenge secrets; sandboxes reach egress only via design-sandbox-egress. -# ----------------------------------------------------------------------------- -FROM ${PYTHON_IMAGE} AS design-runtime +FROM runtime-base AS relearn-mm-challenge USER root RUN apt-get update \ - && apt-get install -y --no-install-recommends bash ca-certificates \ + && apt-get install -y --no-install-recommends openssh-client \ && rm -rf /var/lib/apt/lists/* \ - && groupadd --system --gid 65532 base \ - && useradd --system --uid 65532 --gid base --home-dir /home/base --create-home base \ - && mkdir -p /work /out \ - && chown -R base:base /work /out /home/base + && mkdir -p /var/lib/relearn-mm \ + && chown -R base:base /var/lib/relearn-mm USER base -WORKDIR /work -# Harness entry is staged into /work by design-challenge; image only supplies Python. -CMD ["python", "--version"] +COPY --from=artifacts --chown=base:base /out/relearn-mm-challenge /usr/local/bin/relearn-mm-challenge +ENTRYPOINT ["/usr/local/bin/relearn-mm-challenge"] -# ----------------------------------------------------------------------------- -# design-review — containerized anti-cheat review (challenge-agentic + AST). -# design-challenge stages /work (submitted agent + _similar harness, read-only) -# and /out (verdict) per run; the OpenRouter key arrives via container env. -# Python is present so the sandboxed run_command tool can run AST probes. -# ----------------------------------------------------------------------------- -FROM ${PYTHON_IMAGE} AS design-review +FROM runtime-base AS bounty-challenge USER root -RUN apt-get update \ - && apt-get install -y --no-install-recommends bash ca-certificates \ - && rm -rf /var/lib/apt/lists/* \ - && groupadd --system --gid 65532 base \ - && useradd --system --uid 65532 --gid base --home-dir /home/base --create-home base \ - && mkdir -p /work /out \ - && chown -R base:base /work /out /home/base +RUN mkdir -p /var/lib/bounty \ + && chown -R base:base /var/lib/bounty USER base -WORKDIR /work -COPY --from=artifacts --chown=base:base /out/challenge-review /usr/local/bin/challenge-review -ENTRYPOINT ["/usr/local/bin/challenge-review"] +COPY --from=artifacts --chown=base:base /out/bounty-challenge /usr/local/bin/bounty-challenge +ENTRYPOINT ["/usr/local/bin/bounty-challenge"] diff --git a/deploy/compose/env-local.yml b/deploy/compose/env-local.yml index 3e0abd9fd..5b3ae8de6 100644 --- a/deploy/compose/env-local.yml +++ b/deploy/compose/env-local.yml @@ -61,43 +61,37 @@ services: - ./.local/trust-root:/etc/base/config:ro - base-validator-lkg:/var/lib/base - prism-challenge: + relearn-challenge: ports: - - "127.0.0.1:${LOCAL_PRISM_HOST_PORT:-28092}:8092" + - "127.0.0.1:${LOCAL_RELEARN_HOST_PORT:-28095}:8095" environment: BASE_DATABASE_URL: ${LOCAL_DATABASE_URL:-postgres://base:base_dev_only_change_me@postgres:5432/base} # No Lium spend on a laptop unless operator opts in. - PRISM_FORCE_SIM: "${LOCAL_PRISM_FORCE_SIM:-true}" - # Local miners are usually not registered on testnet 541: gating off by - # default here (staging/prod never set this). 1 = enforce 1-max+metagraph. + RELEARN_FORCE_SIM: "${LOCAL_RELEARN_FORCE_SIM:-true}" BASE_SUBMISSION_GATING: "${LOCAL_SUBMISSION_GATING:-0}" - # Hold after each stage so visual A→Z can photograph mid-flight (0 = off). - PRISM_SIM_STAGE_DELAY_MS: "${LOCAL_PRISM_SIM_STAGE_DELAY_MS:-0}" - design-challenge: + relearn-t2i-challenge: ports: - - "127.0.0.1:${LOCAL_DESIGN_HOST_PORT:-28093}:8093" + - "127.0.0.1:${LOCAL_RELEARN_T2I_HOST_PORT:-28097}:8097" environment: BASE_DATABASE_URL: ${LOCAL_DATABASE_URL:-postgres://base:base_dev_only_change_me@postgres:5432/base} - # Host SimSandbox only on this local overlay (never staging/prod). - # Requires BASE_ALLOW_HOST_SIM=1; bin refuses Sim on mainnet / prod. - BASE_ALLOW_HOST_SIM: "${BASE_ALLOW_HOST_SIM:-1}" - # Offline sandbox on a laptop unless operator opts into Docker runtime. - DESIGN_FORCE_SIM: "${LOCAL_DESIGN_FORCE_SIM:-true}" - # Deterministic AST agentic for local e2e; OpenRouter when false + key mounted. - DESIGN_FORCE_AGENTIC_SIM: "${LOCAL_DESIGN_FORCE_AGENTIC_SIM:-false}" - # Local miners are usually not registered on testnet 541: gating off by - # default here (staging/prod never set this). 1 = enforce 1-max+metagraph. + # No Lium spend and no Q-Judger endpoint on a laptop unless opted in. + RELEARN_T2I_FORCE_SIM: "${LOCAL_RELEARN_T2I_FORCE_SIM:-true}" BASE_SUBMISSION_GATING: "${LOCAL_SUBMISSION_GATING:-0}" - # In-process review for local e2e (design-review image needs a built - # artifact); staging/prod run DESIGN_REVIEW_BACKEND=docker. - DESIGN_REVIEW_BACKEND: "${LOCAL_DESIGN_REVIEW_BACKEND:-inline}" - # Hold after each stage so visual A→Z can photograph mid-flight (0 = off). - DESIGN_SIM_STAGE_DELAY_MS: "${LOCAL_DESIGN_SIM_STAGE_DELAY_MS:-0}" - design-egress-proxy: + relearn-mm-challenge: ports: - - "127.0.0.1:${LOCAL_DESIGN_EGRESS_HOST_PORT:-28094}:8094" + - "127.0.0.1:${LOCAL_RELEARN_MM_HOST_PORT:-28098}:8098" environment: - # No OpenRouter spend locally unless a real key is mounted. - DESIGN_EGRESS_SIM: "${LOCAL_DESIGN_EGRESS_SIM:-true}" + BASE_DATABASE_URL: ${LOCAL_DATABASE_URL:-postgres://base:base_dev_only_change_me@postgres:5432/base} + RELEARN_MM_FORCE_SIM: "${LOCAL_RELEARN_MM_FORCE_SIM:-true}" + RELEARN_MM_CHAMPION_LM_HASH: "${RELEARN_MM_CHAMPION_LM_HASH:-}" + BASE_SUBMISSION_GATING: "${LOCAL_SUBMISSION_GATING:-0}" + + bounty-challenge: + ports: + - "127.0.0.1:${LOCAL_BOUNTY_HOST_PORT:-28096}:8096" + environment: + BASE_DATABASE_URL: ${LOCAL_DATABASE_URL:-postgres://base:base_dev_only_change_me@postgres:5432/base} + BOUNTY_CHAT_COMMAND: "${BOUNTY_CHAT_COMMAND:-}" + BOUNTY_BACKEND_PUBLIC_URL: "${BOUNTY_BACKEND_PUBLIC_URL:-}" diff --git a/deploy/compose/env-prod.yml b/deploy/compose/env-prod.yml index 78eb8a137..9271f9704 100644 --- a/deploy/compose/env-prod.yml +++ b/deploy/compose/env-prod.yml @@ -54,47 +54,25 @@ services: volumes: - ./deploy/secrets/wallets:/run/base/wallets:ro - ./deploy/secrets/gateway_admin_token:/run/secrets/gateway_admin_token:ro - prism-challenge: + relearn-challenge: environment: - PRISM_FORCE_SIM: "false" - # G1–G8 battery (two-phase train/eval). Default harness is v3; pin here - # so pods never silently fall back to BPB-only even on older images. - PRISM_FLOW: "v3" - # Public v9 boots B200/5090; avoids private DO template create without cred. - # Official public daturaai/pytorch CUDA 13 DIND (miner BYOK can rent). - # Do not pin private prism-recipe-v9 f2f5e84c — rent 400s for other accounts. - PRISM_POD_TEMPLATE_ID: "345273fa-4818-46f7-a8fa-32f0e331713c" - # Full public pack (not tiny caps). Host path staged by - # deploy/scripts/prism-overnight-battery.sh / build_private_pack. - PRISM_EVAL_ASSETS_DIR: "/var/lib/prism/eval-assets" - PRISM_TEST_EVAL_CAPS: "0" - # Recipe 2.0 AutoModel pin (stage with deploy/scripts/stage-automodel-pin.sh). - # Fail-closed intake when unset/unmounted — miners see code=pin. - PRISM_AUTOMODEL_PIN_DIR: "/var/lib/prism/automodel-pin" - # Short-TTL encrypted BYOK seals (cleanup after restart). Host dir + key file. - PRISM_PAYER_VAULT_DIR: "/var/lib/prism/payer-vault" - PRISM_PAYER_VAULT_KEY_FILE: "/run/secrets/prism_payer_vault_key" - # Horizontal scale: N orchestrator workers each claim_next under a semaphore. - PRISM_MAX_CONCURRENT_EVALS: "8" - # Emitter/gating/epoch-feed chain reads share the gateway failover list. + RELEARN_FORCE_SIM: "false" + RELEARN_TEACHER_BACKEND: "http_api" + RELEARN_TEACHER_API_URL: "${RELEARN_TEACHER_API_URL:-}" + RELEARN_TEACHER_MODEL: "${RELEARN_TEACHER_MODEL:-}" BASE_CHAIN_ENDPOINTS: "wss://bittensor-finney.api.onfinality.io/public-ws,wss://entrypoint-finney.opentensor.ai:443" - # Top-model HuggingFace publish (optional; no-op without token file). - PRISM_TOPMODEL_HF_TOKEN_FILE: "/run/base/huggingface/token" - PRISM_TOPMODEL_HF_REPO: "BaseIntelligence/top-prism-architecture" - volumes: - - /var/lib/prism/automodel-pin:/var/lib/prism/automodel-pin:ro - - /var/lib/prism/payer-vault:/var/lib/prism/payer-vault - - /var/lib/prism/eval-assets:/var/lib/prism/eval-assets:ro - - ./deploy/secrets/prism/payer_vault_key:/run/secrets/prism_payer_vault_key:ro - - ./deploy/secrets/huggingface:/run/base/huggingface:ro - design-challenge: + relearn-t2i-challenge: + environment: + RELEARN_T2I_FORCE_SIM: "false" + RELEARN_T2I_JUDGE_BACKEND: "http_api" + RELEARN_T2I_JUDGE_API_URL: "${RELEARN_T2I_JUDGE_API_URL:-}" + BASE_CHAIN_ENDPOINTS: "wss://bittensor-finney.api.onfinality.io/public-ws,wss://entrypoint-finney.opentensor.ai:443" + relearn-mm-challenge: environment: - # Docker-only on prod. Never set BASE_ALLOW_HOST_SIM / DESIGN_FORCE_SIM here. - DESIGN_FORCE_SIM: "false" - # Trust root is design = 0 bps. Extra design leaves 409 D24 and hide - # the prism WTA winner. Gate emit until design share is non-zero. - DESIGN_SKIP_LEAF_EMIT: "1" + RELEARN_MM_FORCE_SIM: "false" + RELEARN_MM_CHAMPION_LM_HASH: "${RELEARN_MM_CHAMPION_LM_HASH:-}" BASE_CHAIN_ENDPOINTS: "wss://bittensor-finney.api.onfinality.io/public-ws,wss://entrypoint-finney.opentensor.ai:443" - design-egress-proxy: + bounty-challenge: environment: - DESIGN_EGRESS_SIM: "false" + # CortexLM/backend public base. Empty → skip / sim. Never bake a host. + BOUNTY_BACKEND_PUBLIC_URL: "${BOUNTY_BACKEND_PUBLIC_URL:-}" diff --git a/deploy/compose/env-staging.yml b/deploy/compose/env-staging.yml index fbff0ec9b..262bd4fde 100644 --- a/deploy/compose/env-staging.yml +++ b/deploy/compose/env-staging.yml @@ -21,7 +21,7 @@ services: BASE_VALIDATOR_WALLET: base-validator volumes: - ./deploy/secrets/wallets:/run/base/wallets:ro - # Staging-only trust root: mirrors prod emission (design/prism 0/10000). + # Staging-only trust root: mirrors prod emission (relearn 10000). - ./config/challenges.staging.toml:/etc/base/config/challenges.toml:ro - ./config/challenges.staging.toml.sig:/etc/base/config/challenges.toml.sig:ro - base-validator-lkg:/var/lib/base @@ -46,54 +46,35 @@ services: - ./deploy/secrets/wallets:/run/base/wallets:ro - ./deploy/secrets/gateway_admin_token:/run/secrets/gateway_admin_token:ro # The sealer must sign with the same staging trust root the validator - # verifies against (design/prism 0/10000), or /v1/weights/latest never - # shows design weight and the validator flags emission share mismatch (D23). + # verifies against (relearn 4000 + relearn-t2i 1500 + relearn-mm 1500 + + # bounty 3000), or /v1/weights/latest + # never shows challenge weight and the validator flags emission share mismatch (D23). - ./config/challenges.staging.toml:/etc/base/config/challenges.toml:ro - ./config/challenges.staging.toml.sig:/etc/base/config/challenges.toml.sig:ro - prism-challenge: + relearn-challenge: environment: BASE_CHAIN_ENDPOINT: "wss://test.chain.opentensor.ai:443" BASE_CHAIN_ENDPOINTS: "wss://test.chain.opentensor.ai:443,wss://test.finney.opentensor.ai:443" - # Compressed e2e proof: offline deterministic backend (no GPU spend) + - # tiny model / short train cap. A real Lium RTX 5090 run stays a - # follow-up validation (flip PRISM_FORCE_SIM to "false" and unset the - # PRISM_TEST_* knobs). - PRISM_FORCE_SIM: "true" - PRISM_TEST_TRAIN_MINUTES: "15" - PRISM_TEST_MAX_PARAMS: "2000000" - # v3 battery even under short-train knobs (tiny grids via default - # PRISM_TEST_* → tiny_caps; set CAPS=0 for full G1–G8 on real Lium). - PRISM_FLOW: "v3" - # Same pin path as prod when testing live AutoModel intake on staging - # (stage with deploy/scripts/stage-automodel-pin.sh). Sim/fixture pins - # still work when miners submit automodel@fixture-v1. - PRISM_AUTOMODEL_PIN_DIR: "/var/lib/prism/automodel-pin" - PRISM_PAYER_VAULT_DIR: "/var/lib/prism/payer-vault" - PRISM_PAYER_VAULT_KEY_FILE: "/run/secrets/prism_payer_vault_key" - PRISM_TOPMODEL_HF_TOKEN_FILE: "/run/base/huggingface/token" - PRISM_TOPMODEL_HF_REPO: "BaseIntelligence/top-prism-architecture" - volumes: - - /var/lib/prism/automodel-pin:/var/lib/prism/automodel-pin:ro - - /var/lib/prism/payer-vault:/var/lib/prism/payer-vault - - ./deploy/secrets/prism/payer_vault_key:/run/secrets/prism_payer_vault_key:ro - - ./deploy/secrets/huggingface:/run/base/huggingface:ro - design-challenge: + # Live path on droplets: sim is local-only (`env-local.yml`). + RELEARN_FORCE_SIM: "false" + RELEARN_TEACHER_BACKEND: "http_api" + RELEARN_TEACHER_API_URL: "${RELEARN_TEACHER_API_URL:-}" + RELEARN_TEACHER_MODEL: "${RELEARN_TEACHER_MODEL:-}" + relearn-t2i-challenge: + environment: + BASE_CHAIN_ENDPOINT: "wss://test.chain.opentensor.ai:443" + BASE_CHAIN_ENDPOINTS: "wss://test.chain.opentensor.ai:443,wss://test.finney.opentensor.ai:443" + # Live path on droplets: sim is local-only (`env-local.yml`). + RELEARN_T2I_FORCE_SIM: "false" + RELEARN_T2I_JUDGE_BACKEND: "http_api" + RELEARN_T2I_JUDGE_API_URL: "${RELEARN_T2I_JUDGE_API_URL:-}" + relearn-mm-challenge: environment: BASE_CHAIN_ENDPOINT: "wss://test.chain.opentensor.ai:443" BASE_CHAIN_ENDPOINTS: "wss://test.chain.opentensor.ai:443,wss://test.finney.opentensor.ai:443" - # Staging uses real Docker via socket-proxy. Host SimSandbox is fail-closed - # here (never set BASE_ALLOW_HOST_SIM / DESIGN_FORCE_SIM on droplets). - DESIGN_FORCE_SIM: "false" - # Same as prod: staging trust root is design = 0 bps. - DESIGN_SKIP_LEAF_EMIT: "1" - # Compressed lifecycle for the staging e2e: ~15-minute rounds (prod - # defaults stay 8640s / 1800s when these are unset). - DESIGN_ROUND_SECS: "900" - DESIGN_AGENT_RUN_TIMEOUT_SECS: "600" - # Containerized anti-cheat review (design-review one-shot image). - DESIGN_REVIEW_BACKEND: "docker" - DESIGN_REVIEW_IMAGE: "design-review:0.1.0" - design-egress-proxy: + RELEARN_MM_FORCE_SIM: "false" + RELEARN_MM_CHAMPION_LM_HASH: "${RELEARN_MM_CHAMPION_LM_HASH:-}" + bounty-challenge: environment: - # Real OpenRouter when deploy/secrets/openrouter/api_key is non-empty. - DESIGN_EGRESS_SIM: "false" + # CortexLM/backend public base. Empty → skip / sim. Never bake a host. + BOUNTY_BACKEND_PUBLIC_URL: "${BOUNTY_BACKEND_PUBLIC_URL:-}" diff --git a/deploy/compose/role-validator.yml b/deploy/compose/role-validator.yml index f209a7fd7..eb2355cad 100644 --- a/deploy/compose/role-validator.yml +++ b/deploy/compose/role-validator.yml @@ -19,14 +19,16 @@ services: # Updater disabled on validator until GHCR pins wired. updater: profiles: ["never"] - # Challenges + Docker sock proxy are master-only (harness eval / sandbox). + # Challenges + Docker sock proxy are master-only. # Validators fetch sealed weights from the master gateway; they do not run # miner code or hold challenge mini-secrets. - prism-challenge: + relearn-challenge: profiles: ["never"] - design-challenge: + relearn-t2i-challenge: profiles: ["never"] - design-egress-proxy: + relearn-mm-challenge: + profiles: ["never"] + bounty-challenge: profiles: ["never"] socket-proxy: profiles: ["never"] diff --git a/deploy/env/bounty-challenge.env.example b/deploy/env/bounty-challenge.env.example new file mode 100644 index 000000000..17c69369c --- /dev/null +++ b/deploy/env/bounty-challenge.env.example @@ -0,0 +1,24 @@ +# operator-managed, never committed to git. +# Bounty challenge orchestration surface. + +# Compose requires this file. +# Must match deploy/env/postgres.env on the host (see postgres.env.example). +BASE_DATABASE_URL=postgres://base:base_dev_only_change_me@postgres:5432/base +BASE_NETUID=541 + +# Unguessable Cortex Chat inject command. Empty placeholder — never commit a live token. +BOUNTY_CHAT_COMMAND= + +# CortexLM/backend public base (no trailing path). Empty → skip / sim (CI). +# Cortex GETs {url}/v1/bounty/public/leaderboard and .../reports. +# Never bake a host into git. +BOUNTY_BACKEND_PUBLIC_URL= + +# Optional miner-pays-Lium. Skip live Lium when unset. +# LIUM_API_KEY= + +# Operator bearer tokens for POST /v1/admin/adjudicate. +# BOUNTY_ADMIN_TOKENS_FILE=/run/base/bounty/admin_tokens + +# Session HMAC for pairing claims. Generate on the host; never commit. +# BOUNTY_SESSION_SECRET_FILE=/run/base/bounty/session_secret diff --git a/deploy/env/relearn-challenge.env.example b/deploy/env/relearn-challenge.env.example new file mode 100644 index 000000000..1cb47a90c --- /dev/null +++ b/deploy/env/relearn-challenge.env.example @@ -0,0 +1,16 @@ +# operator-managed, never committed to git. +# Relearn challenge orchestration surface. + +# Compose requires this file. +# Must match deploy/env/postgres.env on the host (see postgres.env.example). +BASE_DATABASE_URL=postgres://base:base_dev_only_change_me@postgres:5432/base +BASE_NETUID=541 + +# Teacher HTTP API (judge only). Operator sets these on the host. Never commit values. +RELEARN_TEACHER_BACKEND=http_api +RELEARN_TEACHER_API_URL= +RELEARN_TEACHER_MODEL= +RELEARN_TEACHER_API_KEY= + +# Operator bearer tokens for POST /v1/admin/promote. +# RELEARN_ADMIN_TOKENS_FILE=/run/base/relearn/admin_tokens diff --git a/deploy/env/relearn-mm-challenge.env.example b/deploy/env/relearn-mm-challenge.env.example new file mode 100644 index 000000000..55ecbea8c --- /dev/null +++ b/deploy/env/relearn-mm-challenge.env.example @@ -0,0 +1,18 @@ +# operator-managed, never committed to git. +# Relearn Multimodal challenge orchestration surface. + +# Compose requires this file. +# Must match deploy/env/postgres.env on the host (see postgres.env.example). +BASE_DATABASE_URL=postgres://base:base_dev_only_change_me@postgres:5432/base +BASE_NETUID=541 + +# SHA-256 hex of the champion Relearn LLM weights. This is gate 1's reference: +# an encoder-only submission must hash-match it to prove the language model is +# unchanged. Leave empty only on a host that should reject encoder-only submits. +RELEARN_MM_CHAMPION_LM_HASH= + +# Live path on droplets. Sim is local/CI only. +RELEARN_MM_FORCE_SIM=false + +# Operator bearer tokens for POST /v1/admin/promote. +# RELEARN_MM_ADMIN_TOKENS_FILE=/run/base/relearn-mm/admin_tokens diff --git a/deploy/env/relearn-t2i-challenge.env.example b/deploy/env/relearn-t2i-challenge.env.example new file mode 100644 index 000000000..c5663bfde --- /dev/null +++ b/deploy/env/relearn-t2i-challenge.env.example @@ -0,0 +1,25 @@ +# operator-managed, never committed to git. +# Relearn T2I challenge orchestration surface. + +# Compose requires this file. +# Must match deploy/env/postgres.env on the host (see postgres.env.example). +BASE_DATABASE_URL=postgres://base:base_dev_only_change_me@postgres:5432/base +BASE_NETUID=541 + +# Q-Judger (Qwen/Qwen-Image-Bench) is the only judge for this challenge. +# The operator sets the endpoint on the host; never commit a value or a host. +# With no endpoint and no RELEARN_T2I_FORCE_SIM=1, the service refuses to score. +RELEARN_T2I_JUDGE_BACKEND=http_api +RELEARN_T2I_JUDGE_API_URL= +RELEARN_T2I_JUDGE_API_KEY= + +# Frozen holdout prompt records. Generated off-git with the private salt: +# cargo run -p xtask -- relearn-t2i-holdout --bench \ +# --salt "$RELEARN_T2I_HOLDOUT_SALT" --size 40 --exclude \ +# --out deploy/secrets/relearn-t2i/holdout.json +# The file is verified against holdout_commitment in config/relearn-t2i-pin.toml +# at boot. A mismatch means submissions 503 rather than scoring the public split. +# RELEARN_T2I_HOLDOUT_FILE=/run/base/relearn-t2i/holdout.json + +# Operator bearer tokens for POST /v1/admin/promote. +# RELEARN_T2I_ADMIN_TOKENS_FILE=/run/base/relearn-t2i/admin_tokens diff --git a/deploy/scripts/assert-compose-matrix.sh b/deploy/scripts/assert-compose-matrix.sh index 04500cc3a..4c86108fd 100755 --- a/deploy/scripts/assert-compose-matrix.sh +++ b/deploy/scripts/assert-compose-matrix.sh @@ -51,7 +51,7 @@ services=$(render \ if echo "$services" | grep -qx "gateway"; then fail "validator role renders gateway (must not)" fi -for banned in design-challenge design-egress-proxy prism-challenge socket-proxy; do +for banned in relearn-challenge relearn-t2i-challenge relearn-mm-challenge bounty-challenge socket-proxy design-challenge design-egress-proxy prism-challenge; do if echo "$services" | grep -qx "$banned"; then fail "validator role renders $banned (master-only; must not)" fi @@ -67,11 +67,16 @@ services=$(render \ if ! echo "$services" | grep -qx "gateway"; then fail "master role does not render gateway (must)" fi -for required in design-challenge design-egress-proxy prism-challenge socket-proxy; do +for required in relearn-challenge relearn-t2i-challenge relearn-mm-challenge bounty-challenge socket-proxy; do if ! echo "$services" | grep -qx "$required"; then fail "master role does not render $required (must)" fi done +for retired in design-challenge design-egress-proxy prism-challenge; do + if echo "$services" | grep -qx "$retired"; then + fail "master role still renders retired $retired" + fi +done if echo "$services" | grep -qx "validator"; then fail "master role renders validator (dual submitter; must not — use validator host)" fi @@ -104,7 +109,7 @@ services=$(render \ if echo "$services" | grep -qx "evil-gateway"; then fail "prod validator renders evil-gateway (must not)" fi -for banned in design-challenge design-egress-proxy prism-challenge socket-proxy; do +for banned in relearn-challenge relearn-t2i-challenge relearn-mm-challenge bounty-challenge socket-proxy design-challenge design-egress-proxy prism-challenge; do if echo "$services" | grep -qx "$banned"; then fail "prod validator renders $banned (master-only; must not)" fi @@ -123,30 +128,31 @@ for env_file in deploy/compose/env-staging.yml deploy/compose/env-prod.yml; do fail "$env_file enables BASE_ALLOW_HOST_SIM (host Sim forbidden on droplets)" fi if echo "$rendered" | grep -qE 'DESIGN_FORCE_SIM:[[:space:]]*["'\'']?(1|true|TRUE|yes)["'\'']?'; then - fail "$env_file enables DESIGN_FORCE_SIM (Docker-only on droplets)" - fi - # Screenshot Chromium isolation: must force egress proxy (not empty / direct). - if ! echo "$rendered" | grep -qE 'DESIGN_SCREENSHOT_PROXY:[[:space:]]*http://design-egress-proxy:8094'; then - fail "$env_file master render missing DESIGN_SCREENSHOT_PROXY=http://design-egress-proxy:8094" + fail "$env_file enables DESIGN_FORCE_SIM (retired; must not ship)" fi + for sim_var in RELEARN_FORCE_SIM RELEARN_T2I_FORCE_SIM RELEARN_MM_FORCE_SIM; do + if echo "$rendered" | grep -qE "${sim_var}:[[:space:]]*[\"']?(1|true|TRUE|yes)[\"']?"; then + fail "$env_file enables $sim_var (sim is local-only; must not ship on droplets)" + fi + done done echo "OK: staging/prod do not enable host SimSandbox" -echo "OK: staging/prod force screenshot Chromium through design-egress-proxy" -# --- prism-challenge + design-challenge present in default --- +# --- all four live challenges present in default; design/prism retired --- default_services=$(render \ -f docker-compose.yml \ config --services) -if ! echo "$default_services" | grep -qx "prism-challenge"; then - fail "prism-challenge not in default compose" -fi -if ! echo "$default_services" | grep -qx "design-challenge"; then - fail "design-challenge not in default compose" -fi -if ! echo "$default_services" | grep -qx "design-egress-proxy"; then - fail "design-egress-proxy not in default compose" -fi -echo "OK: prism-challenge + design-challenge + design-egress-proxy in default compose" +for required in relearn-challenge relearn-t2i-challenge relearn-mm-challenge bounty-challenge; do + if ! echo "$default_services" | grep -qx "$required"; then + fail "$required not in default compose" + fi +done +for retired in prism-challenge design-challenge design-egress-proxy; do + if echo "$default_services" | grep -qx "$retired"; then + fail "retired $retired still in default compose" + fi +done +echo "OK: relearn, relearn-t2i, relearn-mm, bounty in default compose; design/prism retired" # --- no fake chain backend survives anywhere in the matrix --- for env_file in deploy/compose/env-staging.yml deploy/compose/env-prod.yml; do @@ -211,14 +217,17 @@ echo "$local_services" | grep -qx "gateway" \ || fail "env-local master stack does not render gateway" echo "$local_services" | grep -qx "validator" \ || fail "env-local master stack does not render co-located validator" -echo "$local_services" | grep -qx "prism-challenge" \ - || fail "env-local master stack does not render prism-challenge" -echo "$local_services" | grep -qx "design-challenge" \ - || fail "env-local master stack does not render design-challenge" -echo "$local_services" | grep -qx "design-egress-proxy" \ - || fail "env-local master stack does not render design-egress-proxy" -echo "$local_rendered" | grep -qE 'published: "?28093"?' \ - || fail "env-local does not publish design-challenge on 28093" +for required in relearn-challenge relearn-t2i-challenge relearn-mm-challenge bounty-challenge; do + echo "$local_services" | grep -qx "$required" \ + || fail "env-local master stack does not render $required" +done +for probe in 28095:relearn-challenge 28096:bounty-challenge \ + 28097:relearn-t2i-challenge 28098:relearn-mm-challenge; do + port=${probe%%:*} + svc=${probe#*:} + echo "$local_rendered" | grep -qE "published: \"?${port}\"?" \ + || fail "env-local does not publish $svc on $port" +done for banned in agent-challenge hypertraining-challenge miner-agent miner-socket-proxy base-agent; do if echo "$local_services" | grep -qx "$banned"; then fail "removed service still rendered: $banned" @@ -230,6 +239,6 @@ for banned in agent-challenge hypertraining-challenge miner-agent miner-socket-p fail "removed service still in default compose: $banned" fi done -echo "OK: env-local preserves testnet/541; design on 28093; removed agent/hypertraining/miner services" +echo "OK: env-local preserves testnet/541; challenges on 28095-28098; removed agent/hypertraining/miner services" echo "assert-compose-matrix: all checks passed" diff --git a/deploy/scripts/local-e2e.sh b/deploy/scripts/local-e2e.sh index 9c0f104fe..3cfce1f61 100755 --- a/deploy/scripts/local-e2e.sh +++ b/deploy/scripts/local-e2e.sh @@ -38,8 +38,10 @@ TUNNEL_CONFIG="$ROOT/.local/cloudflared-quick.yml" COMPOSE_PROJECT="${COMPOSE_PROJECT_NAME:-base}" GATEWAY_HOST_PORT="${LOCAL_GATEWAY_HOST_PORT:-8080}" VALIDATOR_HOST_PORT="${LOCAL_VALIDATOR_HOST_PORT:-28080}" -PRISM_HOST_PORT="${LOCAL_PRISM_HOST_PORT:-28092}" -DESIGN_HOST_PORT="${LOCAL_DESIGN_HOST_PORT:-28093}" +RELEARN_HOST_PORT="${LOCAL_RELEARN_HOST_PORT:-28095}" +BOUNTY_HOST_PORT="${LOCAL_BOUNTY_HOST_PORT:-28096}" +RELEARN_T2I_HOST_PORT="${LOCAL_RELEARN_T2I_HOST_PORT:-28097}" +RELEARN_MM_HOST_PORT="${LOCAL_RELEARN_MM_HOST_PORT:-28098}" BASE_SECRETS_DIR="${BASE_SECRETS_DIR:-${HOME}/.base-secrets}" # Default public-only hotkey for smoke (same placeholder as gateway.env.example usage). @@ -79,8 +81,8 @@ Prerequisites: - deploy/env/*.env via materialize-env.sh (examples OK for smoke) - For --live: base-owner wallet under deploy/secrets/wallets/ (btcli layout) - For --live on-chain weight submit: base-validator wallet - - Secret files: gateway_sk (seal), prism_sk/design_sk (challenge leaf sigs). - Smoke prefers ~/.base-secrets/challenge-*.sk when pubs match trust root; + - Secret files: gateway_sk (seal), relearn_sk (challenge leaf sigs). + Smoke prefers ~/.base-secrets/challenge-relearn.sk when pubs match trust root; otherwise mints and rebuilds the local trust root. Gateway wallet is NOT required to serve sealed weights. @@ -89,18 +91,14 @@ Environment knobs (optional): BASE_SECRETS_DIR Challenge/owner age/sk sources (default: ~/.base-secrets) BASE_DOCKER_BUILD_FROM prebuilt|source (default: prebuilt) LOCAL_GATEWAY_HOTKEY Override smoke public hotkey (64 hex) - LOCAL_PRISM_FORCE_SIM default true - LOCAL_DESIGN_FORCE_SIM default true - LOCAL_DESIGN_EGRESS_SIM default true - LOCAL_DESIGN_SIM_STAGE_DELAY_MS ms pause after each design stage (default 0) - LOCAL_PRISM_SIM_STAGE_DELAY_MS ms pause after each prism stage (default 0) + LOCAL_RELEARN_FORCE_SIM default true LOCAL_ATTEST_VERIFIER default mock_ok Wallet roles: - Gateway owner wallet / REQUIRE_OWNER: master-only identity check (live). Not required for POST /v1/weights/raw, admin seal, or GET /v1/weights/latest. - gateway_sk: mini-secret for bundle seal signatures (required for seal). - - prism_sk / design_sk: challenge leaf signatures (must match trust root pubs). + - relearn_sk: challenge leaf signatures (must match trust root pubs). - Validator wallet: on-chain weight submit only (not weights/latest serving). EOF @@ -220,8 +218,10 @@ start_tunnel() { ensure_env_files() { # Challenge env files are required by compose (BASE_DATABASE_URL → Postgres). if [[ ! -f deploy/env/postgres.env || ! -f deploy/env/gateway.env \ - || ! -f deploy/env/validator.env || ! -f deploy/env/design-challenge.env \ - || ! -f deploy/env/prism-challenge.env ]]; then + || ! -f deploy/env/validator.env || ! -f deploy/env/relearn-challenge.env \ + || ! -f deploy/env/relearn-t2i-challenge.env \ + || ! -f deploy/env/relearn-mm-challenge.env \ + || ! -f deploy/env/bounty-challenge.env ]]; then log "materializing deploy/env/*.env from examples" ./deploy/scripts/materialize-env.sh fi @@ -230,7 +230,8 @@ ensure_env_files() { local url url="$(database_url_from_postgres_env)" for f in deploy/env/gateway.env deploy/env/validator.env \ - deploy/env/design-challenge.env deploy/env/prism-challenge.env; do + deploy/env/relearn-challenge.env deploy/env/relearn-t2i-challenge.env \ + deploy/env/relearn-mm-challenge.env deploy/env/bounty-challenge.env; do if grep -q '^BASE_DATABASE_URL=' "$f" 2>/dev/null; then sed -i "s|^BASE_DATABASE_URL=.*|BASE_DATABASE_URL=${url}|" "$f" else @@ -241,9 +242,10 @@ ensure_env_files() { } ensure_state_dirs() { - mkdir -p "$STATE_DIR/design/staging" - # Design sandbox binds must be writable by uid 65532 inside containers. - chmod 777 "$STATE_DIR/design/staging" 2>/dev/null || true + for area in relearn relearn-t2i relearn-mm; do + mkdir -p "$STATE_DIR/$area" + chmod 777 "$STATE_DIR/$area" 2>/dev/null || true + done } # Create a 32-byte secret file if missing. live mode refuses to invent wallets. @@ -342,20 +344,75 @@ PY ensure_secrets() { ensure_secret_file deploy/secrets/gateway_sk "gateway seal mini-secret" ensure_challenge_sk_aligned \ - deploy/secrets/prism_sk prism \ - "${BASE_SECRETS_DIR}/challenge-prism.sk" + deploy/secrets/relearn_sk relearn \ + "${BASE_SECRETS_DIR}/challenge-relearn.sk" + ensure_challenge_sk_aligned \ + deploy/secrets/relearn_t2i_sk relearn-t2i \ + "${BASE_SECRETS_DIR}/challenge-relearn-t2i.sk" + ensure_challenge_sk_aligned \ + deploy/secrets/relearn_mm_sk relearn-mm \ + "${BASE_SECRETS_DIR}/challenge-relearn-mm.sk" ensure_challenge_sk_aligned \ - deploy/secrets/design_sk design \ - "${BASE_SECRETS_DIR}/challenge-design.sk" - mkdir -p deploy/secrets/lium deploy/secrets/openrouter deploy/secrets/design + deploy/secrets/bounty_sk bounty \ + "${BASE_SECRETS_DIR}/challenge-bounty.sk" + mkdir -p deploy/secrets/lium deploy/secrets/relearn \ + deploy/secrets/relearn-t2i deploy/secrets/relearn-mm deploy/secrets/bounty # Touch placeholders so compose bind-mounts stay files/dirs of the right kind. [[ -e deploy/secrets/lium/api_key ]] || : >deploy/secrets/lium/api_key [[ -e deploy/secrets/lium/ssh_ed25519 ]] || : >deploy/secrets/lium/ssh_ed25519 [[ -e deploy/secrets/lium/ssh_ed25519.pub ]] || : >deploy/secrets/lium/ssh_ed25519.pub - [[ -e deploy/secrets/openrouter/api_key ]] || : >deploy/secrets/openrouter/api_key - [[ -e deploy/secrets/design/annotator_tokens ]] || : >deploy/secrets/design/annotator_tokens - chown 65532:65532 deploy/secrets/design/annotator_tokens 2>/dev/null || true - chmod 0400 deploy/secrets/design/annotator_tokens 2>/dev/null || true + for area in relearn relearn-t2i relearn-mm bounty; do + [[ -e "deploy/secrets/$area/admin_tokens" ]] || : >"deploy/secrets/$area/admin_tokens" + done + [[ -e deploy/secrets/bounty/session_secret ]] || dd if=/dev/urandom bs=32 count=1 status=none of=deploy/secrets/bounty/session_secret + # Relearn T2I refuses submissions without holdout records matching the pin + # commitment. Local smoke materializes them from the documented dev salt. + ensure_t2i_holdout + local guarded=(deploy/secrets/bounty/session_secret) + for area in relearn relearn-t2i relearn-mm bounty; do + guarded+=("deploy/secrets/$area/admin_tokens") + done + [[ -e deploy/secrets/relearn-t2i/holdout.json ]] \ + && guarded+=(deploy/secrets/relearn-t2i/holdout.json) + chown 65532:65532 "${guarded[@]}" 2>/dev/null || true + chmod 0400 "${guarded[@]}" 2>/dev/null || true +} + +# Materialize Relearn T2I holdout prompt records for a local run. +# +# Needs the Qwen-Image-Bench prompt file. Without it the service still boots and +# answers /health but 503s submissions, which is the intended fail-closed state. +ensure_t2i_holdout() { + [[ -s deploy/secrets/relearn-t2i/holdout.json ]] && return 0 + local bench="${RELEARN_T2I_BENCH_FILE:-${BASE_SECRETS_DIR}/qwen_image_bench_hf_v0518.jsonl}" + if [[ ! -f "$bench" ]]; then + log "note: no Qwen-Image-Bench file at $bench; relearn-t2i will 503 submissions" + log " fetch it from the Qwen/Qwen-Image-Bench dataset or set RELEARN_T2I_BENCH_FILE" + return 0 + fi + local -a excludes=() + local id + while read -r id; do + excludes+=(--exclude "$id") + done < <(public_prompt_ids) + log "generating relearn-t2i holdout records from $bench" + cargo run -q -p xtask -- relearn-t2i-holdout \ + --bench "$bench" \ + --salt "${RELEARN_T2I_HOLDOUT_SALT:-cortex-t2i-dev-holdout-v0}" \ + --size 40 \ + "${excludes[@]}" \ + --out "$ROOT/deploy/secrets/relearn-t2i/holdout.json" +} + +# Public prompt ids from the committed T2I pin, one per line. +public_prompt_ids() { + python3 -c ' +import sys, tomllib +from pathlib import Path +doc = tomllib.loads(Path(sys.argv[1]).read_text()) +for i in doc["prompts"]["public_ids"]: + print(i) +' "$ROOT/config/relearn-t2i-pin.toml" } # Ephemeral owner-signed trust root for local stacks (prod owner key is not required). @@ -363,31 +420,35 @@ ensure_secrets() { # /etc/base/config inside gateway/validator. BASE_TRUST_ROOT_DIR must be the # *in-container* path — a host absolute path is invisible to containers. # -# Challenge public_keys ALWAYS come from deploy/secrets/{prism,design}_sk so +# Challenge public_keys ALWAYS come from deploy/secrets/*_sk so # leaf signatures verify. A stale trust root with mismatched pubs is rebuilt. ensure_local_trust_root() { local dir="$ROOT/.local/trust-root" mkdir -p "$dir" export BASE_TRUST_ROOT_DIR=/etc/base/config - local prism_pk design_pk - prism_pk="$(pubkey_hex_from_sk_file "$ROOT/deploy/secrets/prism_sk")" - design_pk="$(pubkey_hex_from_sk_file "$ROOT/deploy/secrets/design_sk")" + local relearn_pk t2i_pk mm_pk bounty_pk + relearn_pk="$(pubkey_hex_from_sk_file "$ROOT/deploy/secrets/relearn_sk")" + t2i_pk="$(pubkey_hex_from_sk_file "$ROOT/deploy/secrets/relearn_t2i_sk")" + mm_pk="$(pubkey_hex_from_sk_file "$ROOT/deploy/secrets/relearn_mm_sk")" + bounty_pk="$(pubkey_hex_from_sk_file "$ROOT/deploy/secrets/bounty_sk")" local need_rebuild=0 if [[ ! -f "$dir/challenges.toml" || ! -f "$dir/challenges.toml.sig" || ! -f "$dir/owner.pubkey" ]]; then need_rebuild=1 else - python3 - "$dir/challenges.toml" "$prism_pk" "$design_pk" <<'PY' || need_rebuild=1 + python3 - "$dir/challenges.toml" "$relearn_pk" "$t2i_pk" "$mm_pk" "$bounty_pk" <<'PY' || need_rebuild=1 import sys, tomllib from pathlib import Path doc = tomllib.loads(Path(sys.argv[1]).read_text()) rows = {c["id"]: c.get("public_key", "").lower() for c in doc.get("challenges", [])} -want = {"prism": sys.argv[2].lower(), "design": sys.argv[3].lower()} -for cid, pk in want.items(): - if rows.get(cid) != pk: - sys.exit(1) -sys.exit(0) +want = { + "relearn": sys.argv[2].lower(), + "relearn-t2i": sys.argv[3].lower(), + "relearn-mm": sys.argv[4].lower(), + "bounty": sys.argv[5].lower(), +} +sys.exit(0 if rows == want else 1) PY fi @@ -412,16 +473,23 @@ PY --out-secret "$dir/owner.age" \ --age-recipient "$recip" fi - python3 - "$dir/challenges.toml" "$prism_pk" "$design_pk" <<'PY2' + python3 - "$dir/challenges.toml" "$relearn_pk" "$t2i_pk" "$mm_pk" "$bounty_pk" <<'PY2' import pathlib, sys -prism_pk, design_pk = sys.argv[2], sys.argv[3] -text = ( - "version = 1\nintroduced_epoch = 0\n\n" - f'[[challenges]]\nid = "prism"\npublic_key = "{prism_pk}"\n' - "emission_share_bps = 10000\npolicy = \"all_metagraph_hotkeys\"\n\n" - f'[[challenges]]\nid = "design"\npublic_key = "{design_pk}"\n' - "emission_share_bps = 0\npolicy = \"all_metagraph_hotkeys\"\n\n" -) + +# Mirrors config/challenges.toml. Shares must sum to 10000 or the validator +# flags an emission-share mismatch (D23). +rows = [ + ("relearn", sys.argv[2], 4000), + ("relearn-t2i", sys.argv[3], 1500), + ("relearn-mm", sys.argv[4], 1500), + ("bounty", sys.argv[5], 3000), +] +text = "version = 1\nintroduced_epoch = 0\n" +for cid, pk, bps in rows: + text += ( + f'\n[[challenges]]\nid = "{cid}"\npublic_key = "{pk}"\n' + f'emission_share_bps = {bps}\npolicy = "all_metagraph_hotkeys"\n' + ) pathlib.Path(sys.argv[1]).write_text(text) PY2 # measurements: empty allowlist (base-agent CVM path removed) @@ -477,7 +545,8 @@ port_in_use() { check_host_ports() { local p - for p in "$GATEWAY_HOST_PORT" "$VALIDATOR_HOST_PORT" "$PRISM_HOST_PORT" "$DESIGN_HOST_PORT"; do + for p in "$GATEWAY_HOST_PORT" "$VALIDATOR_HOST_PORT" "$RELEARN_HOST_PORT" \ + "$RELEARN_T2I_HOST_PORT" "$RELEARN_MM_HOST_PORT"; do if port_in_use "$p"; then # Allow re-bind when this compose project already publishes the port. if docker ps --format '{{.Names}} {{.Ports}}' \ @@ -494,16 +563,13 @@ export_mode_env() { export BASE_STATE_DIR="$STATE_DIR" export BASE_DOCKER_BUILD_FROM="${BASE_DOCKER_BUILD_FROM:-prebuilt}" export BASE_GATEWAY_ENDPOINT="${BASE_GATEWAY_ENDPOINT:-http://gateway:8080}" - export LOCAL_PRISM_FORCE_SIM="${LOCAL_PRISM_FORCE_SIM:-true}" - export LOCAL_DESIGN_FORCE_SIM="${LOCAL_DESIGN_FORCE_SIM:-true}" - # Host SimSandbox opt-in for local overlay only (bin refuse on prod/mainnet). - export BASE_ALLOW_HOST_SIM="${BASE_ALLOW_HOST_SIM:-1}" - export LOCAL_DESIGN_EGRESS_SIM="${LOCAL_DESIGN_EGRESS_SIM:-true}" + export LOCAL_RELEARN_FORCE_SIM="${LOCAL_RELEARN_FORCE_SIM:-true}" export LOCAL_ATTEST_VERIFIER="${LOCAL_ATTEST_VERIFIER:-mock_ok}" export LOCAL_GATEWAY_HOST_PORT="$GATEWAY_HOST_PORT" export LOCAL_VALIDATOR_HOST_PORT="$VALIDATOR_HOST_PORT" - export LOCAL_PRISM_HOST_PORT="$PRISM_HOST_PORT" - export LOCAL_DESIGN_HOST_PORT="$DESIGN_HOST_PORT" + export LOCAL_RELEARN_HOST_PORT="$RELEARN_HOST_PORT" + export LOCAL_RELEARN_T2I_HOST_PORT="$RELEARN_T2I_HOST_PORT" + export LOCAL_RELEARN_MM_HOST_PORT="$RELEARN_MM_HOST_PORT" # Align app DATABASE_URL with whatever postgres.env will create (avoids # stale gateway.env pointing at a different database name). if [[ -z "${LOCAL_DATABASE_URL:-}" && -f "$ROOT/deploy/env/postgres.env" ]]; then @@ -582,15 +648,19 @@ wait_all_health() { local soft_wait=30 local saved="$WAIT_SECS" WAIT_SECS="$soft_wait" - wait_health prism-challenge "http://127.0.0.1:${PRISM_HOST_PORT}/health" || \ - log "warning: prism not healthy (continuing; try BASE_DOCKER_BUILD_FROM=source)" - wait_health design-challenge "http://127.0.0.1:${DESIGN_HOST_PORT}/health" || \ - log "warning: design not healthy (continuing; try BASE_DOCKER_BUILD_FROM=source)" + wait_health relearn-challenge "http://127.0.0.1:${RELEARN_HOST_PORT}/health" || \ + log "warning: relearn not healthy (continuing; try BASE_DOCKER_BUILD_FROM=source)" + wait_health relearn-t2i-challenge "http://127.0.0.1:${RELEARN_T2I_HOST_PORT}/health" || \ + log "warning: relearn-t2i not healthy (continuing; try BASE_DOCKER_BUILD_FROM=source)" + wait_health relearn-mm-challenge "http://127.0.0.1:${RELEARN_MM_HOST_PORT}/health" || \ + log "warning: relearn-mm not healthy (continuing; try BASE_DOCKER_BUILD_FROM=source)" + wait_health bounty-challenge "http://127.0.0.1:${BOUNTY_HOST_PORT}/health" || \ + log "warning: bounty not healthy (continuing; try BASE_DOCKER_BUILD_FROM=source)" WAIT_SECS="$saved" } # Prove seal→serve without a gateway owner wallet: signed leaves + admin seal + -# GET /v1/weights/latest must be 200. Uses prism_sk + gateway_sk only. +# GET /v1/weights/latest must be 200. Uses relearn_sk + gateway_sk only. probe_weights_latest() { if [[ "$DO_WEIGHTS_SMOKE" -ne 1 ]]; then log "skipping weights seal smoke (--no-weights-smoke)" @@ -614,8 +684,8 @@ probe_weights_latest() { log "running weights-smoke (leaf submit → admin/seal → weights/latest)" ./target/release/weights-smoke \ --gateway "$gw" \ - --challenge-sk "$ROOT/deploy/secrets/prism_sk" \ - --challenge-id prism \ + --challenge-sk "$ROOT/deploy/secrets/relearn_sk" \ + --challenge-id relearn \ --netuid "$netuid" \ --chain-endpoint "$endpoint" \ | tee /tmp/local-e2e-weights-latest.json \ @@ -639,8 +709,10 @@ print_summary() { Internal (compose network): gateway: http://gateway:8080 validator probe: http://127.0.0.1:${VALIDATOR_HOST_PORT}/healthz - prism: http://127.0.0.1:${PRISM_HOST_PORT}/health - design: http://127.0.0.1:${DESIGN_HOST_PORT}/health + relearn: http://127.0.0.1:${RELEARN_HOST_PORT}/health + relearn-t2i: http://127.0.0.1:${RELEARN_T2I_HOST_PORT}/health + relearn-mm: http://127.0.0.1:${RELEARN_MM_HOST_PORT}/health + bounty: http://127.0.0.1:${BOUNTY_HOST_PORT}/health EOF if [[ -n "$pub" ]]; then diff --git a/deploy/scripts/materialize-env.sh b/deploy/scripts/materialize-env.sh index 9bbd9328e..2fa069ed6 100755 --- a/deploy/scripts/materialize-env.sh +++ b/deploy/scripts/materialize-env.sh @@ -38,8 +38,8 @@ materialize_one() { } # Challenge env files carry BASE_DATABASE_URL (must match postgres.env). -# Without them, compose refuses to start design/prism (no silent memory store). -for svc in postgres validator gateway updater design-challenge prism-challenge design-egress-proxy; do +# Without them, compose refuses to start challenges (required env_file). +for svc in postgres validator gateway updater relearn-challenge relearn-t2i-challenge relearn-mm-challenge bounty-challenge; do materialize_one "$svc" done diff --git a/deploy/scripts/promote.sh b/deploy/scripts/promote.sh index 82712ca3b..d46ac85d4 100755 --- a/deploy/scripts/promote.sh +++ b/deploy/scripts/promote.sh @@ -55,7 +55,7 @@ done [[ -n "$ENV_NAME" ]] || die "--env staging|prod required" case "$ENV_NAME" in staging|prod) ;; *) die "env must be staging|prod" ;; esac -case "$SERVICE" in validator|gateway|updater|prism-challenge|design-challenge) ;; *) die "service must be validator|gateway|updater|prism-challenge|design-challenge" ;; esac +case "$SERVICE" in validator|gateway|updater|relearn-challenge|relearn-t2i-challenge|relearn-mm-challenge|bounty-challenge) ;; *) die "service must be validator|gateway|updater|relearn-challenge|relearn-t2i-challenge|relearn-mm-challenge|bounty-challenge" ;; esac PIN_PATH="$(pin_path_for_env "$ROOT" "$ENV_NAME")" PROD_PATH="$(prod_pin_path "$ROOT")" diff --git a/deploy/scripts/record-image-digests.sh b/deploy/scripts/record-image-digests.sh index 506145493..5c0e708fc 100755 --- a/deploy/scripts/record-image-digests.sh +++ b/deploy/scripts/record-image-digests.sh @@ -3,7 +3,7 @@ # # record-image-digests.sh [--out deploy/digests/.json] [image:tag ...] # -# Default images: validator:0.1.0 gateway:0.1.0 updater:0.1.0 prism-challenge:0.1.0 design-challenge:0.1.0 +# Default images: validator:0.1.0 gateway:0.1.0 updater:0.1.0 relearn-challenge:0.1.0 relearn-t2i-challenge:0.1.0 relearn-mm-challenge:0.1.0 bounty-challenge:0.1.0 # Writes JSON: # { "commit_sha", "created_at", "images": { "validator": { "id", "digest", "repo_digest", "tag" } } } set -euo pipefail @@ -27,7 +27,7 @@ while [[ $# -gt 0 ]]; do done if [[ ${#IMAGES[@]} -eq 0 ]]; then - IMAGES=(validator:0.1.0 gateway:0.1.0 updater:0.1.0 prism-challenge:0.1.0 design-challenge:0.1.0) + IMAGES=(validator:0.1.0 gateway:0.1.0 updater:0.1.0 relearn-challenge:0.1.0 relearn-t2i-challenge:0.1.0 relearn-mm-challenge:0.1.0 bounty-challenge:0.1.0) fi COMMIT="$(git -C "$ROOT" rev-parse HEAD)" diff --git a/deploy/scripts/register-challenge-backends.sh b/deploy/scripts/register-challenge-backends.sh index d3d2e14a3..8da58b1c5 100755 --- a/deploy/scripts/register-challenge-backends.sh +++ b/deploy/scripts/register-challenge-backends.sh @@ -6,7 +6,6 @@ # # Usage: # GATEWAY_URL=http://127.0.0.1:8080 ./deploy/scripts/register-challenge-backends.sh -# # or via docker network from the host after compose up: # ./deploy/scripts/register-challenge-backends.sh --compose set -euo pipefail @@ -14,16 +13,20 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$ROOT" GATEWAY_URL="${GATEWAY_URL:-http://127.0.0.1:8080}" -PRISM_URL="${PRISM_BACKEND_URL:-http://prism-challenge:8092}" -DESIGN_URL="${DESIGN_BACKEND_URL:-http://design-challenge:8093}" +RELEARN_URL="${RELEARN_BACKEND_URL:-http://relearn-challenge:8095}" +RELEARN_T2I_URL="${RELEARN_T2I_BACKEND_URL:-http://relearn-t2i-challenge:8097}" +RELEARN_MM_URL="${RELEARN_MM_BACKEND_URL:-http://relearn-mm-challenge:8098}" +BOUNTY_URL="${BOUNTY_BACKEND_URL:-http://bounty-challenge:8096}" COMPOSE_MODE=0 while [[ $# -gt 0 ]]; do case "$1" in --compose) COMPOSE_MODE=1; shift ;; --gateway-url) GATEWAY_URL="$2"; shift 2 ;; - --prism-url) PRISM_URL="$2"; shift 2 ;; - --design-url) DESIGN_URL="$2"; shift 2 ;; + --relearn-url) RELEARN_URL="$2"; shift 2 ;; + --relearn-t2i-url) RELEARN_T2I_URL="$2"; shift 2 ;; + --relearn-mm-url) RELEARN_MM_URL="$2"; shift 2 ;; + --bounty-url) BOUNTY_URL="$2"; shift 2 ;; -h|--help) sed -n '2,12p' "$0" exit 0 @@ -33,7 +36,6 @@ while [[ $# -gt 0 ]]; do done resolve_admin_token() { - # Prefer env token; else file (prod: deploy/secrets/gateway_admin_token). if [[ -n "${BASE_GATEWAY_ADMIN_TOKEN:-}" ]]; then printf '%s' "${BASE_GATEWAY_ADMIN_TOKEN}" return 0 @@ -85,17 +87,18 @@ register_one() { esac } -register_one prism "$PRISM_URL" -register_one design "$DESIGN_URL" +register_one relearn "$RELEARN_URL" +register_one relearn-t2i "$RELEARN_T2I_URL" +register_one relearn-mm "$RELEARN_MM_URL" +register_one bounty "$BOUNTY_URL" -# Smoke the proxy path (health, not healthz — challenge services use /health). -if [[ "$COMPOSE_MODE" -eq 1 ]]; then - docker compose -f docker-compose.yml -f deploy/compose/role-master.yml \ - exec -T gateway curl -fsS -m 5 http://127.0.0.1:8080/challenge/prism/health >/dev/null - docker compose -f docker-compose.yml -f deploy/compose/role-master.yml \ - exec -T gateway curl -fsS -m 5 http://127.0.0.1:8080/challenge/design/health >/dev/null -else - curl -fsS -m 5 "${GATEWAY_URL%/}/challenge/prism/health" >/dev/null - curl -fsS -m 5 "${GATEWAY_URL%/}/challenge/design/health" >/dev/null -fi -echo "challenge proxy health: ok (prism + design)" +for challenge_id in relearn relearn-t2i relearn-mm bounty; do + if [[ "$COMPOSE_MODE" -eq 1 ]]; then + docker compose -f docker-compose.yml -f deploy/compose/role-master.yml \ + exec -T gateway curl -fsS -m 5 \ + "http://127.0.0.1:8080/challenge/${challenge_id}/health" >/dev/null + else + curl -fsS -m 5 "${GATEWAY_URL%/}/challenge/${challenge_id}/health" >/dev/null + fi +done +echo "challenge proxy health: ok (relearn, relearn-t2i, relearn-mm, bounty)" diff --git a/deploy/scripts/remote-deploy.sh b/deploy/scripts/remote-deploy.sh index aaa789137..9b7f400a1 100755 --- a/deploy/scripts/remote-deploy.sh +++ b/deploy/scripts/remote-deploy.sh @@ -31,7 +31,7 @@ REMOTE_DIR="${BASE_REMOTE_DIR:-/opt/base}" # bind source and the container's BASE_VERIFY_WORK_ROOT byte-for-byte. STATE_ROOT="${BASE_STATE_DIR:-/var/lib/base}" GHCR_PREFIX="${BASE_GHCR_PREFIX:-ghcr.io/baseintelligence/base}" -PIN_SERVICES=(validator gateway updater prism-challenge design-challenge) +PIN_SERVICES=(validator gateway updater relearn-challenge relearn-t2i-challenge relearn-mm-challenge bounty-challenge) SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new) if [[ -n "${BASE_SSH_IDENTITY:-}" ]]; then SSH_OPTS+=(-i "$BASE_SSH_IDENTITY") @@ -160,8 +160,8 @@ fi echo "remote-deploy: rsync tree" if [[ "$BUILD_FROM" == "prebuilt" ]]; then - for b in validator gateway updater prism-challenge design-challenge design-egress-proxy challenge-review; do - [[ -x "$ROOT/target/release/$b" ]] || die "missing prebuilt binary target/release/$b — run: cargo build --release --features validator-bin/dcap -p validator-bin -p gateway-bin -p updater-bin -p prism-challenge-bin -p design-challenge-bin -p design-egress-proxy-bin && cargo build --release -p challenge-review-bin" + for b in validator gateway updater relearn-challenge relearn-t2i-challenge relearn-mm-challenge bounty-challenge; do + [[ -x "$ROOT/target/release/$b" ]] || die "missing prebuilt binary target/release/$b — run: cargo build --release --features validator-bin/dcap -p validator-bin -p gateway-bin -p updater-bin -p relearn-challenge-bin -p relearn-t2i-challenge-bin -p relearn-mm-challenge-bin -p bounty-challenge-bin" done fi @@ -182,50 +182,52 @@ rsync -az --delete \ "$ROOT/" "$HOST:$REMOTE_DIR/" # Ensure secrets dirs exist (empty OK if not bootstrapped). -# deploy/secrets/lium is bind-mounted by prism-challenge, so it must be a real +# deploy/secrets/lium is bind-mounted by relearn-challenge, so it must be a real # directory with real files: compose would otherwise create directories where # the container expects files. -# Same footgun for file mounts: if prism_sk / design_sk are missing, Docker -# creates *directories* at those paths and the challenge bins fail with +# Same footgun for file mounts: if relearn_sk is missing, Docker +# creates *directories* at those paths and the challenge bin fails with # "Is a directory" / "secret file missing". Materialize empty files when # absent; if a directory already poisoned the path, replace it with a file. ssh_h "mkdir -p '$REMOTE_DIR/deploy/env' '$REMOTE_DIR/deploy/secrets/lium' \ - '$REMOTE_DIR/deploy/secrets/openrouter' '$REMOTE_DIR/deploy/secrets/design' \ - '$REMOTE_DIR/deploy/secrets/github' '$REMOTE_DIR/deploy/secrets/huggingface' \ + '$REMOTE_DIR/deploy/secrets/relearn' \ + '$REMOTE_DIR/deploy/secrets/relearn-t2i' \ + '$REMOTE_DIR/deploy/secrets/relearn-mm' \ + '$REMOTE_DIR/deploy/secrets/bounty' \ '$REMOTE_DIR/deploy/secrets/wallets' \ && chmod 700 '$REMOTE_DIR/deploy/secrets' '$REMOTE_DIR/deploy/secrets/lium' \ && for f in api_key ssh_ed25519 ssh_ed25519.pub; do \ [ -e '$REMOTE_DIR/deploy/secrets/lium/'\$f ] || : > '$REMOTE_DIR/deploy/secrets/lium/'\$f; \ done \ - && [ -e '$REMOTE_DIR/deploy/secrets/openrouter/api_key' ] || : > '$REMOTE_DIR/deploy/secrets/openrouter/api_key' \ - && [ -e '$REMOTE_DIR/deploy/secrets/design/annotator_tokens' ] || : > '$REMOTE_DIR/deploy/secrets/design/annotator_tokens' \ - && [ -e '$REMOTE_DIR/deploy/secrets/github/token' ] || : > '$REMOTE_DIR/deploy/secrets/github/token' \ - && [ -e '$REMOTE_DIR/deploy/secrets/huggingface/token' ] || : > '$REMOTE_DIR/deploy/secrets/huggingface/token' \ - && for sk in prism_sk design_sk; do \ + && [ -e '$REMOTE_DIR/deploy/secrets/relearn/admin_tokens' ] || : > '$REMOTE_DIR/deploy/secrets/relearn/admin_tokens' \ + && [ -e '$REMOTE_DIR/deploy/secrets/relearn-t2i/admin_tokens' ] || : > '$REMOTE_DIR/deploy/secrets/relearn-t2i/admin_tokens' \ + && [ -e '$REMOTE_DIR/deploy/secrets/relearn-mm/admin_tokens' ] || : > '$REMOTE_DIR/deploy/secrets/relearn-mm/admin_tokens' \ + && [ -e '$REMOTE_DIR/deploy/secrets/bounty/admin_tokens' ] || : > '$REMOTE_DIR/deploy/secrets/bounty/admin_tokens' \ + && [ -e '$REMOTE_DIR/deploy/secrets/bounty/session_secret' ] || : > '$REMOTE_DIR/deploy/secrets/bounty/session_secret' \ + && for sk in relearn_sk relearn_t2i_sk relearn_mm_sk bounty_sk; do \ p='$REMOTE_DIR/deploy/secrets/'\$sk; \ if [ -d \"\$p\" ]; then rm -rf \"\$p\"; fi; \ [ -e \"\$p\" ] || : > \"\$p\"; \ chmod 400 \"\$p\"; chown 65532:65532 \"\$p\"; \ done \ && chmod 400 '$REMOTE_DIR/deploy/secrets/lium/'* \ - '$REMOTE_DIR/deploy/secrets/openrouter/api_key' \ - '$REMOTE_DIR/deploy/secrets/design/annotator_tokens' \ - '$REMOTE_DIR/deploy/secrets/github/token' \ - '$REMOTE_DIR/deploy/secrets/huggingface/token' \ + '$REMOTE_DIR/deploy/secrets/relearn/admin_tokens' \ + '$REMOTE_DIR/deploy/secrets/relearn-t2i/admin_tokens' \ + '$REMOTE_DIR/deploy/secrets/relearn-mm/admin_tokens' \ + '$REMOTE_DIR/deploy/secrets/bounty/admin_tokens' \ + '$REMOTE_DIR/deploy/secrets/bounty/session_secret' \ && chown -R 65532:65532 '$REMOTE_DIR/deploy/secrets/lium' \ - '$REMOTE_DIR/deploy/secrets/openrouter' \ - '$REMOTE_DIR/deploy/secrets/design' \ - '$REMOTE_DIR/deploy/secrets/github' \ - '$REMOTE_DIR/deploy/secrets/huggingface' \ + '$REMOTE_DIR/deploy/secrets/relearn' \ + '$REMOTE_DIR/deploy/secrets/relearn-t2i' \ + '$REMOTE_DIR/deploy/secrets/relearn-mm' \ + '$REMOTE_DIR/deploy/secrets/bounty' \ && chmod -R a-w '$REMOTE_DIR/deploy/secrets/wallets' 2>/dev/null; \ chown -R 65532:65532 '$REMOTE_DIR/deploy/secrets/wallets' 2>/dev/null; true" -# Design sandbox staging root. The challenge container stages bind sources -# here and hands them to the host's Docker daemon, which resolves bind sources -# on the host filesystem, so the path must exist on the host with the container -# uid as owner, or 'docker compose' would auto-create it root-owned and every -# run would fail on staging I/O. -ssh_h "install -d -m 0775 -o 65532 -g 65532 '$STATE_ROOT/design/staging'" +# Relearn artifact staging (host paths for harvested receipts). +for area in relearn relearn-t2i relearn-mm; do + ssh_h "install -d -m 0775 -o 65532 -g 65532 '$STATE_ROOT/\$area'" +done # Materialize missing env from examples (dev-safe placeholders) if absent @@ -262,21 +264,22 @@ case "$ENV" in prod) COMPOSE_FILES+=(-f deploy/compose/env-prod.yml) ;; esac -# Recipe 2.0 AutoModel pin: env-*/yml mounts /var/lib/prism/automodel-pin into -# prism-challenge. Fail loud on master when the staged tree is missing so a -# redeploy does not silently return code=pin to miners. Also pick up a -# host-local overlay outside the rsync tree (survives --delete) when present. +# Challenge pins are rsynced with the tree. Live Lium rent refuses until each +# eval_image_digest is a real sha256 pin. if [[ "$ROLE" == "master" ]]; then - if ssh_h "test -d /var/lib/prism/automodel-pin/.git"; then - echo "remote-deploy: AutoModel pin present at /var/lib/prism/automodel-pin" - else - echo "remote-deploy: WARNING: AutoModel pin missing at /var/lib/prism/automodel-pin" >&2 - echo "remote-deploy: stage with: ./deploy/scripts/stage-automodel-pin.sh --dir /var/lib/prism/automodel-pin" >&2 - echo "remote-deploy: (Prism AutoModel intake fails closed with code=pin until staged)" >&2 - fi - if ssh_h "test -f /var/lib/prism/docker-compose.automodel-pin.yml"; then - COMPOSE_FILES+=(-f /var/lib/prism/docker-compose.automodel-pin.yml) - echo "remote-deploy: including host AutoModel pin overlay" + for pin in relearn-pin.toml relearn-t2i-pin.toml relearn-mm-pin.toml; do + if ssh_h "test -f '$REMOTE_DIR/config/$pin'"; then + echo "remote-deploy: pin present at $REMOTE_DIR/config/$pin" + else + echo "remote-deploy: WARNING: pin missing at $REMOTE_DIR/config/$pin" >&2 + fi + done + # Relearn T2I refuses submissions without a holdout file matching the pin's + # commitment. Warn loudly rather than letting the operator find out via 503s. + if ! ssh_h "test -s '$REMOTE_DIR/deploy/secrets/relearn-t2i/holdout.json'"; then + echo "remote-deploy: WARNING: relearn-t2i holdout records missing at" \ + "$REMOTE_DIR/deploy/secrets/relearn-t2i/holdout.json;" \ + "generate with: cargo run -p xtask -- relearn-t2i-holdout" >&2 fi fi @@ -292,9 +295,10 @@ if [[ "$BUILD_FROM" == "prebuilt" ]]; then "$ROOT/target/release/validator" \ "$ROOT/target/release/gateway" \ "$ROOT/target/release/updater" \ - "$ROOT/target/release/prism-challenge" \ - "$ROOT/target/release/design-challenge" \ - "$ROOT/target/release/design-egress-proxy" \ + "$ROOT/target/release/relearn-challenge" \ + "$ROOT/target/release/relearn-t2i-challenge" \ + "$ROOT/target/release/relearn-mm-challenge" \ + "$ROOT/target/release/bounty-challenge" \ "$HOST:$REMOTE_DIR/target/release/" fi @@ -354,11 +358,10 @@ PY python3 - "\$DIGESTS_FILE" <<'PY' | while IFS=\$'\t' read -r service image digest tag; do import json, sys optional = { - "prism-challenge", - "design-challenge", - "design-egress-proxy", - "design-runtime", - "design-review", + "relearn-challenge", + "relearn-t2i-challenge", + "relearn-mm-challenge", + "bounty-challenge", "base-attest-helper", } data = json.load(open(sys.argv[1], encoding="utf-8")) @@ -382,40 +385,12 @@ PY pull_retag "\$ref" "\$tag" done else - echo "remote-deploy: no \$DIGESTS_FILE — skipping optional design/prism/attest-helper pulls" - echo "remote-deploy: (those images are only required for staging --build-from source)" + echo "remote-deploy: no \$DIGESTS_FILE — skipping optional attest-helper pull" + echo "remote-deploy: (that image is only required for staging --build-from source)" fi - # design-runtime is not a compose service; retag for sandbox pulls when present. else # Build service images from current tree (source) or prebuilt binaries. docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} build - # Sandbox runtime + anti-cheat review images are not compose services; build - # them explicitly. BUILD_FROM must match the compose build — without it these - # silently fell back to a source compile even on prebuilt deploys, so the - # review image could stay byte-identical (BuildKit cache) while the rest of - # the stack moved to new binaries. --iidfile + inspect prove the tag moved - # to the just-built image; a no-op build that leaves the tag on an old image - # must fail the deploy, not report success. - build_local_image() { - local target="\$1" tag="\$2" iid before after built - iid="\$(mktemp)" - before="\$(docker image inspect "\$tag" --format '{{.Id}}' 2>/dev/null || echo none)" - docker build -f deploy/Dockerfile --target "\$target" \ - --build-arg BUILD_FROM="\$BUILD_FROM" \ - --iidfile "\$iid" -t "\$tag" . - built="\$(cat "\$iid")" - rm -f "\$iid" - after="\$(docker image inspect "\$tag" --format '{{.Id}}')" - echo "remote-deploy: \$tag \$before -> \$after" - if [[ -z "\$built" || "\$after" != "\$built" ]]; then - echo "remote-deploy: ERROR: \$tag resolves to \$after but the build produced \$built — refusing to continue" >&2 - exit 1 - fi - } - # Sandbox runtime image (not a long-running compose service). - build_local_image design-runtime design-runtime:0.1.0 - # Anti-cheat review image (one-shot containers spawned by design-challenge). - build_local_image design-review design-review:0.1.0 fi # The updater can only pull from a registry. Enable it only when the desired @@ -454,7 +429,8 @@ docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} \$UP_PROFILE "\${UP_ARGS[@ # validator-host wallet for WeightsSetRateLimit / CRV4 commits. if [[ '$ROLE' == 'validator' ]]; then docker compose ${COMPOSE_FILES[*]} rm -sf \ - prism-challenge design-challenge design-egress-proxy socket-proxy \ + relearn-challenge relearn-t2i-challenge relearn-mm-challenge \ + bounty-challenge socket-proxy \ >/dev/null 2>&1 || true elif [[ '$ROLE' == 'master' ]]; then docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} rm -sf validator \ @@ -531,8 +507,10 @@ headers = { "Authorization": f"Bearer {token}", } backends = [ - ("prism", "http://prism-challenge:8092"), - ("design", "http://design-challenge:8093"), + ("relearn", "http://relearn-challenge:8095"), + ("relearn-t2i", "http://relearn-t2i-challenge:8097"), + ("relearn-mm", "http://relearn-mm-challenge:8098"), + ("bounty", "http://bounty-challenge:8096"), ] failed = False for cid, url in backends: diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index e818567bf..687e95fb4 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -3,8 +3,10 @@ Containers run as `base` (uid **65532**). Host secret files MUST be: ```bash -chown 65532:65532 deploy/secrets/gateway_sk deploy/secrets/prism_sk deploy/secrets/design_sk -chmod 0400 deploy/secrets/gateway_sk deploy/secrets/prism_sk deploy/secrets/design_sk +for f in gateway_sk relearn_sk relearn_t2i_sk relearn_mm_sk bounty_sk; do + chown 65532:65532 "deploy/secrets/$f" + chmod 0400 "deploy/secrets/$f" +done ``` Bind-mounts use the file inode; directory mode 0700 is OK. @@ -15,9 +17,11 @@ Bind-mounts use the file inode; directory mode 0700 is OK. |------|---------|-------| | `gateway_sk` | gateway | Bundle seal mini-secret (`BASE_GATEWAY_SK_FILE`) | | `gateway_admin_token` | gateway + seal scripts | Bearer for `/v1/admin/*` (`BASE_GATEWAY_ADMIN_TOKEN_FILE`). **Required** when `BASE_GATEWAY_REQUIRE_OWNER=1`. Mode **0400**, uid **65532** | -| `prism_sk` | prism-challenge | PRISM challenge mini-secret | -| `design_sk` | design-challenge **only** | Design challenge mini-secret; never mount on egress proxy | -| `challenge_sk` | legacy placeholder | Prefer `prism_sk` / `design_sk`; do not reuse across challenges | +| `relearn_sk` | relearn-challenge | Relearn LLM leaf mini-secret; pub must match `config/challenges.toml` | +| `relearn_t2i_sk` | relearn-t2i-challenge | Relearn T2I leaf mini-secret; pub must match `config/challenges.toml` | +| `relearn_mm_sk` | relearn-mm-challenge | Relearn Multimodal leaf mini-secret; pub must match `config/challenges.toml` | +| `bounty_sk` | bounty-challenge | Bounty leaf mini-secret; pub must match `config/challenges.toml` | +| `prism_sk` / `design_sk` | retired products | Do not mount on the live compose path | ```bash # Generate once per environment; never commit the bytes. @@ -29,6 +33,44 @@ chmod 0400 deploy/secrets/gateway_admin_token Local dummy for development: decrypt with age: `age -d -i ~/.base-secrets/age-identity.txt -o deploy/secrets/design_sk ~/.base-secrets/design-dummy.age` +## Relearn T2I / Relearn Multimodal + +| Path | Used by | Notes | +|------|---------|-------| +| `relearn-t2i/holdout.json` | relearn-t2i-challenge | Frozen holdout prompt records. **Never commit.** Verified at boot against `holdout_commitment` in `config/relearn-t2i-pin.toml`; a mismatch means submissions answer **503** rather than falling back to the public split. Mode **0400**, uid **65532** | +| `relearn-t2i/admin_tokens` | relearn-t2i-challenge | One operator bearer per line for `POST /v1/admin/promote` | +| `relearn-mm/admin_tokens` | relearn-mm-challenge | One operator bearer per line for `POST /v1/admin/promote` | + +Regenerate the T2I holdout with the private salt (keep the salt off git — it is +what makes the holdout unguessable): + +```bash +mkdir -p deploy/secrets/relearn-t2i deploy/secrets/relearn-mm +# Public split ids come from the pin; every one of them must be excluded. +mapfile -t EXCLUDE < <(python3 -c ' +import tomllib +from pathlib import Path +doc = tomllib.loads(Path("config/relearn-t2i-pin.toml").read_text()) +for i in doc["prompts"]["public_ids"]: + print(f"--exclude={i}") +') +cargo run -p xtask -- relearn-t2i-holdout \ + --bench ~/.base-secrets/qwen_image_bench_hf_v0518.jsonl \ + --salt "$RELEARN_T2I_HOLDOUT_SALT" \ + --size 40 "${EXCLUDE[@]}" \ + --out deploy/secrets/relearn-t2i/holdout.json +# Paste the printed holdout_commitment into config/relearn-t2i-pin.toml and +# re-sign the trust root (config/CEREMONY.md). +touch deploy/secrets/relearn-t2i/admin_tokens deploy/secrets/relearn-mm/admin_tokens +chown -R 65532:65532 deploy/secrets/relearn-t2i deploy/secrets/relearn-mm +chmod 0400 deploy/secrets/relearn-t2i/* deploy/secrets/relearn-mm/* +``` + +The Relearn Multimodal service also needs `RELEARN_MM_CHAMPION_LM_HASH` (the +SHA-256 of the champion Relearn LLM weights). It is not a secret — it is the +reference an encoder-only submission must hash-match — but without it those +submissions are rejected. + ## Design challenge | Path | Used by | Notes | diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index de648f302..5b0e3c353 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -12,22 +12,29 @@ services: - "8089:8080" environment: BASE_ROLE: validator - prism-challenge: + relearn-challenge: ports: - - "8092:8092" + - "8095:8095" environment: - BASE_CHALLENGE_BIND: 0.0.0.0:8092 - design-challenge: + BASE_CHALLENGE_BIND: 0.0.0.0:8095 + RELEARN_FORCE_SIM: "true" + relearn-t2i-challenge: ports: - - "8093:8093" + - "8097:8097" environment: - BASE_CHALLENGE_BIND: 0.0.0.0:8093 - # Test-only host Sim (never set on staging/prod overlays). - BASE_ALLOW_HOST_SIM: "1" - DESIGN_FORCE_SIM: "true" - design-egress-proxy: + BASE_CHALLENGE_BIND: 0.0.0.0:8097 + RELEARN_T2I_FORCE_SIM: "true" + relearn-mm-challenge: + ports: + - "8098:8098" + environment: + BASE_CHALLENGE_BIND: 0.0.0.0:8098 + RELEARN_MM_FORCE_SIM: "true" + bounty-challenge: + ports: + - "8096:8096" environment: - DESIGN_EGRESS_SIM: "true" + BASE_CHALLENGE_BIND: 0.0.0.0:8096 socket-proxy: ports: - "2375:2375" diff --git a/docker-compose.yml b/docker-compose.yml index 5188eb26d..909c8dea5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ # base control-plane stack # -# Default: postgres + validator + updater + socket-proxy + prism-challenge -# + design-challenge + design-egress-proxy. +# Default: postgres + validator + updater + socket-proxy + the four live +# challenges (relearn, relearn-t2i, relearn-mm, bounty). # Master/owner host only: # docker compose --profile master up -d # brings gateway as an additional service (D3). @@ -181,14 +181,15 @@ services: # --------------------------------------------------------------------------- - # prism-challenge — operator PRISM challenge health + miner submit (:8092). + # relearn-challenge — Relearn LLM post-training factory (:8095). + # Miner pays Lium (BYOK). Control plane rents a digest-pinned eval image. # --------------------------------------------------------------------------- - prism-challenge: - image: prism-challenge:0.1.0 + relearn-challenge: + image: relearn-challenge:0.1.0 build: context: . dockerfile: deploy/Dockerfile - target: prism-challenge + target: relearn-challenge args: BUILD_FROM: ${BASE_DOCKER_BUILD_FROM:-prebuilt} restart: unless-stopped @@ -196,62 +197,32 @@ services: postgres: condition: service_healthy environment: - BASE_CHALLENGE_BIND: 0.0.0.0:8092 + BASE_CHALLENGE_BIND: 0.0.0.0:8095 BASE_CHALLENGE_SK_FILE: /run/base/challenge_sk - # Real Lium is used whenever an API key is present. Set PRISM_FORCE_SIM=true - # to keep a deployment on the offline deterministic backend (no GPU spend). - PRISM_FORCE_SIM: "${PRISM_FORCE_SIM:-false}" - LIUM_API_KEY_FILE: /run/base/lium/api_key - LIUM_SSH_PRIVATE_KEY: /run/base/lium/ssh_ed25519 - LIUM_SSH_PUBLIC_KEY_FILE: /run/base/lium/ssh_ed25519.pub - OPENROUTER_API_KEY_FILE: /run/base/openrouter/api_key + # Sim eval unless a miner BYOK key is presented. Never log LIUM_API_KEY. + RELEARN_FORCE_SIM: "${RELEARN_FORCE_SIM:-false}" + RELEARN_PIN_FILE: /etc/base/config/relearn-pin.toml + RELEARN_TEACHER_BACKEND: "${RELEARN_TEACHER_BACKEND:-http_api}" + RELEARN_TEACHER_API_URL: "${RELEARN_TEACHER_API_URL:-}" + RELEARN_TEACHER_MODEL: "${RELEARN_TEACHER_MODEL:-}" + RELEARN_ADMIN_TOKENS_FILE: /run/base/relearn/admin_tokens BASE_CHALLENGE_GATEWAY_ENDPOINT: ${BASE_CHALLENGE_GATEWAY_ENDPOINT:-http://gateway:8080} - PRISM_MAX_CONCURRENT_EVALS: "${PRISM_MAX_CONCURRENT_EVALS:-8}" - # Pods need a while for sshd after RUNNING on the control plane. - PRISM_SSH_ATTEMPTS: "${PRISM_SSH_ATTEMPTS:-30}" - PRISM_SSH_RETRY_SECS: "${PRISM_SSH_RETRY_SECS:-10}" - PRISM_SSH_RUNNING_TIMEOUT_SECS: "${PRISM_SSH_RUNNING_TIMEOUT_SECS:-900}" - # Top-model GitHub publish (BaseIntelligence/prism top-model/): no-op - # when the token file is absent/empty. - PRISM_TOPMODEL_GITHUB_TOKEN_FILE: /run/base/github/token - # Top-model HuggingFace publish (BaseIntelligence/top-prism-architecture): - # no-op when the token file is absent/empty. - PRISM_TOPMODEL_HF_TOKEN_FILE: /run/base/huggingface/token - PRISM_TOPMODEL_HF_REPO: "${PRISM_TOPMODEL_HF_REPO:-BaseIntelligence/top-prism-architecture}" - # Require harvested checkpoint for top-model journal (set 0 for source-only). - PRISM_TOPMODEL_REQUIRE_WEIGHTS: "${PRISM_TOPMODEL_REQUIRE_WEIGHTS:-1}" - # Parked checkpoints harvested from Lium pods (master-local). - PRISM_ARTIFACT_DIR: /var/lib/prism/artifacts - # G1–G8 eval assets pack (optional; harness falls back to public_dev). - PRISM_EVAL_ASSETS_DIR: "${PRISM_EVAL_ASSETS_DIR:-}" - PRISM_FLOW: "${PRISM_FLOW:-v3}" - # Recipe 2.0 AutoModel pin checkout (deploy/scripts/stage-automodel-pin.sh). - # Required for live AutoModel intake; unset → pin unavailable (fail-closed). - PRISM_AUTOMODEL_PIN_DIR: "${PRISM_AUTOMODEL_PIN_DIR:-}" - # Operator bearer (retry + playground + gating + artifacts). Empty → 503. - PRISM_ADMIN_TOKENS_FILE: /run/base/prism/admin_tokens env_file: - # Required: BASE_DATABASE_URL (+ BASE_NETUID). Missing file → compose - # fails closed (binaries would otherwise fall back to in-memory store). - - path: ./deploy/env/prism-challenge.env + - path: ./deploy/env/relearn-challenge.env required: true volumes: - # prism signs with its OWN mini secret: - # the gateway verifies leaves against the trust root per-challenge key. - - ./deploy/secrets/prism_sk:/run/base/challenge_sk:ro + - ./config:/etc/base/config:ro + - ./deploy/secrets/relearn_sk:/run/base/challenge_sk:ro - ./deploy/secrets/lium:/run/base/lium:ro - - ./deploy/secrets/openrouter:/run/base/openrouter:ro - - ./deploy/secrets/github:/run/base/github:ro - - ./deploy/secrets/huggingface:/run/base/huggingface:ro - - ./deploy/secrets/prism:/run/base/prism:ro - - prism-artifacts:/var/lib/prism/artifacts + - ./deploy/secrets/relearn:/run/base/relearn:ro + - relearn-artifacts:/var/lib/relearn expose: - - "8092" + - "8095" healthcheck: test: [ "CMD-SHELL", - "curl -fsS -m 5 http://127.0.0.1:8092/health || exit 1", + "curl -fsS -m 5 http://127.0.0.1:8095/health || exit 1", ] interval: 10s timeout: 3s @@ -261,39 +232,49 @@ services: - base # --------------------------------------------------------------------------- - # design-egress-proxy — open Internet egress for sandboxes (install + run) - # with an internal-target blocklist (metadata / loopback / RFC1918 / CGNAT / - # control-plane names, enforced post-DNS-resolution) plus the budgeted - # OpenRouter chat path. Holds OPENROUTER key; never mount design_sk here. - # On base + internal design-sandbox-egress so sandboxes can reach it without - # direct internet. + # relearn-t2i-challenge — Relearn T2I (:8097). Miners fine-tune the pinned + # Cosmos3 generator; Q-Judger scores frozen prompt cells. The holdout prompt + # records are an operator secret, mounted read-only from deploy/secrets. # --------------------------------------------------------------------------- - design-egress-proxy: - image: design-egress-proxy:0.1.0 + relearn-t2i-challenge: + image: relearn-t2i-challenge:0.1.0 build: context: . dockerfile: deploy/Dockerfile - target: design-egress-proxy + target: relearn-t2i-challenge args: BUILD_FROM: ${BASE_DOCKER_BUILD_FROM:-prebuilt} restart: unless-stopped + depends_on: + postgres: + condition: service_healthy environment: - DESIGN_EGRESS_BIND: 0.0.0.0:8094 - OPENROUTER_API_KEY_FILE: /run/base/openrouter/api_key - DESIGN_TOKEN_BUDGET: "${DESIGN_TOKEN_BUDGET:-8000}" - DESIGN_EGRESS_SIM: "${DESIGN_EGRESS_SIM:-false}" + BASE_CHALLENGE_BIND: 0.0.0.0:8097 + BASE_CHALLENGE_SK_FILE: /run/base/challenge_sk + # Sim eval unless a judge endpoint is configured. Never log the key. + RELEARN_T2I_FORCE_SIM: "${RELEARN_T2I_FORCE_SIM:-false}" + RELEARN_T2I_PIN_FILE: /etc/base/config/relearn-t2i-pin.toml + RELEARN_T2I_JUDGE_BACKEND: "${RELEARN_T2I_JUDGE_BACKEND:-http_api}" + RELEARN_T2I_JUDGE_API_URL: "${RELEARN_T2I_JUDGE_API_URL:-}" + RELEARN_T2I_HOLDOUT_FILE: /run/base/relearn-t2i/holdout.json + RELEARN_T2I_ADMIN_TOKENS_FILE: /run/base/relearn-t2i/admin_tokens + BASE_CHALLENGE_GATEWAY_ENDPOINT: ${BASE_CHALLENGE_GATEWAY_ENDPOINT:-http://gateway:8080} env_file: - - path: ./deploy/env/design-egress-proxy.env - required: false + - path: ./deploy/env/relearn-t2i-challenge.env + required: true volumes: - - ./deploy/secrets/openrouter:/run/base/openrouter:ro + - ./config:/etc/base/config:ro + - ./deploy/secrets/relearn_t2i_sk:/run/base/challenge_sk:ro + - ./deploy/secrets/lium:/run/base/lium:ro + - ./deploy/secrets/relearn-t2i:/run/base/relearn-t2i:ro + - relearn-t2i-artifacts:/var/lib/relearn-t2i expose: - - "8094" + - "8097" healthcheck: test: [ "CMD-SHELL", - "curl -fsS -m 5 http://127.0.0.1:8094/health || exit 1", + "curl -fsS -m 5 http://127.0.0.1:8097/health || exit 1", ] interval: 10s timeout: 3s @@ -301,71 +282,97 @@ services: start_period: 10s networks: - base - - design-sandbox-egress # --------------------------------------------------------------------------- - # design-challenge — miner harness API + sandbox orchestrator (:8093). - # Docker ONLY via socket-proxy (DESIGN_DOCKER_BASE). No raw docker.sock. - # Sandbox LLM traffic goes through design-egress-proxy (no key in sandbox). - # Agentic anti-cheat on this service needs the OpenRouter key at the default - # DESIGN_AGENTIC_OPENROUTER_KEY_FILE path (never passed into miner sandboxes). + # relearn-mm-challenge — Relearn Multimodal (:8098). Miners attach a + # permissive vision encoder to the champion Relearn LLM. The champion LM + # weights hash is gate 1's reference and must be set for encoder-only submits. # --------------------------------------------------------------------------- - design-challenge: - image: design-challenge:0.1.0 + relearn-mm-challenge: + image: relearn-mm-challenge:0.1.0 build: context: . dockerfile: deploy/Dockerfile - target: design-challenge + target: relearn-mm-challenge args: BUILD_FROM: ${BASE_DOCKER_BUILD_FROM:-prebuilt} restart: unless-stopped depends_on: postgres: condition: service_healthy - socket-proxy: - condition: service_started - design-egress-proxy: + environment: + BASE_CHALLENGE_BIND: 0.0.0.0:8098 + BASE_CHALLENGE_SK_FILE: /run/base/challenge_sk + RELEARN_MM_FORCE_SIM: "${RELEARN_MM_FORCE_SIM:-false}" + RELEARN_MM_PIN_FILE: /etc/base/config/relearn-mm-pin.toml + RELEARN_MM_CHAMPION_LM_HASH: "${RELEARN_MM_CHAMPION_LM_HASH:-}" + RELEARN_MM_ADMIN_TOKENS_FILE: /run/base/relearn-mm/admin_tokens + BASE_CHALLENGE_GATEWAY_ENDPOINT: ${BASE_CHALLENGE_GATEWAY_ENDPOINT:-http://gateway:8080} + env_file: + - path: ./deploy/env/relearn-mm-challenge.env + required: true + volumes: + - ./config:/etc/base/config:ro + - ./deploy/secrets/relearn_mm_sk:/run/base/challenge_sk:ro + - ./deploy/secrets/lium:/run/base/lium:ro + - ./deploy/secrets/relearn-mm:/run/base/relearn-mm:ro + - relearn-mm-artifacts:/var/lib/relearn-mm + expose: + - "8098" + healthcheck: + test: + [ + "CMD-SHELL", + "curl -fsS -m 5 http://127.0.0.1:8098/health || exit 1", + ] + interval: 10s + timeout: 3s + retries: 6 + start_period: 10s + networks: + - base + + # bounty-challenge — bug-report challenge (:8096). Pair + reports; Chat inject env-only. + bounty-challenge: + image: bounty-challenge:0.1.0 + build: + context: . + dockerfile: deploy/Dockerfile + target: bounty-challenge + args: + BUILD_FROM: ${BASE_DOCKER_BUILD_FROM:-prebuilt} + restart: unless-stopped + depends_on: + postgres: condition: service_healthy environment: - BASE_CHALLENGE_BIND: 0.0.0.0:8093 + BASE_CHALLENGE_BIND: 0.0.0.0:8096 BASE_CHALLENGE_SK_FILE: /run/base/challenge_sk - DESIGN_FORCE_SIM: "${DESIGN_FORCE_SIM:-false}" - DESIGN_DOCKER_BASE: http://socket-proxy:2375 - # Host path must equal bind source below (daemon resolves binds on host). - DESIGN_STAGING_ROOT: ${BASE_STATE_DIR:-/var/lib/base}/design/staging - DESIGN_LLM_PROXY: http://design-egress-proxy:8094 - # Screenshot Chromium (--no-sandbox, file://) must not reach control-plane - # targets on the shared `base` network: force all http(s) through the - # egress proxy blocklist (incl. loopback/metadata via <-loopback>). - DESIGN_SCREENSHOT_PROXY: http://design-egress-proxy:8094 - DESIGN_ANNOTATOR_TOKENS_FILE: /run/base/design/annotator_tokens - DESIGN_AGENTIC_OPENROUTER_KEY_FILE: /run/base/openrouter/api_key - DESIGN_MAX_CONCURRENT: "${DESIGN_MAX_CONCURRENT:-2}" - DESIGN_INSTALL_TIMEOUT_SECS: "${DESIGN_INSTALL_TIMEOUT_SECS:-300}" + BOUNTY_CHAT_COMMAND: "${BOUNTY_CHAT_COMMAND:-}" + BOUNTY_BACKEND_PUBLIC_URL: "${BOUNTY_BACKEND_PUBLIC_URL:-}" + BOUNTY_ADMIN_TOKENS_FILE: /run/base/bounty/admin_tokens + BOUNTY_SESSION_SECRET_FILE: /run/base/bounty/session_secret BASE_CHALLENGE_GATEWAY_ENDPOINT: ${BASE_CHALLENGE_GATEWAY_ENDPOINT:-http://gateway:8080} env_file: - # Required: BASE_DATABASE_URL (+ BASE_NETUID). Missing file → compose - # fails closed (binaries would otherwise fall back to in-memory store). - - path: ./deploy/env/design-challenge.env + - path: ./deploy/env/bounty-challenge.env required: true volumes: - - ./deploy/secrets/design_sk:/run/base/challenge_sk:ro - - ./deploy/secrets/design:/run/base/design:ro - - ./deploy/secrets/openrouter:/run/base/openrouter:ro - - design-artifacts:/var/lib/design - - ${BASE_STATE_DIR:-/var/lib/base}/design/staging:${BASE_STATE_DIR:-/var/lib/base}/design/staging + - ./config:/etc/base/config:ro + - ./deploy/secrets/bounty_sk:/run/base/challenge_sk:ro + - ./deploy/secrets/lium:/run/base/lium:ro + - ./deploy/secrets/bounty:/run/base/bounty:ro expose: - - "8093" + - "8096" healthcheck: test: [ "CMD-SHELL", - "curl -fsS -m 5 http://127.0.0.1:8093/health || exit 1", + "curl -fsS -m 5 http://127.0.0.1:8096/health || exit 1", ] interval: 10s timeout: 3s retries: 6 - start_period: 15s + start_period: 10s networks: - base @@ -373,7 +380,7 @@ services: image: tecnativa/docker-socket-proxy@sha256:9e4b9e7517a6b660f2cc903a19b257b1852d5b3344794e3ea334ff00ae677ac2 restart: unless-stopped environment: - # Shared proxy: updater rolls + design-challenge sandbox. App-level + # Shared proxy: updater rolls. App-level # Allowlist::updater / Allowlist::verifier enforce method/path; tecnativa # CONTAINERS includes DELETE for sandbox cleanup. NETWORKS stays off — # design-sandbox-egress is pre-created by compose (NetworkMode by name). @@ -413,17 +420,10 @@ volumes: base-pgdata: base-updater-state: base-validator-lkg: - design-artifacts: - prism-artifacts: + relearn-artifacts: + relearn-t2i-artifacts: + relearn-mm-artifacts: networks: base: driver: bridge - # Sandbox containers attach here (NetworkMode); only egress member is - # design-egress-proxy. internal=true blocks direct internet from sandboxes. - # Pin the Docker name so NetworkMode "design-sandbox-egress" matches (no - # compose project prefix) — socket-proxy cannot create networks at runtime. - design-sandbox-egress: - name: design-sandbox-egress - driver: bridge - internal: true diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 0a0424a5e..9ff4e7533 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -33,18 +33,21 @@ Public miner docs live **outside** this monorepo (examples + human guides only | Challenge | Repo | |-----------|------| -| Design | [`BaseIntelligence/design-challenge`](https://github.com/BaseIntelligence/design-challenge) | -| Prism | [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) | +| Relearn LLM | [`CortexLM/relearn`](https://github.com/CortexLM/relearn) | +| Relearn T2I | [`CortexLM/relearn`](https://github.com/CortexLM/relearn) + this repo [`external-miner/relearn-t2i.md`](./external-miner/relearn-t2i.md) | +| Relearn Multimodal | [`CortexLM/relearn`](https://github.com/CortexLM/relearn) + this repo [`external-miner/relearn-mm.md`](./external-miner/relearn-mm.md) | +| Bounty | this repo [`external-miner/bounty.md`](./external-miner/bounty.md) (subnet **reads** CortexLM/backend public API; it does not serve one) | -`docs/external-miner/` remains the in-repo mirror for CI (`external-docs-check`) and operators. When challenge APIs or rules change, update **both** the public repo and `external-miner/` (see root [`../AGENTS.md`](../AGENTS.md) § Challenge public docs). +`docs/external-miner/relearn.md` is the short Cortex pointer. The long Relearn guide lives in [`CortexLM/relearn`](https://github.com/CortexLM/relearn). Bounty pairing lives in this repo. Validators: [`external-miner/validators.md`](./external-miner/validators.md). When challenge APIs change, update **both** the public repo (when one exists) and `external-miner/` (see root [`../AGENTS.md`](../AGENTS.md) § Challenge public docs). ## Challenge / local E2E verification When updating challenge or local-subnet docs/runbooks, keep these invariants: -- **Master-only eval** — design/prism challenge services run on master; validator has **no challenge exec** (fetch sealed weights only). -- **Simulate submissions** — submit baseline **and** a cheat fixture through the challenge service; poll `/events` + `/logs`; do not treat `/health` alone as proof. -- **Design admin winners** — after clean `awaiting_admin`, operator bearer awards 1|2 winners; then leaf → seal path. +- **Master-only eval** — every challenge service (`relearn-challenge`, `relearn-t2i-challenge`, `relearn-mm-challenge`, `bounty-challenge`) runs on master; validator has **no challenge exec** (fetch sealed weights only). +- **Simulate submissions** — Relearn: `POST /v1/submissions` then poll `GET /v1/submissions/{id}`; Bounty: pair + `POST /v1/reports`. Do not treat `/health` alone as proof. A regression must not become champion. +- **Relearn promote** — after clean `awaiting_admin`, operator bearer `POST /v1/admin/promote`; then leaf → seal path. +- **Bounty adjudicate** — operator bearer `POST /v1/admin/adjudicate` (`valid` / `already_fixed_not_prod` / `invalid_malicious` / `duplicate`). Public leaderboard/reports are **CortexLM/backend**; Cortex **reads** `BOUNTY_BACKEND_PUBLIC_URL` (unset → skip). Do not serve `/v1/public/*` from this repo. - **No host Sim in staging/prod** — Docker only; `SimSandbox` / `BASE_ALLOW_HOST_SIM` are CI/local opt-in. - **Seal path** — `POST /v1/weights/raw` → seal → `GET /v1/weights/latest` with `sealed: true` (unsealed burn fallback is always available). That path needs `challenge_sk` + `gateway_sk`, **not** a gateway owner wallet. Validator wallets are for on-chain submit only. - Normative local procedure: [`runbooks/local-testnet-e2e.md`](runbooks/local-testnet-e2e.md). Repo contract: [`../AGENTS.md`](../AGENTS.md) § Challenge verification. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7b5181ef3..9b1fe6b30 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -5,15 +5,23 @@ Operator-facing map of the control plane. Normative byte contracts live in the f | Spec | Status | Role | |------|--------|------| | [`BUNDLE_SPEC.md`](./BUNDLE_SPEC.md) | **FROZEN** | Epoch bundle SCALE layout, merkle, aggregation, on-chain payload bounds | -| [`DESIGN_CHALLENGE.md`](./DESIGN_CHALLENGE.md) | **FROZEN** | `design` challenge: harness sandbox, agentic review, admin winners, D24 leaves | -| [`PRISM.md`](./PRISM.md) | live | `prism` Lium GPU recipe challenge (HTTP submit) | +| [`DESIGN_CHALLENGE.md`](./DESIGN_CHALLENGE.md) | archived freeze | Retired `design` product (not live) | +| [`PRISM.md`](./PRISM.md) | archived | Retired `prism` product (Lium rails reused by Relearn) | +| [`RELEARN.md`](./RELEARN.md) | live | `relearn` post-training factory (HTTP submit; miners pay Lium) | +| [`RELEARN-T2I.md`](./RELEARN-T2I.md) | live | `relearn-t2i` — Cosmos3 fine-tunes judged by Q-Judger | +| [`RELEARN-MM.md`](./RELEARN-MM.md) | live | `relearn-mm` — permissive vision encoder on the champion LLM | +| [`BOUNTY.md`](./BOUNTY.md) | live | `bounty` — paired bug reports | Do not restate those contracts here. Link them. +Audiences (do not mix): + +- Miners: [`external-miner/README.md`](./external-miner/README.md) indexes all four challenges +- Validators: [`external-miner/validators.md`](./external-miner/validators.md) + Security claim and what it excludes: [`THREAT_MODEL.md`](./THREAT_MODEL.md). Operator checklist: [`OPERATOR_SECURITY.md`](./OPERATOR_SECURITY.md). -Runbooks: [`runbooks/`](./runbooks/). -Miner-facing docs (version-pinned): [`external-miner/`](./external-miner/). +Runbooks: [`runbooks/`](./runbooks/). --- @@ -23,7 +31,7 @@ Miner-facing docs (version-pinned): [`external-miner/`](./external-miner/). - Gateway runs **only** as subnet owner (master). Startup asserts hotkey == on-chain `SubnetOwnerHotkey` or exits `2` before bind. - Validators **recompute** the weight vector from a signed, merkle-rooted epoch bundle. Challenge keys and measurements come from **owner-signed local files**, never from gateway HTTP. - CRV4 timelock commit-reveal on Bittensor testnet/mainnet as configured. Reveal is automatic on-chain. -- Challenges accept miner work over **HTTP** (design Python harness sandbox; prism Lium GPU eval). No miner Phala/CVM path. +- Live challenges accept miner work over **HTTP** (Relearn → Lium/sim eval; Bounty → pair + reports). Miners pay Lium when a key is present. --- @@ -34,8 +42,10 @@ Miner-facing docs (version-pinned): [`external-miner/`](./external-miner/). │ Master host (compose profile master) │ │ postgres · gateway · validator · │ │ updater · socket-proxy · │ - │ prism-challenge · design-challenge · │ - │ design-egress-proxy │ + │ relearn-challenge · │ + │ relearn-t2i-challenge · │ + │ relearn-mm-challenge · │ + │ bounty-challenge │ └───────────────┬─────────────────────┘ │ TLS terminates in gateway (D20) │ /challenge/{id}/* /v1/bundle/* @@ -47,8 +57,10 @@ Miner-facing docs (version-pinned): [`external-miner/`](./external-miner/). └───────────────┬─────────────────────┘ │ HTTP submit ┌───────────────▼─────────────────────┐ - │ Miner clients (no TEE required) │ - │ design harness / prism scripts │ + │ Miner clients │ + │ relearn artifact digest + Lium BYOK │ + │ t2i / mm artifact + manifest │ + │ bounty pair + bug reports │ └─────────────────────────────────────┘ ``` @@ -56,9 +68,10 @@ Miner-facing docs (version-pinned): [`external-miner/`](./external-miner/). |----------------|------| | `gateway` | Master-only: registry, reverse proxy, bundle seal/serve, sole TLS owner; mounts marketing [`SITE_API.md`](./SITE_API.md) (`GET /v1/site/*`) | | `validator` | Fetch/mirror bundle, verify, recompute, peer cross-check, CRV4 submit, dissent | -| `design-challenge` | **Master-only:** sandbox harness runs, sanitize/viewer, scoring, sign leaves | -| `design-egress-proxy` | **Master-only:** open sandbox egress (internal-target blocklist) + budgeted LLM path | -| `prism-challenge` | **Master-only:** Lium (or sim) recipe eval, review gate, sign leaves | +| `relearn-challenge` | **Master-only:** digest freeze, holdout unseal, Lium/sim eval, operator promote, sign leaves | +| `relearn-t2i-challenge` | **Master-only:** frozen prompt cells at pinned seeds, Q-Judger scoring, pillar / replay / contamination gates, sign leaves | +| `relearn-mm-challenge` | **Master-only:** text-intact rerun (hard gate), vision + agentic holdout with a pixel-shuffle control, sign leaves | +| `bounty-challenge` | **Master-only:** internal pair/reports/adjudicate; **reads** CortexLM/backend public API for scoring; sign leaves | | `updater` | Digest-pinned rollouts via `docker-socket-proxy` (master) | | `trustroot` | Offline keygen / sign / verify for owner-signed TOML | | `bundle` | SCALE types, seal, verify (`PROTOCOL_VERSION`) | @@ -95,7 +108,9 @@ Miner-facing docs (version-pinned): [`external-miner/`](./external-miner/). | `config/measurements.toml` + `.sig` | yes | every validator from **disk** | | Challenge / owner mini-secrets | **never** | challenge service / offline ceremony only | -Current emission posture: `design = 0` bps, `prism = 10000` bps (100% prism; sum = 10000). +Current emission posture: `relearn = 4000`, `relearn-t2i = 1500`, +`relearn-mm = 1500`, `bounty = 3000` bps (sum 10000; operator can retune). +Each challenge signs leaves under its **own** key; no two rows share one. Gateway DB is **routing only**. It is never a source of challenge keys, emission shares, or measurements (D18, D23). diff --git a/docs/BOUNTY.md b/docs/BOUNTY.md new file mode 100644 index 000000000..8adf18c0b --- /dev/null +++ b/docs/BOUNTY.md @@ -0,0 +1,41 @@ +# Bounty Challenge (live challenge) + +Control-plane notes. Miners start at [`external-miner/bounty.md`](./external-miner/bounty.md). +Validators start at [`external-miner/validators.md`](./external-miner/validators.md). + +**Public transparency lives in CortexLM/backend.** This subnet **reads** +`GET {BOUNTY_BACKEND_PUBLIC_URL}/v1/bounty/public/leaderboard` and +`GET {BOUNTY_BACKEND_PUBLIC_URL}/v1/bounty/public/reports`. It does not +serve `/v1/public/*` (or any unauthenticated public leaderboard). Public +consumers hit the backend. Empty `BOUNTY_BACKEND_PUBLIC_URL` → skip / sim +(CI stays green). Never bake a host into git. + +Internal ingest (pair / reports / adjudicate) stays on this service so +Chat can bind a hotkey. Scoring maps hotkey → lattice from published +backend rows that include both `problem_found` and `justification`. + +| Field | Value | +|-------|--------| +| `challenge_id` | `bounty` | +| `challenge_scoring_version` | `1` | +| Port | `8096` (local host `28096`) | +| Emission | `3000` bps (default; Relearn keeps `7000`) | + +Miners pair a Bittensor hotkey to a dedicated Cortex Chat account, then file +bug reports. Every report is tagged with that hotkey. Operators adjudicate: + +| Verdict | Weight | +|---------|--------| +| `valid` | reward (precision credit) | +| `already_fixed_not_prod` | ack only — no reward, no penalty | +| `invalid_malicious` | penalty (burn toward uid 0) | +| `duplicate` | no extra reward, no penalty | + +Champion is displacement vs the previous bounty champion on a holdout of +adjudicated reports (precision, not spam volume). Validators do not evaluate +reports; they verify sealed bundles. Unmatched emission burns to uid 0. + +Chat inject is env-only (`BOUNTY_CHAT_COMMAND`). Docs and examples use the +placeholder `` only. Never commit the live token. +Optional `X-Lium-Api-Key` is accepted and never logged; live Lium is skipped +when no key is present. diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index e7aedddf8..b8d844a23 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -45,31 +45,59 @@ Honest per-component status as of `main` HEAD. Updated as phases land. | Bundle seal (`POST /v1/weights/raw` → `GET /v1/weights/latest`) | done | Unsealed: fail-closed burn (`sealed: false`, uid 0 = 100%) instead of 404. | | Chain backend | done | Live only. `fake_owner` was removed from `bins/gateway`. | -## agent-challenge / hypertraining-challenge +## agent-challenge / hypertraining-challenge / design / prism (products) -Removed (replaced by design + prism HTTP paths; no Phala/CVM miner). +Removed as **live products**. Shared rails (`prism-lium*`, `prism-competition` paired tests, receipts, emit/carry) stay as libraries. Frozen specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. -## design-challenge +## relearn-challenge | Component | Status | Notes | |-----------|--------|-------| -| Crates (`crates/design-*`) | done | task, harness, prompts, sandbox, sanitize, store, egress-proxy, challenge. Elo lives in `design_rating` Postgres via `design-db` / `design-store-pg` — not a standalone crate. | -| Binary (`bins/design-challenge`) | done | HTTP API on `:8093`. | -| Binary (`bins/design-egress-proxy`) | done | Open egress proxy (internal blocklist) + budgeted LLM path. | -| Spec + checklist | done | [`DESIGN_CHALLENGE.md`](DESIGN_CHALLENGE.md) + checklist; `xtask design-check`. | -| Compose / images | in progress | deploy-wiring todo (port `28093` local). | -| Emission | **0 bps** | Prism 100% (10000 bps; sum `10000`). | +| Crates (`crates/relearn-*`) | **done** | task, score, store, eval, http, challenge. | +| Binary (`bins/relearn-challenge`) | **done** | HTTP API on `:8095`. | +| Compose / images | **done** | Default compose + `images.yml` target `relearn-challenge`. | +| Eval pin | **v0** | `config/relearn-pin.toml` — digest + `CortexLM/relearn` SHA empty until first green challenge CI. | +| Teacher | **v0** | HTTP API from operator env (`RELEARN_TEACHER_API_URL`, `RELEARN_TEACHER_MODEL`, `RELEARN_TEACHER_API_KEY`). Missing URL/key → sim. Judge-only; miner weights never served via that API. | +| Emission | **4000 bps** | Default share (sum across all four challenges is `10000`). | +| Spec | live | [`RELEARN.md`](RELEARN.md). | -## prism-challenge +## relearn-t2i-challenge | Component | Status | Notes | |-----------|--------|-------| -| Crate (`crates/prism-challenge`) | done | Lium client + sim backend + pipeline. | -| Binary (`bins/prism-challenge`) | done | Health + submit on `:8092`. | -| Compose service | done | Added to `docker-compose.yml` on `:8092`. | -| Dockerfile target | done | `deploy/Dockerfile` target `prism-challenge`. | -| GHCR image | done | Added to `images.yml` matrix and `ghcr-public.yml`. | -| Emission | **10000 bps** | Prism 100% (sum `10000`). | +| Crates (`crates/relearn-t2i-*`) | **done** | task (Cosmos3 pin, frozen prompts, seed lattice), judge (Q-Judger wire format), score (pillar / replay / faithfulness / contamination gates), store, eval, http, challenge. | +| Binary (`bins/relearn-t2i-challenge`) | **done** | HTTP API on `:8097`. | +| Compose / images | **done** | Default compose + `images.yml` target `relearn-t2i-challenge`. | +| Generator pin | **done** | `nvidia/Cosmos3-Super-Text2Image` (OpenMDW 1.1, card verified). Flux-family bases refused at pin load, submit, and eval. | +| Judge pin | **done** | Q-Judger (`Qwen/Qwen-Image-Bench`), card-fixed inference. No alternate judge is accepted. | +| Eval pin | **v0** | `config/relearn-t2i-pin.toml` — `eval_image_digest` empty until first green challenge CI; live rent refused until then. | +| Holdout | **done** | Commitment in git, records operator-side (`RELEARN_T2I_HOLDOUT_FILE`) and verified at boot. Mismatch → submissions 503. | +| Emission | **1500 bps** | Default share. | +| Spec | live | [`RELEARN-T2I.md`](RELEARN-T2I.md). | + +## relearn-mm-challenge + +| Component | Status | Notes | +|-----------|--------|-------| +| Crates (`crates/relearn-mm-*`) | **done** | task (permissive encoder policy), score (LM-intact hard gate, vision + agentic paired tests, pixel-shuffle control), store, eval, http, challenge. | +| Binary (`bins/relearn-mm-challenge`) | **done** | HTTP API on `:8098`. | +| Compose / images | **done** | Default compose + `images.yml` target `relearn-mm-challenge`. | +| Encoder pin | **done** | `google/siglip2-so400m-patch14-384` (Apache-2.0, card verified). Miner encoders must be Apache-2.0 / MIT / BSD / ISC. | +| Eval pin | **v0** | `config/relearn-mm-pin.toml` — `eval_image_digest` empty until first green challenge CI. | +| LM gate | **done** | Text holdout rerun vs champion − ε; encoder-only submissions must hash-match `RELEARN_MM_CHAMPION_LM_HASH`. | +| Emission | **1500 bps** | Default share. | +| Spec | live | [`RELEARN-MM.md`](RELEARN-MM.md). | + +## bounty-challenge + +| Component | Status | Notes | +|-----------|--------|-------| +| Crates (`crates/bounty-*`) | **done** | task (pairing), score, store, http (internal ingest), challenge (backend public **consumer**). | +| Binary (`bins/bounty-challenge`) | **done** | Internal HTTP on `:8096`. Does **not** serve `/v1/public/*`. | +| Miner CLI (`bins/cortex-bounty`) | **done** | `pair --hotkey`; Chat inject from `BOUNTY_CHAT_COMMAND`. | +| Compose / images | **done** | Default compose + `images.yml` target `bounty-challenge`. | +| Emission | **3000 bps** | Default share; operator can retune (sum `10000`). | +| Spec | live | [`BOUNTY.md`](BOUNTY.md). | ## Infrastructure @@ -100,10 +128,10 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`] | Component | Status | Notes | |-----------|--------|-------| -| design harness / sandbox | done | Two-phase Docker + `SimSandbox`; `base_design` SDK injected; sanitize + CSP viewer. | -| design rating / elimination | done | Integer Elo (K=32), bottom 20% / 4-round cooldown, exact-E leaves. | -| design API | done | Harness/quota/runs/viewer/annotate/ops on `:8093`. | -| prism Lium backend | done | `PRISM_FORCE_SIM=false` in staging; the binary logs `eval_backend=lium`. API key is mounted from a file so it never appears in `docker inspect`. | +| relearn HTTP / promote | **done** | `POST /v1/submissions` freeze → unseal → paired judge; `POST /v1/admin/promote` bearer; never crowns a regression. | +| bounty HTTP / adjudicate | **done** | Internal ingest: `POST /v1/pair` (sr25519) + `POST /v1/reports`; `POST /v1/admin/adjudicate`. Scoring **fetches** CortexLM/backend `GET /v1/bounty/public/leaderboard` + `/reports` (`BOUNTY_BACKEND_PUBLIC_URL`; unset → skip). | +| relearn Lium rails | **done** (sim default) | Reuses `prism-lium` client + `SimLiumBackend`. Live rent refuses without `sha256:` eval digest; miner BYOK never logged. | +| design / prism product APIs | retired | Crates remain as unused libraries. | | prism orchestration | done | DB-backed claim/execute/review/similarity/score state machine (`prism_submission` + append-only `prism_stage_event`), pre-pod screens (copy gate + static cheat + AST similarity) before Lium rent, sweeper (10h grace + pre-reclaim log harvest; skips live workers), **detached harness + resume-first boot/periodic reconcile** (reattach live pods via sealed BYOK; fail-closed only when unreattachable — `control_plane_restart` / `harness_detached`; `GET /v1/submissions/{id}/logs`), epoch-close batched D24 leaf emission with **WTA** (`prism-emit` outbox: `emitted_epoch` watermark + `prism_emit_cursor` + positive-score carry + `apply_wta`, migration 0012). `PRISM_MAX_CONCURRENT_EVALS` default/prod = 8. | | prism recipe v1 | done | `prism-recipe` contract, fineweb-edu pinned shard (URL + SHA-256, harness re-verifies), 6h train / 7h pod caps, baseline sources, recipe pin hex on the API. | | prism v3 harness | done (branch `prism-better`) | Multi-file harness package (`main.py` + `prismlib/`, miner code in `unshare --net` subprocess), seeded train stream with authoritative token counter, G6 probes, `prismlib/cheatguard.py` AST audit, METRICS_JSON v2, miner-chosen tokenizer, G5 RULER/BABILong/natural (pretrain-only), `RECIPE_VERSION 1.4.0`. | @@ -127,8 +155,10 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`] |-----|--------| | DCAP verify holds the attest mutex | A cold Intel PCS fetch (up to 20 s) serialises attestation submissions. | | DCAP error classification | Matches on `anyhow` message text; re-run `cargo test -p attest-policy --features dcap` after any `dcap-qvl` bump. | -| Design compose/images | deploy-wiring in progress; local port `28093` documented. | -| Design emission ceremony | Emission disabled (0 bps); prism at 100% (10000 bps). Optional prod `design_sk` / owner key rotation still pending. | +| Relearn eval image digests | Empty until `CortexLM/relearn` CI publishes digest-pinned `relearn-eval`, `relearn-t2i-eval`, and `relearn-mm-eval` images. Live Lium rent is refused until then. | +| Relearn T2I holdout salt | The committed `holdout_commitment` uses the documented **dev** salt so local and staging work out of the box. Production must rotate to a private salt, replace the commitment, and re-sign. | +| Relearn Multimodal champion LM hash | `RELEARN_MM_CHAMPION_LM_HASH` is operator-supplied. Unset means encoder-only submissions are rejected (they cannot prove the LM is unchanged). | +| Relearn public repo | [`CortexLM/relearn`](https://github.com/CortexLM/relearn) exists; this repo pins `relearn_git_sha`. Seed mirror: `docs/external-miner/relearn-seed/`. | | Mainnet (netuid 100) | Owner wallet not yet on this machine, so prod runs with `BASE_GATEWAY_REQUIRE_OWNER=0`. | | Prod pin placeholders | `deploy/pins/prod.json` still ships zero-digests until the first successful promote; registry mode rejects placeholders. | | Spaces backup secrets | First prod promote is fail-closed without `BASE_BACKUP_ENDPOINT` + `SPACES_ACCESS_KEY_ID` / `SPACES_SECRET_ACCESS_KEY` (or AWS_* fallbacks) in GitHub. | diff --git a/docs/NAMING.md b/docs/NAMING.md index a7c680eb0..cf1ca09ac 100644 --- a/docs/NAMING.md +++ b/docs/NAMING.md @@ -82,19 +82,9 @@ not the product name. ## Public miner repos (other GitHub repositories) -Miner-facing documentation still lives in the historical public repos until -those orgs publish a CortexLM mirror: - -- [`BaseIntelligence/design-challenge`](https://github.com/BaseIntelligence/design-challenge) -- [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) - -This monorepo’s operator mirror is [`docs/external-miner/`](external-miner/). -Do not rewrite those URLs here unless the public repos actually move. - -HuggingFace / GitHub top-model defaults (`BaseIntelligence/top-prism-architecture`, -`BaseIntelligence/prism` `top-model/`) are publish targets configured in -compose. Changing the default without a matching Hub/GitHub move breaks -top-model publish. +Live miner docs: [`CortexLM/relearn`](https://github.com/CortexLM/relearn). +Short pointer in this repo: [`docs/external-miner/relearn.md`](external-miner/relearn.md). +Design and Prism public repos are historical, not live miner paths. ## Postgres diff --git a/docs/OPERATOR_SECURITY.md b/docs/OPERATOR_SECURITY.md index eec7db5a4..cb1d6ce4c 100644 --- a/docs/OPERATOR_SECURITY.md +++ b/docs/OPERATOR_SECURITY.md @@ -13,7 +13,8 @@ Use this before every promote and after every incident. Architecture: [`ARCHITEC - [ ] Challenge signing secrets are **files** mounted into the challenge service, not env values (D11). - [ ] Owner and challenge mini-secrets never committed; only `*.pubkey` / TOML bodies + detached `.sig` in git. - [ ] Cloudflare / DO / Phala tokens live only in operator secret stores, not in docs or CI logs. -- [ ] Design agentic review: OpenRouter key is mounted on `design-challenge` / `design-egress-proxy` as a **file**, never into miner sandboxes. The ephemeral `design-review` container must receive the key via a **file mount** (`OPENROUTER_API_KEY_FILE`), never as `OPENROUTER_API_KEY` in container env (`/proc//environ` is boot-fixed). `run_command` must keep procfs and `/run/review-secrets` denied. Do not turn off `AGENTIC_ENABLE_RUN_COMMAND` in prod without a replacement inspection path. +- [ ] Relearn miner BYOK (`LIUM_API_KEY` / `X-Lium-Api-Key`) is never written to git, compose env committed files, or logs. Control-plane Lium mounts under `deploy/secrets/lium` are files, mode **0400**, uid **65532**. +- [ ] Teacher HTTP API (`RELEARN_TEACHER_API_URL` + `RELEARN_TEACHER_API_KEY`) is **judge-only**. Never commit the key. Never point the API at miner weights as the served / scored artifact. --- @@ -22,8 +23,8 @@ Use this before every promote and after every incident. Architecture: [`ARCHITEC - [ ] Every image reference is digest-pinned (`repo@sha256:<64 hex>`). No `:latest`. - [ ] Exactly one mount of `/var/run/docker.sock`: on `socket-proxy` (read-only). - [ ] socket-proxy allowlist matches updater needs (`CONTAINERS`, `IMAGES`, `POST` as configured). -- [ ] `design-challenge` sets `DESIGN_SCREENSHOT_PROXY=http://design-egress-proxy:8094` (screenshot Chromium must not talk direct to the `base` network). -- [ ] Staging/prod never set `BASE_ALLOW_HOST_SIM` / `DESIGN_FORCE_SIM=true` (asserted by `assert-compose-matrix.sh`). +- [ ] Staging/prod never set `BASE_ALLOW_HOST_SIM` / `DESIGN_FORCE_SIM` / `RELEARN_FORCE_SIM=true` as a live scoring path (asserted by `assert-compose-matrix.sh` for host Sim). +- [ ] Relearn live rent requires `config/relearn-pin.toml` `eval_image_digest` starting with `sha256:`. No floating eval tags. - [ ] Gateway service uses compose profile **`master`** only on the owner host. - [ ] Profile `evil-gateway` is **absent** from prod hosts. Spot-check: diff --git a/docs/RELEARN-MM.md b/docs/RELEARN-MM.md new file mode 100644 index 000000000..50bc74a19 --- /dev/null +++ b/docs/RELEARN-MM.md @@ -0,0 +1,69 @@ +# Relearn Multimodal (live challenge) + +Control-plane notes. Miners start at [`external-miner/relearn-mm.md`](./external-miner/relearn-mm.md). +Validators start at [`external-miner/validators.md`](./external-miner/validators.md). + +Eval image and harness live in [`CortexLM/relearn`](https://github.com/CortexLM/relearn). +This repo pins them in `config/relearn-mm-pin.toml`. + +| Field | Value | +|-------|--------| +| `challenge_id` | `relearn-mm` | +| `challenge_scoring_version` | `1` | +| Language side | `Qwen/Qwen3.8-Flash-Next` (the Relearn champion's base) | +| Vision encoder pin | `google/siglip2-so400m-patch14-384` (Apache-2.0) | +| Accepted encoder licenses | Apache-2.0, MIT, BSD-2/3-Clause, ISC | +| Port | `8098` (local host `28098`) | +| Emission | `1500` bps | + +Miner pays Lium (`LIUM_API_KEY` / `X-Lium-Api-Key`). Operator promote is +`POST /v1/admin/promote`. Epoch emit is champion lattice; others `NoScore` (D24). + +## Two gates, both mandatory + +**Gate 1 — the LLM is intact.** The submitted language model is rerun on the +existing Relearn text holdout with the vision modules ignored. A drop past +`LM_EPSILON` (0.01 absolute) makes the submission worth **zero on this +challenge**, not a reduced score, no matter how good the vision numbers are. +This is the whole design: attaching an encoder must never become a way to get +paid for damaging the champion. + +An **encoder-only** submission must additionally hash-match the champion LM +weights (`RELEARN_MM_CHAMPION_LM_HASH`, published on `/v1/status`). That hash is +the proof nothing on the text side moved at all. An **encoder-plus-LM** +submission may differ and is judged on the text score alone. Promoting an +encoder-plus-LM champion moves the reference hash, so the next encoder-only +submission is measured against the new language model. + +**Gate 2 — the vision side improved.** A frozen image holdout across four task +families — captioning, VQA, OCR / text-in-image, spatial relations — plus +agentic traces where the model must look at a screenshot, diagram, or UI before +calling a tool. The families are deliberately **not** ImageNet or COCO test: +both sit in every candidate encoder's pretraining mix, so a score there would +measure memorization rather than sight. + +Every agentic trace is replayed with the image pixels shuffled. A model that +really reads the image loses at least `MIN_SHUFFLE_DROP` (0.10); one that +pattern-matches the text prompt does not move, and a flat shuffle delta fails +the run. Zero traces cannot satisfy the control. + +Both comparisons use the same bootstrap paired test as the text challenge, which +refuses a verdict below 100 decided examples — which is why the pin's +`agentic_traces` floor is 100 rather than a smaller, cheaper number. + +## Gate summary + +| Gate | Rule | +|------|------| +| LM intact | Text holdout ≥ champion − `LM_EPSILON`; hard zero on failure | +| LM weights | Encoder-only submissions must hash-match the champion LM | +| Vision displacement | Bootstrap paired test on the pooled image holdout | +| Task coverage | Every one of the four families must have items | +| Agentic displacement | Bootstrap paired test on the tool-using traces | +| Pixel shuffle | Shuffled-image score must fall by ≥ `MIN_SHUFFLE_DROP` | +| Encoder license | OSI-permissive only; OpenRAIL and non-commercial are refused | + +`RELEARN_MM_FORCE_SIM=1` selects a deterministic offline eval for CI and local +development, reported on `/v1/status` as `eval_backend: sim`. +`deploy/scripts/assert-compose-matrix.sh` fails if a staging or prod overlay +enables it. diff --git a/docs/RELEARN-T2I.md b/docs/RELEARN-T2I.md new file mode 100644 index 000000000..6806b853b --- /dev/null +++ b/docs/RELEARN-T2I.md @@ -0,0 +1,84 @@ +# Relearn T2I (live challenge) + +Control-plane notes. Miners start at [`external-miner/relearn-t2i.md`](./external-miner/relearn-t2i.md). +Validators start at [`external-miner/validators.md`](./external-miner/validators.md). + +Eval image and harness live in [`CortexLM/relearn`](https://github.com/CortexLM/relearn). +This repo pins them in `config/relearn-t2i-pin.toml`. + +| Field | Value | +|-------|--------| +| `challenge_id` | `relearn-t2i` | +| `challenge_scoring_version` | `1` | +| Generator seed | `nvidia/Cosmos3-Super-Text2Image` (OpenMDW 1.1) | +| Judge | Q-Judger — `Qwen/Qwen-Image-Bench` (Apache-2.0, from Qwen3.6-27B) | +| Prompt set | `Qwen/Qwen-Image-Bench` dataset, ids 1..=1000 | +| Port | `8097` (local host `28097`) | +| Emission | `1500` bps | + +Miner pays Lium (`LIUM_API_KEY` / `X-Lium-Api-Key`). Operator promote is +`POST /v1/admin/promote`. Epoch emit is champion lattice; others `NoScore` (D24). + +## Non-negotiables + +- **Flux is rejected.** `relearn_t2i_task::base_is_rejected` refuses any + Flux-family base at pin load, at submit, and before scoring. Its weights are + non-commercial, which is incoherent for a subnet that pays for + redistributable artifacts. +- **Q-Judger is the only judge.** `relearn_t2i_judge::assert_judge_model` + refuses any other model, and the pin refuses to load with a different + `judge_model`. Judge inference is fixed by the model card: seed 42, + temperature 0, `top_k` 1, `top_p` 1.0, repetition penalty 1.05, thinking on, + 4096 max new tokens. +- **Eval prompts are frozen.** NVIDIA recommends upsampling a prompt into a + JSON document before generation. That is fine for a miner's own training and + fatal for a benchmark, so the scored strings live in the pin and are replayed + verbatim. Miners do not bring an upsampler to the scored split. +- **Same prompt ids, same seeds, every miner.** Seeds are + `sha256(domain ‖ pin_salt ‖ prompt_id ‖ variation_index)`, published on + `GET /v1/prompts` for the public split so any miner can reproduce a cell. +- **The holdout is not in git.** `config/relearn-t2i-pin.toml` carries only + `holdout_commitment` and `holdout_size`. Records come from + `RELEARN_T2I_HOLDOUT_FILE` and are verified against the commitment at boot; a + wrong file means submissions answer 503 rather than scoring the public split. + +## Scoring + +Q-Judger returns a thinking trace followed by a JSON score tree over five L1 +pillars. Raw `0|1|2` map to `0|60|100`; `N/A` is **excluded, not zeroed** (a +prompt where a criterion does not apply must not be punished). Level 3 averages +into level 2, level 2 into level 1, and the five pillars into the total, exactly +as the paper specifies. Series are normalized to `0..=1` so one +`prism_competition` dead-zone unit equals one paper point. + +Promotion requires every gate: + +| Gate | Rule | +|------|------| +| Holdout displacement | Bootstrap paired test on the private prompt split (the published split is informational) | +| Paired A/B | Same `(prompt_id, seed)` cells; win rate must be at least 5000 bps | +| Pillar regression | No L1 pillar may drop more than `PILLAR_EPSILON` (2 paper points). A large Alignment gain cannot hide a Quality collapse | +| Seed replay | Three pinned cells regenerated; exact image hash, or embedding drift ≤ `MAX_REPLAY_DRIFT` | +| Prompt faithfulness | ≥ 8 agentic spot checks (counts, rendered text, spatial relations) agreeing with Q-Judger Alignment ≥ 75 % | +| Contamination | Any eval prompt id in the submitted training metadata rejects the submission | +| Judge N/A rate | Above 25 % the run is void, never a score of zero | +| Public–holdout gap | Public far above holdout signals memorization | + +## Operator + +```bash +# Rotate the holdout (records never enter git). +cargo run -p xtask -- relearn-t2i-holdout \ + --bench ~/.base-secrets/qwen_image_bench_hf_v0518.jsonl \ + --salt "$RELEARN_T2I_HOLDOUT_SALT" --size 40 \ + --exclude 1 --exclude 26 … \ + --out deploy/secrets/relearn-t2i/holdout.json +``` + +Paste the printed `holdout_commitment` into `config/relearn-t2i-pin.toml`, then +re-sign the trust root ([`../config/CEREMONY.md`](../config/CEREMONY.md)). + +`RELEARN_T2I_FORCE_SIM=1` selects a deterministic offline judge for CI and local +development. It is reported on `/v1/status` as `judge_backend: sim` so it cannot +be mistaken for a real run, and `deploy/scripts/assert-compose-matrix.sh` fails +if a staging or prod overlay enables it. diff --git a/docs/RELEARN.md b/docs/RELEARN.md new file mode 100644 index 000000000..87da03da1 --- /dev/null +++ b/docs/RELEARN.md @@ -0,0 +1,24 @@ +# Relearn LLM (live challenge) + +Sibling challenges: [`RELEARN-T2I.md`](./RELEARN-T2I.md) (image generation, +judged by Q-Judger) and [`RELEARN-MM.md`](./RELEARN-MM.md) (vision encoder on +this challenge's champion). They share the champion-versus-challenger holdout +shape and the Lium payment model, and each signs leaves under its own key. + +Control-plane notes. Miners start at [`external-miner/relearn.md`](./external-miner/relearn.md). +Validators start at [`external-miner/validators.md`](./external-miner/validators.md). + +Eval image and harness live in [`CortexLM/relearn`](https://github.com/CortexLM/relearn). +This repo pins them in `config/relearn-pin.toml`. + +| Field | Value | +|-------|--------| +| `challenge_id` | `relearn` | +| `challenge_scoring_version` | `1` | +| Base model | `Qwen/Qwen3.8-Flash-Next` | +| Teacher / judge | HTTP API (operator sets `RELEARN_TEACHER_*`) | +| Port | `8095` (local host `28095`) | +| Emission | `4000` bps (default) | + +Miner pays Lium (`LIUM_API_KEY` / `X-Lium-Api-Key`). Operator promote is +`POST /v1/admin/promote`. Epoch emit is champion lattice; others `NoScore` (D24). diff --git a/docs/external-miner/README.md b/docs/external-miner/README.md index d03a7b59d..ad4b9c40e 100644 --- a/docs/external-miner/README.md +++ b/docs/external-miner/README.md @@ -1,59 +1,36 @@ -# External miner docs +# How to mine **Bundle `protocol_version`:** `1` -**Miner path:** HTTP submit only — **no Phala/CVM** +**Miner pays Lium** (`LIUM_API_KEY` / `X-Lium-Api-Key`). -This badge must match `bundle::PROTOCOL_VERSION` in crate `bundle`. +This badge must match `bundle::PROTOCOL_VERSION` in crate `bundle`. CI gate: `cargo run -p xtask -- external-docs-check`. -Agent-v1 / Phala CVM / hypertraining miner paths are **removed**. Miners submit -over HTTP to the live challenges: +Live challenges: **Relearn LLM** (`relearn`), **Relearn T2I** (`relearn-t2i`), +**Relearn Multimodal** (`relearn-mm`), and **Bounty** (`bounty`). HTTP submit. -| Challenge | `challenge_id` | Scoring | Guide | Public miner repo | -|-----------|----------------|---------|-------|-------------------| -| Design | `design` | `challenge_scoring_version` **2** (daily share ≥2 wins + agentic) | [design.md](./design.md) | [BaseIntelligence/design-challenge](https://github.com/BaseIntelligence/design-challenge) | -| Prism | `prism` | `challenge_scoring_version` **4** (G2 public-suite benchmarks) | [prism.md](./prism.md) | [BaseIntelligence/prism](https://github.com/BaseIntelligence/prism) | +| Challenge | Guide (this repo) | Notes | +|-----------|-------------------|-------| +| Relearn LLM | [relearn.md](./relearn.md) | Long guide + eval image: [CortexLM/relearn](https://github.com/CortexLM/relearn) | +| Relearn T2I | [relearn-t2i.md](./relearn-t2i.md) | Fine-tune `nvidia/Cosmos3-Super-Text2Image` (OpenMDW 1.1). Judge is **Q-Judger** (`Qwen/Qwen-Image-Bench`) on Qwen-Image-Bench prompts. **Flux is rejected** | +| Relearn Multimodal | [relearn-mm.md](./relearn-mm.md) | Attach an **Apache-2.0 / MIT / BSD / ISC** vision encoder to the champion LLM. Regressing the LLM scores zero | +| Bounty | [bounty.md](./bounty.md) | Pair via `cortex-bounty`; Chat inject is `BOUNTY_CHAT_COMMAND` (env-only). Public leaderboard/reports live in CortexLM/backend; Cortex reads `BOUNTY_BACKEND_PUBLIC_URL` | -Do **not** conflate version axes: +Pinned models: `Qwen/Qwen3.8-Flash-Next`, teacher `kimi-k3`, +`nvidia/Cosmos3-Super-Text2Image`, `google/siglip2-so400m-patch14-384`. +Bundle bytes: [`BUNDLE_SPEC.md`](../BUNDLE_SPEC.md). -| Axis | Value | Meaning | -|------|-------|---------| -| Bundle `protocol_version` | **1** | Leaf / merkle / weight bytes ([`BUNDLE_SPEC.md`](../BUNDLE_SPEC.md)) | -| Design scoring | **1** | Agentic anti-cheat + admin winners 1\|2 ([`DESIGN_CHALLENGE.md`](../DESIGN_CHALLENGE.md)) | -| Prism scoring | **2** | Pure bpb + agentic/AST/metrics anti-cheat ([`PRISM.md`](../PRISM.md)) | - -| Page | Topic | -|------|-------| -| [design.md](./design.md) | Design harness (`agent.py` + `pyproject.toml`) HTTP submit | -| [examples/design-baseline/](./examples/design-baseline/) | Reference design miner (`llm.chat` → required HTML pages) | -| [prism.md](./prism.md) | Prism AutoModel patch (`automodel.base` + `automodel.patch`) HTTP submit | -| [examples/dense-1b/](./examples/dense-1b/) | Reference Prism miner (dense ~975M, ZeRO-1) | -| [troubleshoot.md](./troubleshoot.md) | Common HTTP / quota / scoring failures | - -Normative contracts: - -- Design freeze: [`../DESIGN_CHALLENGE.md`](../DESIGN_CHALLENGE.md) -- Prism: [`../PRISM.md`](../PRISM.md) + [`../PRISM_RECIPE.md`](../PRISM_RECIPE.md) -- Bundle bytes: [`../BUNDLE_SPEC.md`](../BUNDLE_SPEC.md) -- Threat claim (D19): [`../THREAT_MODEL.md`](../THREAT_MODEL.md) §1 - -## Gateway base URL - -Production/staging miners call the **gateway** reverse proxy: +Every Relearn challenge promotes champion-versus-challenger on a **private +holdout**. Winning the published split is informational; it is not a promotion. ```text -https:///challenge/design/... -https:///challenge/prism/... -``` - -Local smoke (host ports from `env-local.yml`): - -```bash -curl -sS http://127.0.0.1:28093/health # design-challenge -curl -sS http://127.0.0.1:28092/health # prism-challenge +https:///challenge/relearn/... +https:///challenge/relearn-t2i/... +https:///challenge/relearn-mm/... +https:///challenge/bounty/... ``` -Never paste mnemonics or challenge signing keys into miner clients. Hotkeys are -public 64-hex identifiers only. +Never put mnemonics or challenge signing keys in miner clients. +Read `LIUM_API_KEY` from the environment. Do not commit it. diff --git a/docs/external-miner/bounty.md b/docs/external-miner/bounty.md new file mode 100644 index 000000000..0bd58acd4 --- /dev/null +++ b/docs/external-miner/bounty.md @@ -0,0 +1,129 @@ + + +# Bounty Challenge — miners + +Report real Cortex product/backend bugs. Valid unique reports earn subnet +weight. Every report is tagged with your Bittensor hotkey so operators can +patch in real time and pay (or penalize) the right miner. + +Validators do **not** re-run your reports. They verify the sealed weight +bundle. HTTP submit goes through the gateway: + +```text +https:///challenge/bounty/v1/pair +https:///challenge/bounty/v1/reports +``` + +## Dedicated account (required) + +Create a **dedicated** Cortex Chat mining account. Do **not** pair a private +personal account. Pairing is confidential to that account: operators see the +bound hotkey and the reports you file from it. + +A private personal account is for your own chats. A mining account is for +bounty work and is used to fix bugs and to remunerate (or penalize) the +bound hotkey. + +## Terms (blocking) + +You must accept these terms before pairing. The pair API rejects +`terms_accepted: false`. + +> By pairing a Bittensor hotkey to a Cortex Chat account for Bounty +> Challenge, you accept that this dedicated mining account, its logs, and +> its conversations may be used for research, to fix product and backend +> bugs, and to remunerate (or penalize) the bound miner hotkey. Do not pair +> a private personal account. + +## Pair (CLI) + +Never paste a mnemonic into Chat. Sign locally. + +```bash +# Print the Chat inject command + one-time pairing code. +# BOUNTY_CHAT_COMMAND is operator-configured and unguessable. +# Examples below use the public placeholder only. +export BOUNTY_CHAT_COMMAND="${BOUNTY_CHAT_COMMAND:-}" + +cortex-bounty pair --hotkey --account-id +``` + +If you have a local hotkey mini-secret file (32 bytes or 64-hex — not a +mnemonic): + +```bash +cortex-bounty pair \ + --hotkey \ + --account-id \ + --secret-file /path/to/hotkey.sk +``` + +If you sign offline, the CLI prints a challenge string +`cortex-bounty-v1|{account_id}|{nonce}|{exp}`. Sign that exact string with +the hotkey (sr25519, Substrate context), then: + +```bash +cortex-bounty pair \ + --hotkey \ + --account-id \ + --signature <128-hex> +``` + +The CLI prints: + +1. The Chat inject command from `BOUNTY_CHAT_COMMAND` (placeholder + `` when unset — never a live production token). +2. The one-time pairing code. +3. How to pick/switch hotkey if several are linked: + +```bash +# Switch which linked hotkey you are pairing +cortex-bounty pair --hotkey --account-id \ + --wallet-name --wallet-hotkey +``` + +In Cortex Chat, paste the inject command, then the pairing code. That binds +the account to the hotkey and marks the session as bounty-miner. + +## File a report + +After pairing, Chat (or you) posts to the gateway. Optional miner-pays-Lium +header is accepted and never logged; omit it if you are not using Lium. + +```bash +curl -sS -X POST https:///challenge/bounty/v1/reports \ + -H 'content-type: application/json' \ + -H "X-Lium-Api-Key: $LIUM_API_KEY" \ + -d '{ + "session": "", + "hotkey": "", + "title": "", + "body": "", + "repro_steps": "" + }' +``` + +Poll `GET /challenge/bounty/v1/reports/{id}`. That path is internal ingest, +not a public leaderboard. + +**Public consumers** (leaderboard + published reports) hit +**CortexLM/backend** — not this subnet. Cortex **reads** +`/v1/bounty/public/leaderboard` and `/v1/bounty/public/reports` from +`BOUNTY_BACKEND_PUBLIC_URL` (operator env; empty → skip / sim). Scoring +uses `problem_found` + `justification` + hotkey counts on those payloads. + +## Scoring (precision, not volume) + +| Outcome | Result | +|---------|--------| +| Valid unique bug that reproduces | Reward (weight) | +| Already fixed, not yet in prod | Ack only — no reward, no penalty | +| Malicious, fabricated, or does not exist | Penalty (burn toward uid 0) | +| Duplicate of an open report | No extra reward, no penalty | + +Your score is displacement vs the previous bounty champion on a holdout of +adjudicated reports. Stuffing junk reports lowers precision and cannot +crown you. Unmatched emission burns to uid 0. + +Never commit `LIUM_API_KEY`, pairing secrets, or the live Chat inject token. +If something fails, see [troubleshoot.md](./troubleshoot.md). diff --git a/docs/external-miner/design.md b/docs/external-miner/design.md deleted file mode 100644 index 0b342b3a7..000000000 --- a/docs/external-miner/design.md +++ /dev/null @@ -1,213 +0,0 @@ - - -# Design challenge — HTTP harness submit - -**challenge_id:** `design` -**scoring_version:** `3` -**Path:** HTTP only — **no Phala/CVM** - -Normative freeze: [`../DESIGN_CHALLENGE.md`](../DESIGN_CHALLENGE.md). - -## What you submit - -A Python harness bundle (source, not a container image) — prefer a **ZIP**: - -| File | Required | -|------|----------| -| `agent.py` | `def run(task, llm, out) -> None` | -| `pyproject.toml` | Python deps **allowed** — installed at the sandbox install phase | -| Extra files | ≤ 16, ≤ 256 KiB each, total ≤ 1 MiB | - -Optional `env_vars` (API keys, etc.) are injected into the sandbox **run** -phase only — the install phase never sees them. Do not use `DESIGN_*` / -proxy / Python runtime keys. - -### Dependencies (`pyproject.toml`) - -You **may declare any PyPI dependencies** under `[project] dependencies`. -Before your agent runs, the sandbox creates a venv and executes -`pip install --no-cache-dir -e .` against your bundle (install timeout -**300 s**). Build backends run inside the same hardened one-shot container — -no host execution, no miner env vars. - -- Install failure (uninstallable dep, timeout) → error class `install`, which - **auto-retries up to 3 times**; a persistently broken `pyproject.toml` then - fails the run. Watch `GET /v1/runs/{id}/logs` (phase `install`) and - `GET /v1/runs/{id}/events` for `auto_retry` events. -- Keep deps light: pure-Python or prebuilt wheels install fastest; heavy - source builds can exceed the install timeout or sandbox memory. - -### Network access (install + run) - -Both phases reach the **public Internet** through the operator egress proxy -(`HTTP_PROXY` / `HTTPS_PROXY` are set in the sandbox; `pip`, `requests`, -`httpx`, `urllib` honor them). Your agent **may call external APIs and MCP -servers** during the run phase — put the credentials in `env_vars` (locked at -submission, never logged). LLM calls keep going through `llm.chat` (budgeted); -the OpenRouter key is never inside the sandbox. - -Blocked targets (refused with `403`): cloud metadata `169.254.169.254`, -loopback, RFC1918/VPC ranges (`10.0.0.0/8`, `172.16.0.0/12`, -`192.168.0.0/16`), CGNAT `100.64.0.0/10`, and the control plane's internal -services. Blocks are enforced **after DNS resolution** (DNS-rebinding safe). - -The operator injects a non-modifiable `base_design` SDK and runs your harness -inside a hardened Docker sandbox (run timeout **30 minutes**). You never -receive the OpenRouter key or the challenge signing key. - -### Required pages - -Your run must write under `/out/pages/`: - -- `index.html` -- `pricing.html` -- `components.html` - -Missing pages → automatic `Score(0)`. - -## Submit - -```bash -# ZIP via gateway (preferred) -curl -sS -X POST "$BASE_GATEWAY/challenge/design/v1/harness" \ - -H 'content-type: application/zip' \ - -H "X-Miner-Hotkey: <64 lowercase hex>" \ - -H 'X-Env-Json: {"OPENAI_API_KEY":"..."}' \ - --data-binary @harness.zip - -# JSON + zip_base64 -curl -sS -X POST "$BASE_GATEWAY/challenge/design/v1/harness" \ - -H 'content-type: application/json' \ - -d @harness.json - -# Or direct challenge port in local/dev -curl -sS -X POST "http://127.0.0.1:28093/v1/harness" \ - -H 'content-type: application/json' \ - -d @harness.json -``` - -Reference baseline (normative example miners should start from): -[`examples/design-baseline/`](./examples/design-baseline/) — `agent.py` calls -`llm.chat` and writes `index.html` / `pricing.html` / `components.html` via -`out.write_page`. - -Minimal `harness.json` shape: - -```json -{ - "miner_hotkey": "<64 lowercase hex>", - "agent_py": "", - "pyproject_toml": "", - "extra_files": {}, - "env_vars": {} -} -``` - -`POST /v1/harness` is **idempotent** on content digest (`harness_id`). - -## Submission gating (1-max) + auto round enqueue - -- Your hotkey must be **registered on the subnet** (metagraph). Unknown hotkey - → `403 hotkey_not_in_metagraph`. Intake uses a bulk metagraph cache with a - **15 minute** fail-closed TTL (`503 metagraph_unavailable` → retry shortly). -- **One accepted submission per hotkey**. While yours is `registered` / - `blocked` / `rejected`, a *different* harness gets `409 submission_gated`. - Re-POSTing the **identical** bundle is always safe (idempotent - `200 already-queued`). -- After a **terminal** outcome that closes gating (cheat / admin reject / - unscored timeout / budget exhaustion), you cannot submit a **new** digest on - the same hotkey until that hotkey **leaves the metagraph** and you register a - **new UID** (same hotkey is fine). -- Infra auto-retries on the *same* run id (up to 3) are not a new schedule. -- `env_vars` are **locked at submission**; changing them means a new digest, - which requires a free slot. - -## Quotas and rounds - -- **10 rounds per UTC day** (`ROUND_SECS = 8640`; `round_id = floor(unix / 8640)`). -- An accepted harness **waits for the next round**: your `POST /v1/harness` - schedules into `round_id + 1` (never mid-round). After that, the organizer - **auto-enqueues your latest active harness every open round** with that - round's **shared prompt** — you do **not** need to re-POST to keep competing. - Eliminated miners are skipped until their cooldown ends. -- Sandbox **run** timeout is **30 minutes** (`AGENT_RUN_TIMEOUT_SECS = 1800`). -- Each round picks **1 shared prompt** for every harness - (`PROMPTS_PER_ROUND = 1`). -- Daily run quota is **split by origin**: - - **Manual** — **10** runs/day, charged only by your own `POST /v1/harness` - (the initial next-round schedule). - - **Scheduled** — round-loop auto-enqueue / ops requeue (10 rounds × 1 - prompt = **10** runs; cap **20**). You never spend manual quota by being - auto-queued. -- Infra failures (package install, review/LLM infra) **auto-retry up to 3 - times**; cheat / rejected / admin reject / unscored timeout are terminal. - Manual retry of a failed run: `POST /v1/runs/{id}/retry`. - -Check quota: `GET /v1/quota/{hotkey}` — `manual` and `scheduled` objects -(`runs_used` / `limit` / `remaining`) alongside the whole-day `runs_used`. - -## Scoring (summary) - -After sanitize, master-side **agentic anti-cheat** runs in a containerized -reviewer. A pre-LLM **copy gate** rejects a byte/AST copy of an *earlier* -harness outright (`rejected`, `Score(0)`, no LLM call); `cheat` / `suspicious` -from the LLM review → `Score(0)`. Starting from the published **baseline** is -fine — copying another *miner's* harness is not. Both the copy gate and the LLM -review compare you against **other miners' earlier harnesses only**: your own -previous versions (same hotkey **or** same coldkey) are excluded from the -corpus, so iterating via a new hotkey under the same coldkey is never read as -self-copying. - -Clean runs await **admin winners** (1 or 2 harnesses per round); each round win -is one **point**. Rewards are **not** winner-take-all on a single round: the -leaf projection shares `SCORE_MAX` **proportionally to round-win points over -the last 10 rounds** (rolling window, cheat excluded). Prompt bank is -automatic (`bank_v1.json`). Inspiration (Mobbin, image gen, UI libs) and -**external API / MCP calls** are allowed; near-identical corpus copies / -scrape-clones are not. Full rules in the freeze doc. - -If a clean run is still unscored **5 chain epochs** after it entered -`awaiting_admin`, it is **auto-rejected** (`reject_reason` on -`GET /v1/runs/{id}`). Admin may also reject with a reason string you can read -on that same route. Either way you need a **new UID** before submitting again. - -Admin APIs are **master-local only** (not proxied on the public gateway). - -## Viewer - -Screenshots only: `GET /v1/view/{run_id}/index.png` returns the full-page PNG -screenshot the orchestrator captures right after sanitize. Produced HTML is -never served — `.html` requests return `410 Gone` (the gateway still wraps -view responses in a CSP `sandbox` (no scripts) lockdown as defense in depth). -Your pages stay static HTML + **embedded** CSS (`