diff --git a/.env.example b/.env.example index 7c8a925..01c0303 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,9 @@ OPENROUTER_API_KEY= # --- RIMAPI (RimWorld mod) --- RIMAPI_URL=http://localhost:8765 +# Optional override for RimWorld's Saves folder (native load staging). +# Default: OS AppData / Application Support / ~/.config/unity3d/.../Saves +# RLE_RIMWORLD_SAVES= # AppSprout runs: point these at the compiled fork checkout (not Workshop). # Summaries record rimapi_dll_path, rimapi_dll_sha256, and rimapi_fork_commit. # RIMAPI_DLL_PATH=../RIMAPI/1.6/Assemblies/RIMAPI.dll diff --git a/CLAUDE.md b/CLAUDE.md index 18cc516..1d4e033 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ Four things must be set up before RLE can run against a live game: 1. **RimWorld** — Steam install at `C:\Steam\steamapps\common\RimWorld\` (or wherever Steam is) 2. **Harmony + RIMAPI mods** — Subscribe on Steam Workshop, then **enable both** in the in-game Mods menu. Load order: Harmony → Core → Royalty → RIMAPI. RIMAPI exposes REST API on `:8765` + SSE events. 3. **LLM provider** — [LM Studio](https://lmstudio.ai/) (local, port 1234) or [OpenRouter](https://openrouter.ai/) (cloud) -4. **Save file** — `rle_crashlanded_v1` save must exist in RimWorld's save folder (`C:\Users\\AppData\LocalLow\Ludeon Studios\RimWorld by Ludeon Studios\Saves\`). The scenario auto-loads it. +4. **Save file** — `rle_crashlanded_v1` is pinned in `docker/saves/` (YAML `save_sha256`). Native runs stage that file into RimWorld AppData Saves (`C:\Users\\AppData\LocalLow\Ludeon Studios\RimWorld by Ludeon Studios\Saves\`) when the live hash diverges, then `POST /game/load` by name. Docker entrypoint already symlinks `/opt/saves`. ### RIMAPI mod setup (critical) @@ -272,11 +272,12 @@ Tick-specific priorities injected into all Felix agents (other harnesses get no ### Save Loading + Item Setup `run_scenario.py` automatically: -1. Loads the scenario's save file (`rle_crashlanded_v1` seeds a built `SimpleResearchBench` at (128,136) and queues `Smithing`; the other five saves are derived from this base) -2. Polls until game is ready (colonist_count > 0) -3. Unforbids all starting items (via `POST /api/v1/things/set-forbidden`) -4. Runs any `setup_commands` declared in the scenario YAML (spawn_pawn, spawn_item, change_weather, drop_pod) -5. Unpauses game at speed 3 (if `--no-pause`) +1. Native only: `ensure_live_save()` copies `docker/saves/.rws` into RimWorld AppData Saves when the live SHA ≠ YAML `save_sha256` (fail closed). Docker already symlinks `/opt/saves`. Summaries record `live_save_sha256` + `live_save_copied`. +2. Loads the scenario's save file (`rle_crashlanded_v1` seeds a built `SimpleResearchBench` at (128,136) and queues `Smithing`; the other five saves are derived from this base) +3. Polls until game is ready (colonist_count > 0) +4. Unforbids all starting items (via `POST /api/v1/things/set-forbidden`) +5. Runs any `setup_commands` declared in the scenario YAML (spawn_pawn, spawn_item, change_weather, drop_pod) +6. Unpauses game at speed 3 (if `--no-pause`) ### Regenerating scenario saves diff --git a/README.md b/README.md index be98a2e..6df3ace 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ You need four things set up: 1. **RimWorld** (Steam) with **Harmony** and **[RIMAPI](https://github.com/IlyaChichkov/RIMAPI)** mods subscribed and **enabled** in the Mods menu. Load order: Harmony → Core → (DLCs) → RIMAPI. 2. **LLM provider** — [LM Studio](https://lmstudio.ai/) (local, free) or [OpenRouter](https://openrouter.ai/) (cloud) 3. **Python 3.14+** with [uv](https://docs.astral.sh/uv/) -4. **Save file** — `rle_crashlanded_v1` in RimWorld's save folder (the scenario auto-loads it). The Crashlanded seed includes a built `SimpleResearchBench` so research can leave the 7/31 starting floor. +4. **Save file** — `rle_crashlanded_v1` is pinned in `docker/saves/` (YAML `save_sha256`). Native runs copy it into RimWorld's AppData Saves when the live hash diverges, then auto-load by name. The Crashlanded seed includes a built `SimpleResearchBench` so research can leave the 7/31 starting floor. > **RIMAPI note:** The Workshop version may not have our contributed endpoints yet. See [CLAUDE.md](CLAUDE.md) for instructions on building and deploying our fork DLL. AppSprout runs set `RIMAPI_DLL_PATH` and `RIMAPI_FORK_PATH` to the compiled checkout; summaries record that path, the DLL SHA, and the fork commit. Workshop is not source of truth. diff --git a/saves/README.md b/saves/README.md index e7f9dbc..b123428 100644 --- a/saves/README.md +++ b/saves/README.md @@ -30,4 +30,8 @@ gunzip -k rle_crashlanded_v1.rws.gz cp rle_crashlanded_v1.rws ~/.config/unity3d/Ludeon\ Studios/RimWorld\ by\ Ludeon\ Studios/Saves/ ``` -`run_scenario.py` auto-loads the save by name — just make sure the `.rws` file exists in the saves folder. +`run_scenario.py` auto-loads the save by name. Native (non-docker) runs +stage `docker/saves/.rws` into RimWorld's AppData Saves folder when +the live file's SHA-256 does not match the scenario YAML pin, then fail +closed if the copy cannot satisfy the pin. Docker entrypoint already +symlinks `/opt/saves` into the container Saves folder. diff --git a/scripts/create_scenario_saves.py b/scripts/create_scenario_saves.py index 7e0f5e6..d41655c 100644 --- a/scripts/create_scenario_saves.py +++ b/scripts/create_scenario_saves.py @@ -21,37 +21,17 @@ import argparse import asyncio -import os import shutil import sys from pathlib import Path from typing import Any from rle.rimapi.client import RimAPIClient +from rle.scenarios.loader import default_rimworld_saves_dir DOCKER_SAVES_DIR = Path(__file__).resolve().parent.parent / "docker" / "saves" BASE_SAVE = "rle_crashlanded_v1" - -def _default_rimworld_save_dir() -> Path: - """Return RimWorld's default save directory for the current OS.""" - if sys.platform == "win32": - user_profile = Path(os.environ.get("USERPROFILE", "")) - return ( - user_profile / "AppData" / "LocalLow" / "Ludeon Studios" - / "RimWorld by Ludeon Studios" / "Saves" - ) - if sys.platform == "darwin": - return ( - Path.home() / "Library" / "Application Support" - / "RimWorld" / "Saves" - ) - # Linux/other - return ( - Path.home() / ".config" / "unity3d" / "Ludeon Studios" - / "RimWorld by Ludeon Studios" / "Saves" - ) - # Colony center (from the base save) COLONY_X, COLONY_Z = 132, 137 @@ -576,7 +556,7 @@ def main() -> None: save_dir = ( Path(args.save_dir) if args.save_dir - else _default_rimworld_save_dir() + else default_rimworld_saves_dir() ) if not save_dir.exists(): print( diff --git a/scripts/run_benchmark.py b/scripts/run_benchmark.py index 2aa51f6..5b8ef1f 100644 --- a/scripts/run_benchmark.py +++ b/scripts/run_benchmark.py @@ -30,7 +30,7 @@ from rle.orchestration.save_loader import load_save_and_settle from rle.rimapi.client import RimAPIClient from rle.scenarios.evaluator import ScenarioEvaluator -from rle.scenarios.loader import list_scenarios +from rle.scenarios.loader import LiveSavePinError, LiveSaveStatus, list_scenarios from rle.scenarios.schema import ScenarioConfig from rle.scoring.composite import CompositeScorer from rle.scoring.delta import PairedResult, print_paired_leaderboard @@ -67,16 +67,44 @@ class RunError(RuntimeError): """A harness could not be constructed for this run.""" -async def _load_save(client: RimAPIClient, config: RLEConfig, scenario: ScenarioConfig) -> bool: - """Load + settle the scenario save. Returns False when the run must be skipped.""" +async def _load_save( + client: RimAPIClient, + config: RLEConfig, + scenario: ScenarioConfig, + *, + stage_live: bool = False, +) -> tuple[bool, LiveSaveStatus | None]: + """Load + settle the scenario save. + + Returns ``(ok, live_save)``. ``ok`` is False when the run must be skipped + for a transient load failure. ``LiveSavePinError`` is not caught — a + native pin miss fails closed instead of loading a stale AppData file. + """ if not scenario.save_name: - return True + return True, None try: - await load_save_and_settle(client, config.rimapi_url, scenario.save_name) + loaded = await load_save_and_settle( + client, config.rimapi_url, scenario.save_name, + save_sha256=scenario.save_sha256, + stage_live=stage_live, + ) + except LiveSavePinError: + raise except Exception as e: logger.warning("Could not load save %s: %s", scenario.save_name, e) - return False - return True + return False, None + if loaded.live_save is not None: + logger.info( + "live_save_sha256=%s copied=%s", + loaded.live_save.live_save_sha256, + loaded.live_save.copied, + ) + print( + f" live save {scenario.save_name}: " + f"sha256={loaded.live_save.live_save_sha256} " + f"copied={loaded.live_save.copied}", + ) + return True, loaded.live_save def _harness_failed(event_log: EventLog | None, start_index: int) -> bool: @@ -287,9 +315,17 @@ async def _run_ablation( # noqa: PLR0913 for scenario in scenarios: for run_id in range(num_runs): run_label = f" (run {run_id + 1}/{num_runs})" if num_runs > 1 else "" - if not use_mock_rimapi and not await _load_save(client, config, scenario): - print(f" SKIP {scenario.name}{run_label} ({tag}): save load failed") - continue + if not use_mock_rimapi: + loaded_ok, _live = await _load_save( + client, config, scenario, + stage_live=not args.docker, + ) + if not loaded_ok: + print( + f" SKIP {scenario.name}{run_label} " + f"({tag}): save load failed", + ) + continue print(f" {scenario.name}{run_label} ({tag})...") options = {**harness_options, "exclude_agent": exclude} @@ -477,9 +513,15 @@ async def main(args: argparse.Namespace) -> None: # noqa: PLR0912, PLR0915 for run_id in range(num_runs): run_label = f" (run {run_id + 1}/{num_runs})" if num_runs > 1 else "" - if not use_mock_rimapi and not await _load_save(client, config, scenario): - print(f" SKIP {scenario.name}{run_label}: save load failed") - continue + live_save: LiveSaveStatus | None = None + if not use_mock_rimapi: + loaded_ok, live_save = await _load_save( + client, config, scenario, + stage_live=not args.docker, + ) + if not loaded_ok: + print(f" SKIP {scenario.name}{run_label}: save load failed") + continue print( f"\nRunning: {scenario.name} ({scenario.difficulty}) " @@ -499,6 +541,9 @@ async def main(args: argparse.Namespace) -> None: # noqa: PLR0912, PLR0915 except RunError as exc: exit_with_harness_error(exc) return + if live_save is not None: + result["live_save_sha256"] = live_save.live_save_sha256 + result["live_save_copied"] = live_save.copied results.append(result) if paired: paired.agent_scores.append(result["score"]) @@ -512,7 +557,11 @@ async def main(args: argparse.Namespace) -> None: # noqa: PLR0912, PLR0915 # Baseline run (reload same save, unmanaged colony) if paired is not None: - if not await _load_save(client, config, scenario): + reload_ok, _live = await _load_save( + client, config, scenario, + stage_live=not args.docker, + ) + if not reload_ok: logger.warning("Could not reload save for baseline") print(f" baseline{run_label}...") baseline = await _run_scenario( diff --git a/scripts/run_scenario.py b/scripts/run_scenario.py index ccfa57f..95ecc80 100644 --- a/scripts/run_scenario.py +++ b/scripts/run_scenario.py @@ -30,7 +30,7 @@ from rle.rimapi.client import RimAPIClient from rle.rimapi.sse_client import RimAPISSEClient from rle.scenarios.evaluator import ScenarioEvaluator -from rle.scenarios.loader import list_scenarios, load_scenario +from rle.scenarios.loader import LiveSavePinError, list_scenarios, load_scenario from rle.scoring.composite import CompositeScorer from rle.scoring.recorder import TimeSeriesRecorder from rle.tracking.cost_tracker import create_cost_tracker, fetch_billed_costs @@ -83,10 +83,17 @@ def _build_run_summary( # noqa: PLR0913 cost_snapshot_dict: dict[str, object], event_summary_dict: dict[str, object] | None, billed_cost_dict: dict[str, object] | None = None, + live_save_sha256: str | None = None, + live_save_copied: bool | None = None, ) -> dict[str, object]: """Compose the per-scenario summary JSON (metadata + config + result).""" summary: dict[str, object] = { - **collect_metadata(random_seed=args.seed, harness_describe=harness_describe), + **collect_metadata( + random_seed=args.seed, + harness_describe=harness_describe, + live_save_sha256=live_save_sha256, + live_save_copied=live_save_copied, + ), "scenario": scenario_name, "scenario_save_name": scenario_save_name, "harness": harness_name, @@ -210,17 +217,33 @@ async def main(args: argparse.Namespace) -> None: sse = RimAPISSEClient(config.rimapi_url) sse_task = asyncio.create_task(sse.listen()) + live_save_sha256: str | None = None + live_save_copied: bool | None = None async with RimAPIClient(config.rimapi_url) as client: - # Load the scenario's save file for a consistent starting state + # Load the scenario's save file for a consistent starting state. + # Native path: stage docker/saves → AppData when the live hash ≠ pin + # (Docker entrypoint already symlinks /opt/saves; this CLI is native). if scenario.save_name: print(f"Loading save: {scenario.save_name}") try: - unforbid_count = await load_save_and_settle( + loaded = await load_save_and_settle( client, config.rimapi_url, scenario.save_name, + save_sha256=scenario.save_sha256, + stage_live=True, ) - if unforbid_count: - print(f"Unforbid {unforbid_count} items.") + if loaded.live_save is not None: + live_save_sha256 = loaded.live_save.live_save_sha256 + live_save_copied = loaded.live_save.copied + print( + f"Live save sha256={live_save_sha256} " + f"copied={live_save_copied}", + ) + if loaded.unforbid_count: + print(f"Unforbid {loaded.unforbid_count} items.") print("Save loaded, game ready.") + except LiveSavePinError as e: + print(f"ERROR: Refusing to load save '{scenario.save_name}': {e}") + raise SystemExit(1) from e except Exception as e: print(f"Warning: Could not load save '{scenario.save_name}': {e}") print("Continuing with current game state...") @@ -353,6 +376,8 @@ async def main(args: argparse.Namespace) -> None: billed_cost_dict=( billed_report.model_dump() if billed_report else None ), + live_save_sha256=live_save_sha256, + live_save_copied=live_save_copied, ) summary_path = output_dir / f"{scenario_path.stem}_summary.json" summary_path.write_text(json.dumps(summary, indent=2, default=str)) diff --git a/src/rle/orchestration/save_loader.py b/src/rle/orchestration/save_loader.py index 2c2eb48..87825ac 100644 --- a/src/rle/orchestration/save_loader.py +++ b/src/rle/orchestration/save_loader.py @@ -12,9 +12,11 @@ import asyncio import logging +from dataclasses import dataclass from rle.docker import wait_for_rimapi from rle.rimapi.client import RimAPIClient +from rle.scenarios.loader import LiveSaveStatus, ensure_live_save logger = logging.getLogger(__name__) @@ -25,6 +27,14 @@ MAX_POLLS = 30 +@dataclass(frozen=True) +class LoadSettleResult: + """Outcome of ``load_save_and_settle`` (unforbid count + optional staging).""" + + unforbid_count: int + live_save: LiveSaveStatus | None = None + + async def load_save_and_settle( client: RimAPIClient, rimapi_url: str, @@ -32,13 +42,30 @@ async def load_save_and_settle( *, unforbid_items: bool = True, rimapi_timeout_s: float = 30.0, -) -> int: + save_sha256: str | None = None, + stage_live: bool = False, +) -> LoadSettleResult: """Load ``save_name`` and block until the colony is stable. - Returns the number of starting items unforbidden (0 when disabled). - Raises whatever ``load_game`` / ``wait_for_rimapi`` raise so callers can - decide whether to skip the run. + When ``stage_live`` is True and ``save_sha256`` is set (native path), + copy ``docker/saves/.rws`` into RimWorld AppData Saves if the live + file does not already match the pin. Docker skips this — the entrypoint + already symlinks ``/opt/saves``. Staging failures raise + ``LiveSavePinError`` (fail closed) before ``game/load``. + + Returns ``LoadSettleResult`` (unforbid count + live-save status). + Raises whatever ``load_game`` / ``wait_for_rimapi`` / staging raise so + callers can decide whether to skip the run. """ + live_save: LiveSaveStatus | None = None + if stage_live and save_sha256: + live_save = ensure_live_save(save_name, save_sha256) + logger.info( + "live_save_sha256=%s copied=%s path=%s", + live_save.live_save_sha256, + live_save.copied, + live_save.live_path, + ) await client.load_game(save_name) await wait_for_rimapi(rimapi_url, timeout=rimapi_timeout_s) stable_count = 0 @@ -60,6 +87,6 @@ async def load_save_and_settle( else: logger.warning("Save %s never reported a stable population; continuing", save_name) if not unforbid_items: - return 0 + return LoadSettleResult(unforbid_count=0, live_save=live_save) count = await client.unforbid_all_items() - return int(count or 0) + return LoadSettleResult(unforbid_count=int(count or 0), live_save=live_save) diff --git a/src/rle/scenarios/loader.py b/src/rle/scenarios/loader.py index 45cc1f1..a7428eb 100644 --- a/src/rle/scenarios/loader.py +++ b/src/rle/scenarios/loader.py @@ -2,6 +2,11 @@ from __future__ import annotations +import logging +import os +import shutil +import sys +from dataclasses import dataclass from pathlib import Path import yaml @@ -10,23 +15,161 @@ from rle.tracking.metadata import SCORING_VERSION, file_sha256 # Canonical save mirror (the same files that get baked into the Docker image). -# Resolves to /docker/saves/. Live game runs may use a save in -# RimWorld's AppData that diverges — that divergence is the bug the pinned -# save_sha256 is meant to surface, not silently absorb. +# Resolves to /docker/saves/. The YAML pin is checked against this +# path. Native (non-docker) ``POST /game/load`` reads RimWorld AppData Saves, +# not docker/saves — ``ensure_live_save()`` copies the canonical file there +# when the live hash ≠ the pin. Docker entrypoint already symlinks /opt/saves. _REPO_ROOT = Path(__file__).resolve().parents[3] _CANONICAL_SAVES_DIR = _REPO_ROOT / "docker" / "saves" +logger = logging.getLogger(__name__) + class ScenarioSaveMismatchError(RuntimeError): """Raised when a scenario's pinned save_sha256 doesn't match the on-disk save file. Bypass with allow_unpinned=True (intentional override only).""" +class LiveSavePinError(RuntimeError): + """Raised when the native AppData save cannot be made to match the pin. + + Fail-closed: missing canonical, unreadable copy, or a post-copy hash that + still disagrees with ``save_sha256`` all abort rather than load a stale + file via ``POST /api/v1/game/load``. + """ + + +@dataclass(frozen=True) +class LiveSaveStatus: + """Result of staging a pinned save into RimWorld's live Saves folder.""" + + save_name: str + live_path: Path + live_save_sha256: str + copied: bool + pinned_sha256: str + + def canonical_save_path(save_name: str) -> Path: """The pinned, repo-mirrored .rws file path for a given save name.""" return _CANONICAL_SAVES_DIR / f"{save_name}.rws" +def default_rimworld_saves_dir() -> Path: + """RimWorld's OS-specific Saves folder (what ``POST /game/load`` reads). + + Honors ``$RLE_RIMWORLD_SAVES`` when set (tests / nonstandard installs). + Otherwise: Windows AppData LocalLow, macOS Application Support, or the + Linux Unity ``~/.config/unity3d/Ludeon Studios/.../Saves`` path. + """ + override = os.environ.get("RLE_RIMWORLD_SAVES") + if override: + return Path(override) + if sys.platform == "win32": + user_profile = Path(os.environ.get("USERPROFILE", "")) + return ( + user_profile / "AppData" / "LocalLow" / "Ludeon Studios" + / "RimWorld by Ludeon Studios" / "Saves" + ) + if sys.platform == "darwin": + return ( + Path.home() / "Library" / "Application Support" + / "RimWorld" / "Saves" + ) + return ( + Path.home() / ".config" / "unity3d" / "Ludeon Studios" + / "RimWorld by Ludeon Studios" / "Saves" + ) + + +def live_save_path(save_name: str, *, live_dir: Path | None = None) -> Path: + """``/.rws`` — the file RimWorld loads by name.""" + return (live_dir or default_rimworld_saves_dir()) / f"{save_name}.rws" + + +def ensure_live_save( + save_name: str, + expected_sha256: str, + *, + live_dir: Path | None = None, + canonical_path: Path | None = None, +) -> LiveSaveStatus: + """Make the live AppData save match the scenario pin before ``game/load``. + + If the live file already hashes to ``expected_sha256``, this is a no-op. + Otherwise copy ``docker/saves/.rws`` (or ``canonical_path``) into + the live Saves folder and re-hash. Raises ``LiveSavePinError`` when the + canonical file is missing or the live file still disagrees after copy. + """ + if not save_name: + raise ValueError("save_name is required") + if not expected_sha256: + raise ValueError("expected_sha256 is required") + + source = canonical_path if canonical_path is not None else canonical_save_path(save_name) + canonical_hash = file_sha256(source) + if canonical_hash is None: + raise LiveSavePinError( + f"Canonical save for {save_name!r} is missing at {source}. " + "Cannot stage a pinned live save; fail closed.", + ) + if canonical_hash != expected_sha256: + raise LiveSavePinError( + f"Canonical save {source} hashes to {canonical_hash} but the " + f"scenario pins {expected_sha256}. Re-pin via scripts/hash_saves.py " + "or restore docker/saves/; fail closed.", + ) + + dest = live_save_path(save_name, live_dir=live_dir) + live_hash = file_sha256(dest) + if live_hash == expected_sha256: + logger.info( + "Live save %s already matches pin sha256=%s (%s)", + save_name, expected_sha256, dest, + ) + return LiveSaveStatus( + save_name=save_name, + live_path=dest, + live_save_sha256=live_hash, + copied=False, + pinned_sha256=expected_sha256, + ) + + _copy_canonical_to_live(source, dest) + copied_hash = file_sha256(dest) + if copied_hash != expected_sha256: + raise LiveSavePinError( + f"Copied {source} to {dest} but live hash is {copied_hash}, " + f"expected pin {expected_sha256}. Fail closed.", + ) + logger.info( + "Staged live save %s from %s -> %s sha256=%s (was %s)", + save_name, source, dest, copied_hash, live_hash, + ) + return LiveSaveStatus( + save_name=save_name, + live_path=dest, + live_save_sha256=copied_hash, + copied=True, + pinned_sha256=expected_sha256, + ) + + +def _copy_canonical_to_live(source: Path, dest: Path) -> None: + """Atomically replace the live save with the canonical file.""" + dest.parent.mkdir(parents=True, exist_ok=True) + tmp_path = dest.with_name(dest.name + ".staging") + try: + shutil.copy2(source, tmp_path) + os.replace(tmp_path, dest) + except OSError as exc: + if tmp_path.exists(): + tmp_path.unlink(missing_ok=True) + raise LiveSavePinError( + f"Failed to copy canonical save {source} to {dest}: {exc}", + ) from exc + + def load_scenario( path: str | Path, *, allow_unpinned: bool = False, ) -> ScenarioConfig: diff --git a/src/rle/scenarios/schema.py b/src/rle/scenarios/schema.py index 5ca05ea..e2178b3 100644 --- a/src/rle/scenarios/schema.py +++ b/src/rle/scenarios/schema.py @@ -64,8 +64,10 @@ class ScenarioConfig(BaseModel): save_name: str = "" save_sha256: str | None = None """Pinned SHA-256 of the docker/saves/.rws file. When set, the - loader compares it against the on-disk save and refuses to start on - mismatch unless allow_unpinned=True. Generate via scripts/hash_saves.py.""" + loader compares it against the canonical mirror and refuses to start on + mismatch unless allow_unpinned=True. Native loads also stage that file + into RimWorld AppData Saves before POST /game/load. Generate via + scripts/hash_saves.py.""" triggered_incidents: list[TriggeredIncident] = [] setup_commands: list[SetupCommand] = [] diff --git a/src/rle/tracking/metadata.py b/src/rle/tracking/metadata.py index 9b01861..69ce70b 100644 --- a/src/rle/tracking/metadata.py +++ b/src/rle/tracking/metadata.py @@ -40,6 +40,8 @@ def collect_metadata( random_seed: int | None = None, harness_describe: dict[str, str] | None = None, + live_save_sha256: str | None = None, + live_save_copied: bool | None = None, ) -> dict[str, object]: """Gather reproducibility metadata for a benchmark run. @@ -52,6 +54,10 @@ def collect_metadata( (``BaseHarness.describe()``): SDK versions, agent roster, external tool versions. Recorded as ``harness_versions`` so a leaderboard row can be traced to the exact harness build, whichever framework it used. + + ``live_save_sha256`` / ``live_save_copied`` record whether the native + AppData save was already at the scenario pin or had to be copied from + ``docker/saves/`` before ``POST /game/load``. """ dll_path = _rimapi_dll_path() return { @@ -69,6 +75,8 @@ def collect_metadata( "rimapi_dll_path": str(dll_path) if dll_path else None, "rimapi_dll_sha256": file_sha256(dll_path) if dll_path else None, "rimapi_fork_commit": _rimapi_fork_commit(), + "live_save_sha256": live_save_sha256, + "live_save_copied": live_save_copied, } diff --git a/tests/unit/test_live_save.py b/tests/unit/test_live_save.py new file mode 100644 index 0000000..4773cc1 --- /dev/null +++ b/tests/unit/test_live_save.py @@ -0,0 +1,287 @@ +"""Native AppData save staging against the scenario pin.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from rle.orchestration import save_loader as save_loader_mod +from rle.orchestration.save_loader import load_save_and_settle +from rle.scenarios import loader as loader_mod +from rle.scenarios.loader import ( + LiveSavePinError, + LiveSaveStatus, + default_rimworld_saves_dir, + ensure_live_save, + live_save_path, +) +from rle.tracking.metadata import file_sha256 + + +def _write_save(path: Path, payload: bytes) -> str: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + return hashlib.sha256(payload).hexdigest() + + +def test_ensure_live_save_copies_on_mismatch(tmp_path: Path) -> None: + canonical = tmp_path / "canonical" / "rle_crashlanded_v1.rws" + pin = _write_save(canonical, b"canonical-bench-and-smithing") + live_dir = tmp_path / "AppData" / "Saves" + live = live_dir / "rle_crashlanded_v1.rws" + _write_save(live, b"april-stale-appdata") + + status = ensure_live_save( + "rle_crashlanded_v1", + pin, + live_dir=live_dir, + canonical_path=canonical, + ) + + assert status.copied is True + assert status.live_save_sha256 == pin + assert status.pinned_sha256 == pin + assert live.read_bytes() == canonical.read_bytes() + assert file_sha256(live) == pin + + +def test_ensure_live_save_noop_when_live_already_matches( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + payload = b"already-pinned" + pin = hashlib.sha256(payload).hexdigest() + canonical = tmp_path / "canonical" / "rle_crashlanded_v1.rws" + _write_save(canonical, payload) + live_dir = tmp_path / "Saves" + live = live_dir / "rle_crashlanded_v1.rws" + _write_save(live, payload) + + def _boom(*_args: object, **_kwargs: object) -> None: + raise AssertionError("matching live save must not copy") + + monkeypatch.setattr(loader_mod.shutil, "copy2", _boom) + + status = ensure_live_save( + "rle_crashlanded_v1", + pin, + live_dir=live_dir, + canonical_path=canonical, + ) + assert status.copied is False + assert status.live_save_sha256 == pin + assert live.read_bytes() == payload + + +def test_ensure_live_save_copies_when_live_missing(tmp_path: Path) -> None: + canonical = tmp_path / "docker" / "saves" / "rle_crashlanded_v1.rws" + pin = _write_save(canonical, b"new-seed") + live_dir = tmp_path / "Saves" + + status = ensure_live_save( + "rle_crashlanded_v1", + pin, + live_dir=live_dir, + canonical_path=canonical, + ) + assert status.copied is True + assert (live_dir / "rle_crashlanded_v1.rws").read_bytes() == b"new-seed" + assert status.live_save_sha256 == pin + + +def test_ensure_live_save_fails_closed_when_canonical_missing(tmp_path: Path) -> None: + live_dir = tmp_path / "Saves" + missing = tmp_path / "docker" / "saves" / "rle_crashlanded_v1.rws" + with pytest.raises(LiveSavePinError, match="missing"): + ensure_live_save( + "rle_crashlanded_v1", + "a" * 64, + live_dir=live_dir, + canonical_path=missing, + ) + assert not (live_dir / "rle_crashlanded_v1.rws").exists() + + +def test_ensure_live_save_fails_closed_when_canonical_hash_wrong(tmp_path: Path) -> None: + canonical = tmp_path / "canonical.rws" + _write_save(canonical, b"not-the-pin") + live_dir = tmp_path / "Saves" + with pytest.raises(LiveSavePinError, match="hashes to"): + ensure_live_save( + "rle_crashlanded_v1", + "b" * 64, + live_dir=live_dir, + canonical_path=canonical, + ) + + +def test_ensure_live_save_fails_closed_when_post_copy_hash_wrong( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + canonical = tmp_path / "canonical.rws" + pin = _write_save(canonical, b"good-canonical") + live_dir = tmp_path / "Saves" + + def _corrupt_copy(src: object, dst: object) -> None: + Path(str(dst)).write_bytes(b"corrupted-copy") + + monkeypatch.setattr(loader_mod.shutil, "copy2", _corrupt_copy) + + with pytest.raises(LiveSavePinError, match="Copied"): + ensure_live_save( + "rle_crashlanded_v1", + pin, + live_dir=live_dir, + canonical_path=canonical, + ) + + +def test_ensure_live_save_fails_closed_when_copy_raises( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + canonical = tmp_path / "canonical.rws" + pin = _write_save(canonical, b"good-canonical") + live_dir = tmp_path / "Saves" + + def _io_error(*_args: object, **_kwargs: object) -> None: + raise OSError("disk full") + + monkeypatch.setattr(loader_mod.shutil, "copy2", _io_error) + + with pytest.raises(LiveSavePinError, match="Failed to copy"): + ensure_live_save( + "rle_crashlanded_v1", + pin, + live_dir=live_dir, + canonical_path=canonical, + ) + + +def test_live_save_path_uses_live_dir(tmp_path: Path) -> None: + dest = live_save_path("rle_first_winter_v1", live_dir=tmp_path) + assert dest == tmp_path / "rle_first_winter_v1.rws" + + +def test_default_rimworld_saves_dir_honors_env( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("RLE_RIMWORLD_SAVES", str(tmp_path / "custom")) + assert default_rimworld_saves_dir() == tmp_path / "custom" + + +def test_default_rimworld_saves_dir_windows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("RLE_RIMWORLD_SAVES", raising=False) + monkeypatch.setattr(loader_mod.sys, "platform", "win32") + monkeypatch.setenv("USERPROFILE", r"C:\Users\keeper") + assert default_rimworld_saves_dir() == ( + Path(r"C:\Users\keeper") / "AppData" / "LocalLow" / "Ludeon Studios" + / "RimWorld by Ludeon Studios" / "Saves" + ) + + +class _FakeColonyClient: + def __init__(self, order: list[str] | None = None) -> None: + self.loads: list[str] = [] + self.order = order if order is not None else [] + + async def load_game(self, save_name: str) -> None: + self.order.append(f"load:{save_name}") + self.loads.append(save_name) + + async def get_colony(self) -> SimpleNamespace: + return SimpleNamespace(population=3) + + async def unforbid_all_items(self) -> int: + return 4 + + +async def test_load_save_and_settle_stages_before_game_load( + monkeypatch: pytest.MonkeyPatch, +) -> None: + order: list[str] = [] + pin = "c" * 64 + status = LiveSaveStatus( + save_name="rle_crashlanded_v1", + live_path=Path("/tmp/rle_crashlanded_v1.rws"), + live_save_sha256=pin, + copied=True, + pinned_sha256=pin, + ) + + def fake_ensure(save_name: str, expected: str, **_kwargs: object) -> LiveSaveStatus: + order.append(f"stage:{save_name}:{expected}") + return status + + async def fake_wait(_url: str, timeout: float = 30.0) -> None: + order.append("wait") + + monkeypatch.setattr(save_loader_mod, "ensure_live_save", fake_ensure) + monkeypatch.setattr(save_loader_mod, "wait_for_rimapi", fake_wait) + monkeypatch.setattr(save_loader_mod, "STABLE_POLLS_REQUIRED", 1) + monkeypatch.setattr(save_loader_mod, "POLL_INTERVAL_S", 0.0) + + client = _FakeColonyClient(order) + result = await load_save_and_settle( + client, # type: ignore[arg-type] + "http://localhost:8765", + "rle_crashlanded_v1", + save_sha256=pin, + stage_live=True, + ) + assert order[0] == f"stage:rle_crashlanded_v1:{pin}" + assert order[1] == "load:rle_crashlanded_v1" + assert result.unforbid_count == 4 + assert result.live_save == status + assert result.live_save.copied is True + + +async def test_load_save_and_settle_skips_staging_when_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _boom(*_args: object, **_kwargs: object) -> LiveSaveStatus: + raise AssertionError("docker/smoke path must not stage AppData") + + async def fake_wait(_url: str, timeout: float = 30.0) -> None: + return None + + monkeypatch.setattr(save_loader_mod, "ensure_live_save", _boom) + monkeypatch.setattr(save_loader_mod, "wait_for_rimapi", fake_wait) + monkeypatch.setattr(save_loader_mod, "STABLE_POLLS_REQUIRED", 1) + monkeypatch.setattr(save_loader_mod, "POLL_INTERVAL_S", 0.0) + + client = _FakeColonyClient() + result = await load_save_and_settle( + client, # type: ignore[arg-type] + "http://localhost:8765", + "rle_crashlanded_v1", + save_sha256="d" * 64, + stage_live=False, + unforbid_items=False, + ) + assert client.loads == ["rle_crashlanded_v1"] + assert result.live_save is None + assert result.unforbid_count == 0 + + +async def test_load_save_and_settle_pin_error_before_load( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_ensure(*_args: object, **_kwargs: object) -> LiveSaveStatus: + raise LiveSavePinError("canonical missing") + + client = _FakeColonyClient() + monkeypatch.setattr(save_loader_mod, "ensure_live_save", fake_ensure) + + with pytest.raises(LiveSavePinError, match="canonical missing"): + await load_save_and_settle( + client, # type: ignore[arg-type] + "http://localhost:8765", + "rle_crashlanded_v1", + save_sha256="e" * 64, + stage_live=True, + ) + assert client.loads == [] diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index 15d29f5..58d9a44 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -58,6 +58,8 @@ def test_collect_metadata_includes_scoring_version_and_seed() -> None: "rimapi_dll_path", "rimapi_dll_sha256", "rimapi_fork_commit", + "live_save_sha256", + "live_save_copied", ): assert key in md, f"missing metadata field: {key}" @@ -71,6 +73,17 @@ def test_collect_metadata_records_harness_describe() -> None: def test_collect_metadata_default_seed_is_none() -> None: md = collect_metadata() assert md["random_seed"] is None + assert md["live_save_sha256"] is None + assert md["live_save_copied"] is None + + +def test_collect_metadata_records_live_save_staging() -> None: + md = collect_metadata( + live_save_sha256="a" * 64, + live_save_copied=True, + ) + assert md["live_save_sha256"] == "a" * 64 + assert md["live_save_copied"] is True def test_collect_metadata_dll_path_and_hash_pair_consistently() -> None: