diff --git a/CHANGELOG.md b/CHANGELOG.md index aeecdc4..020b890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,32 @@ All notable changes to the CacheKit Protocol Specification. ### Test Vectors +- `tools/python-frame-reference.py generate` now **upserts by vector name** + (LAB-1203): it rebuilds only the vectors the installed `cachekit` wheel can + reproduce and leaves every other committed vector byte-untouched, so dropping + a committed vector is structurally impossible — which deletes the LAB-903 + drop-refusal guard and both wheel-direction refusals, and folds the + append-only `generate-bin-twin` mode into `generate` (a protocol 1.1 wheel + rebuilds the `_bin` twin, a legacy wheel the legacy original; whenever both + default-path vectors are present the pair is still proven to differ only in + envelope encoding before writing, and a partial fixture missing either twin + skips that proof with a stderr note rather than aborting). + The ByteStorage envelope codec is no longer reimplemented there: encode/decode + come from `tools/wire-format-reference.py`, the one shared implementation of + the encoding these fixtures pin. Rewritten vectors carry per-vector + `generator` provenance (and the top-level provenance flips to an explicit + "mixed provenance" statement the first time a previously-unstamped vector is + rewritten); `test-vectors/python-frame.json` is byte-unchanged by this + refactor, and a no-op `generate` never rewrites the file. The stdlib `verify` + leg got strictly stronger (expert-panel findings): it now fully decodes each + `payload_envelope` via the shared codec (enforcing the protocol 1.1 flip + exclusions — checksum stays an array of 8 integers, format stays fixstr), + requires the envelope to re-encode byte-identically (pinning the canonical + rmp_serde shortest-form encoding, including the outer fixarray(4) marker), + and pins the declared `compressed_data_hex`/`checksum_hex`/`original_size`/ + `format` fields against the actual envelope bytes; the generate-time twin + proof now compares frame prefixes at the byte level, not as parsed JSON. + - 7 legacy/`bin` vector pairs in `test-vectors/wire-format.json` (append-only; legacy vectors are retained forever as legacy-read proof; fixture 1.0.0 → 1.1.1). The original six `bin` twins were generated by the stdlib-only diff --git a/tools/python-frame-reference.py b/tools/python-frame-reference.py index a9ec739..8427da4 100644 --- a/tools/python-frame-reference.py +++ b/tools/python-frame-reference.py @@ -13,36 +13,40 @@ independent minimal parser (no cachekit import) and checks the expected header/payload; checks every error vector is rejected. Runs in CI. - generate Regenerates the vector file. Requires the real `cachekit` - package (PyPI wheel with the Rust core) plus `msgpack`; - the Arrow vector additionally needs `pyarrow` + `pandas`. - Every generated frame is round-tripped through the real - cachekit-py deserialization path before being written. - Rewrites the whole fixture, so it refuses to write if that - would drop any already-committed vector — the legacy vectors - were generated by cachekit 0.11.1 (array-of-ints envelopes) - and no post-0.4.0-core wheel (bin envelopes) can reproduce - them. Use generate-bin-twin for post-protocol-1.1 updates. - - generate-bin-twin - Append-only: generates ONLY the protocol 1.1 `bin`-envelope - twin of the default-path vector from the installed `cachekit` - wheel and appends it to the existing fixture, leaving every - other vector byte-untouched (LAB-903). Requires `cachekit` + - `msgpack`; refuses to write if the wheel still emits - array-of-ints envelopes. + generate Upserts the vector file by vector name (LAB-1203): every vector + the installed wheel can reproduce is rebuilt, and rewritten only + if its content actually changed; every other committed vector is + left byte-untouched. The fixture is edited in place, never + rebuilt from scratch, so dropping a committed vector is + structurally impossible. Requires the real `cachekit` package + (PyPI wheel with the Rust core) plus `msgpack`; the Arrow vector + is rebuilt only when `pyarrow` + `pandas` are importable and is + otherwise left as committed. A wheel emitting protocol 1.1 `bin` + envelopes rebuilds the `_bin` twin of the default-path vector; a + legacy (array-of-ints) wheel rebuilds the legacy original — the + vector a wheel cannot produce is simply not touched. Every + generated frame is round-tripped through the real cachekit-py + deserialization path before being written, and the default-path + pair must differ ONLY in envelope encoding. The independent parser below implements exactly the layout documented in spec/wire-format.md: MAGIC b"CK" | VERSION u8 (=3) | HDR_LEN u32-BE | HEADER (UTF-8 JSON) | PAYLOAD + +The ByteStorage envelope codec is NOT reimplemented here: encode/decode come +from tools/wire-format-reference.py, the single shared implementation of the +encoding these fixtures exist to pin (stdlib-only, so `verify` stays +dependency-free). """ from __future__ import annotations +import importlib.util import json import sys from pathlib import Path +from types import ModuleType VECTOR_PATH = Path(__file__).resolve().parent.parent / "test-vectors" / "python-frame.json" @@ -51,6 +55,20 @@ PREFIX_LEN = 7 # magic(2) + version(1) + header_len(4) +def _load_wire_format_codec() -> ModuleType: + """Load tools/wire-format-reference.py as a module (hyphenated filename).""" + path = Path(__file__).resolve().parent / "wire-format-reference.py" + spec = importlib.util.spec_from_file_location("wire_format_reference", path) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load envelope codec from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_wire = _load_wire_format_codec() + + class FrameError(ValueError): pass @@ -94,29 +112,6 @@ def parse_frame(frame: bytes) -> tuple[dict, bytes]: return header, frame[header_end:] -#: msgpack tags a ByteStorage envelope's compressed_data field can legally carry. -#: bin8/bin16/bin32 are the protocol 1.1 canonical encoding; fixarray/array16/ -#: array32 are the legacy array-of-ints encoding readers must accept forever. -_BIN_TAGS = (0xC4, 0xC5, 0xC6) -_INT_ARRAY_TAGS = (0xDC, 0xDD, *range(0x90, 0xA0)) - - -def _envelope_encoding(payload: bytes) -> str: - """Classify an envelope's compressed_data encoding from its msgpack tag. - - Stdlib-only, so the no-dependency CI leg proves the protocol 1.1 dual-read - property on its own instead of delegating it to the Node cross-check. - """ - if len(payload) < 2 or payload[0] != 0x94: - raise FrameError(f"envelope is not a msgpack fixarray(4) (leading byte 0x{payload[:1].hex()})") - tag = payload[1] - if tag in _BIN_TAGS: - return "bin" - if tag in _INT_ARRAY_TAGS: - return "int-array" - raise FrameError(f"compressed_data tag 0x{tag:02x} is neither msgpack bin nor an array") - - def verify() -> int: doc = _load_fixture() failures = 0 @@ -145,17 +140,44 @@ def verify() -> int: print(f"FAIL {name}: payload_envelope must declare envelope_encoding ('bin' or 'int-array')") vec_failed += 1 else: + # Shared, stdlib-only codec (wire-format-reference.py), so the + # no-dependency CI leg proves the protocol 1.1 dual-read + # property on its own instead of delegating it to the Node + # cross-check. decode_envelope also enforces the exclusions + # from the 1.1 flip: checksum stays an array of 8 integers and + # format stays a fixstr, in BOTH encodings. try: - actual = _envelope_encoding(payload) - except FrameError as e: - print(f"FAIL {name}: {e}") + data, checksum, size, fmt, actual = _wire.decode_envelope(payload) + except ValueError as e: + print(f"FAIL {name}: envelope decode: {e}") vec_failed += 1 else: if actual != declared: print(f"FAIL {name}: compressed_data is {actual}, vector declares {declared}") vec_failed += 1 + elif _wire.encode_envelope(data, checksum, size, fmt, encoding=actual) != payload: + # decode_envelope tolerates reader-lenient forms no + # rmp_serde writer emits (array16/array32 outer header, + # non-shortest uints); re-encode byte-fidelity pins the + # canonical writer form, incl. the fixarray(4) marker. + print(f"FAIL {name}: envelope is not in canonical shortest-form encoding (re-encode differs)") + vec_failed += 1 else: - observed_encodings.add(actual) + drifted = [ + fname + for fname, got in ( + ("compressed_data_hex", data.hex()), + ("checksum_hex", checksum.hex()), + ("original_size", size), + ("format", fmt), + ) + if env.get(fname) != got + ] + if drifted: + print(f"FAIL {name}: payload_envelope field(s) disagree with the envelope bytes: {', '.join(drifted)}") + vec_failed += 1 + else: + observed_encodings.add(actual) det = vec.get("arrow_detection") if det: off = det["ipc_magic_offset"] @@ -208,13 +230,14 @@ def verify() -> int: return 0 -def _build_default_path_vector() -> tuple[dict, str]: +def _build_default_path_vector() -> dict: """Build the default-@cache-write vector from the installed cachekit wheel. - Returns (vector_dict, envelope_encoding) where envelope_encoding is - "bin" (msgpack 0xc4/0xc5/0xc6, protocol 1.1 writers) or "int-array" - (legacy array-of-ints writers). Every frame is round-tripped through the - real cachekit-py deserialization path before being returned. + The vector's name follows the envelope encoding the wheel emits: + "bin" (msgpack 0xc4/0xc5/0xc6, protocol 1.1 writers) builds the `_bin` + twin, "int-array" (legacy array-of-ints writers) builds the legacy + original. Every frame is round-tripped through the real cachekit-py + deserialization path before being returned. """ import msgpack # third-party; generation only @@ -231,57 +254,192 @@ def _build_default_path_vector() -> tuple[dict, str]: inner, fmt = ByteStorage("msgpack").retrieve(payload) inner = bytes(inner) _require(msgpack.unpackb(inner) == value and fmt == "msgpack", "ByteStorage.retrieve round-trip mismatch") - env = msgpack.unpackb(payload) # positional fixarray(4) - _require(isinstance(env, list) and len(env) == 4, "envelope is not a 4-element msgpack array") - # msgpack-python decodes bin as bytes and array-of-ints as list — the - # observed type IS the wire encoding of compressed_data. - encoding = "bin" if isinstance(env[0], bytes) else "int-array" - # Protocol 1.1 scopes the bin flip to compressed_data ONLY: checksum - # [u8;8] must stay an array of integers in every encoding. - _require(isinstance(env[1], list), "checksum drifted to msgpack bin (excluded from the protocol 1.1 flip)") - # `format` is excluded alongside `checksum`, and it feeds AAD construction — - # a drift to bin here must be a named invariant, not a json.dumps TypeError. - _require(isinstance(env[3], str), "format drifted off msgpack str (excluded from the protocol 1.1 flip)") + # Shared codec (wire-format-reference.py). decode_envelope enforces the + # protocol 1.1 flip exclusions — checksum must stay an array of 8 integers + # and format a fixstr — so a wheel drifting either field fails here. + data, checksum, original_size, env_fmt, encoding = _wire.decode_envelope(payload) + _require(env_fmt == fmt, "envelope format field disagrees with ByteStorage.retrieve") + # Codec fidelity against the real wheel: the shared encoder must reproduce + # the wheel's envelope byte-identically, or codec and wheel have drifted. + _require( + _wire.encode_envelope(data, checksum, original_size, env_fmt, encoding=encoding) == payload, + "shared envelope codec does not reproduce the wheel's envelope bytes", + ) default_header, _ = parse_frame(frame) _require(default_header["m"] == meta and default_header["s"] == ser_name, "frame header disagrees with unwrap metadata") - vector = { - "name": "default_saas_write_msgpack_bytestorage", - "description": ( + if encoding == "bin": + name = "default_saas_write_msgpack_bytestorage_bin" + description = ( + "Protocol 1.1 twin of default_saas_write_msgpack_bytestorage: same value, same " + "default @cache write path, but the ByteStorage envelope's compressed_data is " + "msgpack bin (serde_bytes) instead of an array of integers. Readers MUST accept " + "both encodings; the legacy encoding stays pinned by the legacy vector's bytes." + ) + encoding_note = ( + "rmp_serde positional fixarray(4); compressed_data encodes as msgpack bin " + "(serde_bytes, protocol 1.1); checksum [u8;8] stays an array of integers" + ) + else: + name = "default_saas_write_msgpack_bytestorage" + description = ( "Exact stored bytes for a default @cache write (StandardSerializer, integrity on): " "CK v3 frame wrapping the ByteStorage envelope of the MessagePack-encoded value. " "This is what any backend — including the SaaS — receives from cachekit-py in auto mode." - ), + ) + encoding_note = ( + "rmp_serde::to_vec positional fixarray(4); Vec/[u8;8] fields encode as msgpack arrays of integers" + ) + return { + "name": name, + "description": description, "value_json": value, "frame_hex": frame.hex(), "expected_header": default_header, "expected_payload_hex": payload.hex(), "payload_envelope": { - "encoding": ( - "rmp_serde positional fixarray(4); compressed_data encodes as msgpack bin " - "(serde_bytes, protocol 1.1); checksum [u8;8] stays an array of integers" - if encoding == "bin" - else "rmp_serde::to_vec positional fixarray(4); Vec/[u8;8] fields encode as msgpack arrays of integers" - ), + "encoding": encoding_note, "envelope_encoding": encoding, - "compressed_data_hex": bytes(env[0]).hex(), - "checksum_hex": bytes(env[1]).hex(), - "original_size": env[2], - "format": env[3], + "compressed_data_hex": data.hex(), + "checksum_hex": checksum.hex(), + "original_size": original_size, + "format": env_fmt, "inner_msgpack_hex": inner.hex(), }, } - return vector, encoding + + +def _upsert(committed: list[dict], built: list[dict], generator_stamp: str) -> list[str]: + """Replace committed vectors the wheel rebuilt (matched by name); append new names. + + Never removes anything: a vector this run did not rebuild stays exactly as + committed, so dropping a committed vector is structurally impossible. A + rebuilt vector whose content matches the committed one (ignoring its + per-vector 'generator' provenance) keeps the committed entry byte-untouched + — a no-op `generate` leaves the fixture byte-identical. Returns the names + of the vectors actually rewritten or added. + """ + index = {v["name"]: i for i, v in enumerate(committed)} + _require(len(index) == len(committed), "committed fixture has duplicate vector names") + changed: list[str] = [] + for vec in built: + i = index.get(vec["name"]) + if i is not None and {k: v for k, v in committed[i].items() if k != "generator"} == vec: + continue + stamped = {**vec, "generator": generator_stamp} + if i is None: + index[vec["name"]] = len(committed) + committed.append(stamped) + else: + committed[i] = stamped + changed.append(vec["name"]) + return changed + + +def _require_twin_equivalence(frame_vectors: list[dict]) -> None: + """The default-path pair must differ ONLY in envelope encoding. + + The `_bin` twin's description asserts the encoding is the sole delta from + the legacy vector. Prove it rather than trusting the wheel: a wheel that + also changed the LZ4 level, msgpack key order, or the frame header would + upsert a vector that lies about what it isolates, into a fixture + downstream SDKs pin (LAB-903). + + No-op when either default-path twin is absent (partial fixture): generate() + only ever rebuilds the encoding the wheel emits, so nothing is comparable + until both exist. Completeness is gated by verify(), not here. + """ + by_name = {v["name"]: v for v in frame_vectors} + legacy = by_name.get("default_saas_write_msgpack_bytestorage") + twin = by_name.get("default_saas_write_msgpack_bytestorage_bin") + if legacy is None or twin is None: + # Partial fixture (fresh bootstrap, or a deliberately removed vector). + # verify's coverage floor pins ENCODINGS, not these names — it fails + # the fixture until both int-array and bin are observed, which today + # only this pair carries. _upsert never removes, so an established + # fixture can never regress into this branch. + print("note: default-path twin pair incomplete; equivalence proof skipped", file=sys.stderr) + return + _require(twin["value_json"] == legacy["value_json"], "twin value_json differs from the legacy vector") + # Header equality must hold at the BYTE level, not just as parsed JSON — a + # wheel that reorders or reformats the header JSON would otherwise slip a + # byte-level non-twin past a dict compare. The frame prefix is everything + # before the payload: magic, version, header length, header bytes. + legacy_prefix = legacy["frame_hex"][: len(legacy["frame_hex"]) - len(legacy["expected_payload_hex"])] + twin_prefix = twin["frame_hex"][: len(twin["frame_hex"]) - len(twin["expected_payload_hex"])] + _require(twin_prefix == legacy_prefix, "twin frame prefix (magic/version/header bytes) differs from the legacy vector") + for field in ("compressed_data_hex", "checksum_hex", "original_size", "format", "inner_msgpack_hex"): + _require( + twin["payload_envelope"][field] == legacy["payload_envelope"][field], + f"twin payload_envelope.{field} differs from the legacy vector — encoding must be the ONLY delta", + ) + + +def _build_error_vectors(raw_frame: bytes) -> list[dict]: + """Build the error vectors, each checked against the REAL implementation. + + Separate from generate() so the frame-vector flow and the error-vector flow + read independently. Every vector here is proven to be rejected by + cachekit-py before it can be written, and the interop vector is proven to + be rejected by a strict msgpack reader. + """ + import msgpack # third-party; generation only + + from cachekit.serializers.wrapper import SerializationWrapper + + built_errors = [ + { + "name": "truncated_frame", + "frame_hex": "434b03", + "error": "shorter than the 7-byte fixed prefix (magic + version + header length)", + }, + { + "name": "unsupported_frame_version", + "frame_hex": "434b04000000027b7d", + "error": "frame version 4 (only version 3 is defined)", + }, + { + "name": "header_overrun", + "frame_hex": "434b03000000ff7b7d", + "error": "declared header length (255) exceeds the bytes present in the frame", + }, + ] + for vec in built_errors: + try: + SerializationWrapper.unwrap(bytes.fromhex(vec["frame_hex"])) + except ValueError: + pass + else: # pragma: no cover - generation-time invariant + raise AssertionError(f"cachekit-py accepted error vector {vec['name']}") + try: + msgpack.unpackb(raw_frame) + except msgpack.exceptions.ExtraData: + pass # exactly the trailing-bytes rejection the spec requires + else: # pragma: no cover - generation-time invariant + raise AssertionError("strict msgpack reader accepted a CK frame as one document") + built_errors.append( + { + "name": "ck_frame_fed_to_interop_reader", + "frame_hex": raw_frame.hex(), + "error": ( + "not a single well-formed MessagePack document: 0x43 is fixint 67, so the frame is one " + "1-byte document plus trailing bytes. Interop readers MUST consume exactly one document " + "and reject trailing bytes; on failure, a 0x43 0x4B prefix SHOULD be reported as " + "'Python-SDK-internal auto-mode entry — not an interop value'" + ), + } + ) + return built_errors def generate() -> int: import msgpack # third-party; generation only - from cachekit.cache_handler import CacheSerializationHandler from cachekit.serializers.wrapper import SerializationWrapper import cachekit - vectors: list[dict] = [] + doc = _load_fixture() + built: list[dict] = [] # 1. Minimal parse vector: real SerializationWrapper.wrap over known raw bytes. raw_payload = b"hello, cachekit!" @@ -293,7 +451,7 @@ def generate() -> int: # vector pins what the frame actually contains (incl. the "v" field, which # cachekit-py's unwrap drops) rather than a hand-maintained copy. raw_header, _ = parse_frame(raw_frame) - vectors.append( + built.append( { "name": "raw_payload_frame", "description": "Minimal frame: SerializationWrapper.wrap over raw bytes. Parse-level vector.", @@ -303,24 +461,21 @@ def generate() -> int: } ) - # 2. Full default-path SaaS write: value -> msgpack -> ByteStorage envelope -> CK frame. - default_vector, encoding = _build_default_path_vector() - if encoding != "int-array": - print( - f"REFUSED: installed cachekit {cachekit.__version__} emits {encoding} envelopes; " - "'generate' cannot reproduce the legacy (array-of-ints) vectors. Use " - "'generate-bin-twin' instead. Nothing written.", - file=sys.stderr, - ) - return 1 - vectors.append(default_vector) + # 2. Full default-path SaaS write: value -> msgpack -> ByteStorage envelope + # -> CK frame. Named by the envelope encoding the wheel emits, so a + # protocol 1.1 wheel rebuilds the _bin twin and a legacy wheel rebuilds the + # legacy original — either way the other vector stays as committed. + built.append(_build_default_path_vector()) # 3. Arrow path: frame wrapping [8-byte xxHash3-64][Arrow IPC file]. - # Hard requirement for generation — writing the fixture without this vector - # would silently shrink the committed vector set. + # Optional: without pandas + pyarrow the committed vector is left untouched. try: import pandas as pd import pyarrow + except ImportError as exc: + print(f"note: pandas/pyarrow not importable ({exc}); arrow_dataframe_write left as committed", file=sys.stderr) + else: + from cachekit.cache_handler import CacheSerializationHandler arrow_handler = CacheSerializationHandler(serializer_name="arrow") df = pd.DataFrame({"id": [1, 2], "score": [1.5, 2.5]}) @@ -332,7 +487,7 @@ def generate() -> int: _require(a_payload[8:14] == b"ARROW1", "Arrow IPC magic not at documented offset") arrow_header, _ = parse_frame(arrow_frame) _require(arrow_header["m"] == a_meta and arrow_header["s"] == a_ser, "Arrow frame header disagrees with unwrap metadata") - vectors.append( + built.append( { "name": "arrow_dataframe_write", "description": ( @@ -351,149 +506,39 @@ def generate() -> int: }, } ) - except ImportError as exc: - print( - "generate requires pandas + pyarrow (the arrow_dataframe_write vector " - f"cannot be regenerated without them); nothing was written: {exc}", - file=sys.stderr, - ) - raise SystemExit(1) from exc - - # Error vectors, verified against the REAL implementation as we build them. - error_vectors = [ - { - "name": "truncated_frame", - "frame_hex": "434b03", - "error": "shorter than the 7-byte fixed prefix (magic + version + header length)", - }, - { - "name": "unsupported_frame_version", - "frame_hex": "434b04000000027b7d", - "error": "frame version 4 (only version 3 is defined)", - }, - { - "name": "header_overrun", - "frame_hex": "434b03000000ff7b7d", - "error": "declared header length (255) exceeds the bytes present in the frame", - }, - ] - for vec in error_vectors: - try: - SerializationWrapper.unwrap(bytes.fromhex(vec["frame_hex"])) - except ValueError: - pass - else: # pragma: no cover - generation-time invariant - raise AssertionError(f"cachekit-py accepted error vector {vec['name']}") - try: - msgpack.unpackb(raw_frame) - except msgpack.exceptions.ExtraData: - pass # exactly the trailing-bytes rejection the spec requires - else: # pragma: no cover - generation-time invariant - raise AssertionError("strict msgpack reader accepted a CK frame as one document") - error_vectors.append( - { - "name": "ck_frame_fed_to_interop_reader", - "frame_hex": raw_frame.hex(), - "error": ( - "not a single well-formed MessagePack document: 0x43 is fixint 67, so the frame is one " - "1-byte document plus trailing bytes. Interop readers MUST consume exactly one document " - "and reject trailing bytes; on failure, a 0x43 0x4B prefix SHOULD be reported as " - "'Python-SDK-internal auto-mode entry — not an interop value'" - ), - } - ) - - doc = { - "description": ( - "Python SDK (cachekit-py) auto-mode storage container: CK v3 frame. " - "Python-SDK-internal — other SDKs identify and reject, never decode. " - "See spec/wire-format.md 'SDK Storage Containers (auto mode)'." - ), - "frame_layout": "MAGIC 'CK' (0x43 0x4B) | VERSION u8 (0x03) | HDR_LEN u32 big-endian | HEADER (UTF-8 JSON: {s, m, v}) | PAYLOAD (raw bytes)", - "generator": f"cachekit {cachekit.__version__} (PyPI wheel; Rust core via PyO3), generated by tools/python-frame-reference.py generate", - "frame_vectors": vectors, - "error_vectors": error_vectors, - } - # `generate` rewrites the whole fixture, so it can only ever be additive by - # accident. Refuse to write if that rewrite would drop a vector that is - # already committed — the encoding sniff above catches a post-1.1 wheel, but - # a pre-1.1 wheel passes it and would silently delete the *_bin twins (and - # any other vector this tool no longer knows how to build). Both directions - # close here, on the names actually present, rather than on a wheel version. - dropped = {v["name"] for v in _load_fixture()["frame_vectors"]} - {v["name"] for v in vectors} - if dropped: - print( - f"REFUSED: regenerating would drop committed vector(s): {', '.join(sorted(dropped))}. " - "Committed vectors are the source of truth four SDKs pin — nothing written.", - file=sys.stderr, - ) - return 1 - VECTOR_PATH.write_text(json.dumps(doc, indent=2, sort_keys=False) + "\n") - print(f"wrote {VECTOR_PATH} ({len(vectors)} frame vectors, {len(error_vectors)} error vectors)") - return 0 + built_errors = _build_error_vectors(raw_frame) -def generate_bin_twin() -> int: - """Append the protocol 1.1 bin-envelope twin of the default-path vector. - - Append-only (LAB-903): every existing vector is left byte-untouched — the - legacy vectors are the legacy-read proof and cannot be regenerated by a - post-0.4.0-core wheel. The twin gets its own generator provenance so the - fixture records which wheel produced which vector. - """ - import cachekit + # Upsert by name. The top-level 'generator' (the legacy-vector provenance) + # is never rewritten; every vector this run rewrites or adds carries its + # own per-vector 'generator' recording which wheel produced it. + generator_stamp = ( + f"cachekit {cachekit.__version__} (PyPI wheel; Rust core via PyO3), " + "generated by tools/python-frame-reference.py generate" + ) + unstamped = {v["name"] for v in doc["frame_vectors"] + doc["error_vectors"] if "generator" not in v} + changed = _upsert(doc["frame_vectors"], built, generator_stamp) + changed += _upsert(doc["error_vectors"], built_errors, generator_stamp) + _require_twin_equivalence(doc["frame_vectors"]) - twin, encoding = _build_default_path_vector() - if encoding != "bin": + if not changed: print( - f"installed cachekit {cachekit.__version__} still emits {encoding} envelopes; " - "generate-bin-twin needs a wheel carrying cachekit-core >= 0.4.0. Nothing written.", - file=sys.stderr, + f"{VECTOR_PATH} already up to date ({len(built)} frame, {len(built_errors)} error " + "vectors rebuilt, all identical to committed); nothing written" ) - return 1 - - twin_name = "default_saas_write_msgpack_bytestorage_bin" - twin["name"] = twin_name - twin["description"] = ( - "Protocol 1.1 twin of default_saas_write_msgpack_bytestorage: same value, same " - "default @cache write path, but the ByteStorage envelope's compressed_data is " - "msgpack bin (serde_bytes) instead of an array of integers. Readers MUST accept " - "both encodings; the legacy encoding stays pinned by the legacy vector's bytes." - ) - twin["generator"] = ( - f"cachekit {cachekit.__version__} (PyPI wheel; Rust core via PyO3), generated by " - "tools/python-frame-reference.py generate-bin-twin" - ) - - doc = _load_fixture() - by_name = {v["name"]: v for v in doc["frame_vectors"]} - if twin_name in by_name: - # Append-only and idempotent: a no-op re-run is success, not failure. - print(f"{twin_name} already present; nothing written.", file=sys.stderr) return 0 - - # The description above asserts the ENCODING is the only difference from the - # legacy vector. Prove it, rather than trusting the wheel: a wheel that also - # changed the LZ4 level, msgpack key order, or the frame header would append a - # vector that lies about what it isolates, into a fixture downstream SDKs pin. - legacy = by_name.get("default_saas_write_msgpack_bytestorage") - _require(legacy is not None, "legacy default-path vector missing — cannot prove the twin is a twin") - assert legacy is not None # narrowing for type checkers; _require already raised - _require(twin["value_json"] == legacy["value_json"], "twin value_json differs from the legacy vector") - _require(twin["expected_header"] == legacy["expected_header"], "twin frame header differs from the legacy vector") - for field in ("compressed_data_hex", "checksum_hex", "original_size", "format", "inner_msgpack_hex"): - _require( - twin["payload_envelope"][field] == legacy["payload_envelope"][field], - f"twin payload_envelope.{field} differs from the legacy vector — encoding must be the ONLY delta", - ) - doc["frame_vectors"].append(twin) - if not doc["generator"].startswith("legacy vectors:"): + if unstamped & set(changed) and not doc["generator"].startswith("mixed provenance"): + # Rewriting a vector the top-level provenance claim covered would turn + # that claim into a lie; per-vector 'generator' becomes authoritative. doc["generator"] = ( - f"legacy vectors: {doc['generator']} (unchanged since); " - "*_bin twins carry their own per-vector 'generator' field" + "mixed provenance — vectors carrying a per-vector 'generator' field record " + f"their own; all others: {doc['generator']}" ) VECTOR_PATH.write_text(json.dumps(doc, indent=2, sort_keys=False) + "\n") - print(f"appended {twin_name} to {VECTOR_PATH} (cachekit {cachekit.__version__})") + print( + f"wrote {VECTOR_PATH} — rewrote/added: {', '.join(changed)} " + f"({len(doc['frame_vectors'])} frame, {len(doc['error_vectors'])} error vectors total)" + ) return 0 @@ -504,9 +549,7 @@ def generate_bin_twin() -> int: sys.exit(0) if mode == "generate": sys.exit(generate()) - if mode == "generate-bin-twin": - sys.exit(generate_bin_twin()) if mode == "verify": sys.exit(verify()) - print(f"unsupported mode: {mode!r}; expected 'verify', 'generate', or 'generate-bin-twin'", file=sys.stderr) + print(f"unsupported mode: {mode!r}; expected 'verify' or 'generate'", file=sys.stderr) sys.exit(2)