diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef44697e..99f9bd64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,7 @@ jobs: - 'capsule-wire/**' - 'capsule-i18n/**' - 'capsule-cli/**' + - 'capsule-e2e/**' - 'capsule-core/**' - 'capsule-sdk/**' - 'xtask/**' diff --git a/AGENTS.md b/AGENTS.md index ecdfc94a..c7d35635 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ - The public server surface is Kynos REST/OpenAPI only. Do not reintroduce Salvo, GraphQL, or gRPC. The served document is **OpenAPI 3.2**: enabling Kynos's `openapi32` feature does not by itself produce one — `capsule-server` pins it with `openapi_as(SpecVersion::V3_2)`. Never emit or commit a 3.1 or 3.0 contract. - Generate clients with Spargen from the checked-in Kynos OpenAPI contract. Do not use Progenitor. Everything that parses or serializes is generated — every body, every typed parameter, and the byte-serving endpoints. Only *orchestration over* generated calls is hand-written, and the resumable upload state machine (`S-D1`) is the whole of it; do not hand-write a second parser. -- Rawshift is the intended owner of media decoding, metadata extraction, and derivative generation, consumed through `capsule-core::media` once Rawshift stabilizes. Neither exists today: Rawshift is a pinned submodule and not a workspace dependency, and `capsule-core::media` has no body to write until it is one — so nothing in Capsule decodes media right now. Capsule imports **Chromahash 0.7.1** directly, never through Rawshift, and LQIP encode/decode lives in its own `capsule-core::lqip` module (slice `S-B14`) so one implementation serves the import pipeline, the FFI, and `capsule-wasm`. **ThumbHash is retired**: neither the `thumbhash` crate nor the npm `thumbhash` package may be reintroduced. Contract: [Thumbnails — LQIP](capsule-docs/src/content/docs/design/thumbnails.md#lqip). +- Rawshift owns media decoding, metadata extraction, and derivative generation, consumed through `capsule-core::media`, which now exists and is wired: `rawshift-image` **0.1.1 from crates.io** — a registry dependency, never the pinned submodule — behind the `media` feature that `native` implies, covering still detection, decode, EXIF orientation, metadata normalisation and the JXL thumbnail tier, with every enabled codec pure Rust (no C links, so the mobile cross-builds stay clean), and absent from the `wasm32-unknown-unknown` sealing build. Every format with no codec in the build is a typed `media::MediaError::UnsupportedFormat` or a recorded per-format deferral, never a silent gap: HEIC/AVIF/RAW/WebP decode, lossy-JXL and AVIF encode, the preview tier, and all video derivatives stay deferred behind the system library, assembler, or (for WebP) the upstream aarch64 fix each needs. Capsule imports **Chromahash 0.7.1** directly, never through Rawshift, and LQIP encode/decode lives in its own `capsule-core::lqip` module (slice `S-B14`) so one implementation serves the import pipeline, the FFI, and `capsule-wasm`. **ThumbHash is retired**: neither the `thumbhash` crate nor the npm `thumbhash` package may be reintroduced. Contract: [Thumbnails — LQIP](capsule-docs/src/content/docs/design/thumbnails.md#lqip). - Blob storage and resumable encrypted upload remain Capsule-owned behind narrow, arbitrary-backend ports. Do not add `object_store` or generic CAS/transfer crates without revisiting the security contract. - Keep authentication state and upload-session state as separate Capsule ports with PostgreSQL, `redis-rs`, and in-memory adapters. Do not introduce a generic TTL/CAS abstraction. - `legacy-review/` is non-buildable reference material. Restore code only after defining its contract and automated tests against the decisions above. diff --git a/Cargo.lock b/Cargo.lock index 575bda5e..16068ffd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,6 +98,21 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -343,7 +358,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.9", "object", "rustc-demangle", "windows-link 0.2.1", @@ -515,6 +530,27 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "1.12.1" @@ -559,6 +595,12 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "byteorder" version = "1.5.0" @@ -668,6 +710,7 @@ dependencies = [ "openmls_memory_storage 0.5.0", "openmls_traits 0.5.0", "p256", + "rawshift-image", "rusqlite", "rustix", "serde", @@ -708,6 +751,27 @@ dependencies = [ "uniffi", ] +[[package]] +name = "capsule-e2e" +version = "0.1.0" +dependencies = [ + "base64", + "capsule-cli", + "capsule-cli-migration", + "capsule-core", + "capsule-sdk", + "capsule-server", + "jiff", + "kynos", + "sea-orm", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "capsule-i18n" version = "0.1.0" @@ -1005,6 +1069,12 @@ dependencies = [ "tracing-error", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.5" @@ -1110,6 +1180,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.7" @@ -1622,6 +1701,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + [[package]] name = "ff" version = "0.13.1" @@ -1660,6 +1745,17 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + [[package]] name = "fluent-uri" version = "0.4.1" @@ -1915,6 +2011,16 @@ dependencies = [ "polyval", ] +[[package]] +name = "gif" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae047235e33e2829703574b54fdec96bfbad892062d97fed2f76022287de61b" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.32.3" @@ -2498,6 +2604,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "img-parts" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19734e3c43b2a850f5889c077056e47c874095f2d87e853c7c41214ae67375f0" +dependencies = [ + "bytes", + "crc32fast", + "miniz_oxide 0.8.9", +] + [[package]] name = "indenter" version = "0.3.4" @@ -2615,6 +2732,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-encoder" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0370574b86f7eca156b9f298392b5e69a23f8c86f3f865add60bbc2e79467a6" + [[package]] name = "js-sys" version = "0.3.77" @@ -2695,6 +2818,184 @@ dependencies = [ "zeroize", ] +[[package]] +name = "jxl-bitstream" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b480e752277e29eb4054f69546887a9b84656fe78c08f54ba5850ced98a378fe" +dependencies = [ + "tracing", +] + +[[package]] +name = "jxl-coding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd972bcd125e776f1eb241ac50e39f956095a1c2770c64736c968f8946bd9a3c" +dependencies = [ + "jxl-bitstream", + "tracing", +] + +[[package]] +name = "jxl-color" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f316b1358c1711755b3ee8e8cb5c4a1dad12e796233088a7a513440782de80b2" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-image", + "jxl-oxide-common", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "jxl-frame" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d967c6fd669c7c01060b5022d8835fa82fd46b06ffc98b549f17600a097c2b3" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-image", + "jxl-modular", + "jxl-oxide-common", + "jxl-threadpool", + "jxl-vardct", + "tracing", +] + +[[package]] +name = "jxl-grid" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01671307879a033bfa52e6e8784b941aca770b3f3a7d33830b455b6844f793fb" +dependencies = [ + "tracing", +] + +[[package]] +name = "jxl-image" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f752d62577c702a94dbbce4045caf08cb58639e8a4d56464b40ecf33ffe565" +dependencies = [ + "jxl-bitstream", + "jxl-grid", + "jxl-oxide-common", + "tracing", +] + +[[package]] +name = "jxl-jbr" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35d032bcec660647828527ff42c6f5776d2fd44b8357f9f6d9ac6dc07218e46" +dependencies = [ + "brotli-decompressor", + "jxl-bitstream", + "jxl-frame", + "jxl-grid", + "jxl-image", + "jxl-modular", + "jxl-oxide-common", + "jxl-threadpool", + "jxl-vardct", + "tracing", +] + +[[package]] +name = "jxl-modular" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2f045b24c738dd91d482be385512b512721ae08a671bd4b27bf1c47f215235" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-oxide-common", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "jxl-oxide" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36c662923f47586880211f3bc7c0d83fb3a9b410d278c7bde93450748abeef3" +dependencies = [ + "brotli-decompressor", + "jxl-bitstream", + "jxl-color", + "jxl-frame", + "jxl-grid", + "jxl-image", + "jxl-jbr", + "jxl-oxide-common", + "jxl-render", + "jxl-threadpool", + "tracing", +] + +[[package]] +name = "jxl-oxide-common" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62394c5021b3a9e7e0dbb2d639d555d019090c9946c39f6d3b09d390db4157b" +dependencies = [ + "jxl-bitstream", +] + +[[package]] +name = "jxl-render" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34386bfdb6a19b5a30cc9beb4d475d537422c31ae8c39bb69640fcce3fcaf19" +dependencies = [ + "bytemuck", + "jxl-bitstream", + "jxl-coding", + "jxl-color", + "jxl-frame", + "jxl-grid", + "jxl-image", + "jxl-modular", + "jxl-oxide-common", + "jxl-threadpool", + "jxl-vardct", + "tracing", +] + +[[package]] +name = "jxl-threadpool" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f15eb830aa77a7f21148d72e153562a26bfe570139bd4922eab1908dd499d3" +dependencies = [ + "rayon", + "rayon-core", + "tracing", +] + +[[package]] +name = "jxl-vardct" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce72a18c6d3a47172ab6c479be2bdb56f22066b5d7092663f03b4490820b4511" +dependencies = [ + "jxl-bitstream", + "jxl-coding", + "jxl-grid", + "jxl-modular", + "jxl-oxide-common", + "jxl-threadpool", + "tracing", +] + [[package]] name = "k256" version = "0.13.4" @@ -3118,6 +3419,20 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "little_exif" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21eeb58b22d31be8dc5c625004fcd4b9b385cd3c05df575f523bcca382c51122" +dependencies = [ + "brotli", + "crc", + "log", + "miniz_oxide 0.8.9", + "paste", + "quick-xml", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -3237,6 +3552,16 @@ dependencies = [ "adler2", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.1" @@ -3805,6 +4130,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pastey" version = "0.2.3" @@ -4145,6 +4476,21 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" @@ -4312,6 +4658,34 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rawshift-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d93e32ed40baedf3ad0a731690f6af5b58b6067af097c9fd3deb6e03080be9b" + +[[package]] +name = "rawshift-image" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "471a16d2c1b56ee8288ba90f675e96c7513839aadac88f440c1643b815e227c3" +dependencies = [ + "gif", + "img-parts", + "jpeg-encoder", + "jxl-oxide", + "little_exif", + "rawshift-core", + "rayon", + "thiserror 2.0.20", + "tiff", + "tracing", + "zune-core", + "zune-jpeg", + "zune-jpegxl", + "zune-png", +] + [[package]] name = "rayon" version = "1.12.0" @@ -5160,6 +5534,12 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simdutf8" version = "0.1.5" @@ -5711,6 +6091,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "time" version = "0.3.47" @@ -6660,6 +7054,12 @@ dependencies = [ "nom", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "whoami" version = "1.6.1" @@ -7388,8 +7788,57 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] + +[[package]] +name = "zune-jpegxl" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cba11ecd07cc23351500e840ae03841b7e4997923ec8e4832ee3ae199ea0b6b4" +dependencies = [ + "zune-core", +] + +[[package]] +name = "zune-png" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a321146329f7617ba0a5b26982cba45b7ce78163135aba78be25850aefcea80" +dependencies = [ + "zune-core", + "zune-inflate", +] diff --git a/Cargo.toml b/Cargo.toml index ffdbdaf4..6ba65a5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "capsule-cli/migration", "capsule-core", "capsule-core-ffi", + "capsule-e2e", "capsule-i18n", "capsule-sdk", "capsule-wasm", @@ -20,6 +21,7 @@ default-members = [ "capsule-cli/migration", "capsule-core", "capsule-core-ffi", + "capsule-e2e", "capsule-i18n", "capsule-sdk", "capsule-server", diff --git a/NOTICE b/NOTICE index 8f65dff5..5ea11538 100644 --- a/NOTICE +++ b/NOTICE @@ -96,10 +96,10 @@ their published upstream releases. Source for each is available from crates.io and from the upstream repository named in its own package metadata. base64urlsafedata, colored, hpke-rs, hpke-rs-crypto, hpke-rs-libcrux, - hpke-rs-rust-crypto, option-ext, uniffi, uniffi_bindgen, uniffi_core, - uniffi_internal_macros, uniffi_macros, uniffi_meta, uniffi_pipeline, - uniffi_udl, webauthn-attestation-ca, webauthn-rs, webauthn-rs-core, - webauthn-rs-proto + hpke-rs-rust-crypto, option-ext, rawshift-core, rawshift-image, uniffi, + uniffi_bindgen, uniffi_core, uniffi_internal_macros, uniffi_macros, + uniffi_meta, uniffi_pipeline, uniffi_udl, webauthn-attestation-ca, + webauthn-rs, webauthn-rs-core, webauthn-rs-proto -------------------------------------------------------------------------------- Conjunctive-license components diff --git a/SLICES.md b/SLICES.md index ce7d3266..05830d61 100644 --- a/SLICES.md +++ b/SLICES.md @@ -210,11 +210,11 @@ row's remainder now lives. | S-A9 | Add-id counter reseed at `Workspace` open | core-crypto | — | S | ACTIVE | done | | | S-A10 | Durable album-key persistence + library open plumbing | core-crypto | — | L | ACTIVE | done | | | S-A11 | Publish the DEK in the device directory | core-crypto | — | M | ACTIVE | done | | -| S-B1 | Thumbnail/LQIP generation | media/import | — | L | RETIRED | ready | | +| S-B1 | Thumbnail/LQIP generation | media/import | — | L | ACTIVE | done\* | lossy JXL/AVIF encode, preview tier, HEIC/RAW decode → #437; WebP → #444 | | S-B2 | Signed-path import-executor rewrite | media/import | S-B1 | L | MIXED | done\* | durable album keys → `S-A10` | | S-B3 | Streaming import (probe, `total_size`, drive mode) | media/import | S-D1, S-D4 | L | MIXED | done | | | S-B4 | Staged uploads (low-data tier ladder) | media/import | S-C1, S-C2, S-D1 | M | MIXED | done | | -| S-B5 | Video derivatives (first-frame still + H.264 preview) | media/import | S-B1 | M | RETIRED | ready | | +| S-B5 | Video derivatives (first-frame still + H.264 preview) | media/import | S-B1 | M | ACTIVE | ready | `rawshift-video` unpublished → #438 | | S-B6 | Google Takeout importer | media/import | S-B2 | M | MIXED | done\* | sidecar-enrichment write → `S-B10` | | S-B7 | iCloud export importer | media/import | S-B6 | M | MIXED | post-v1 | | | S-B8 | Immich importer | media/import | S-B6 | M | MIXED | post-v1 | | @@ -223,8 +223,8 @@ row's remainder now lives. | S-B11 | CLI `import --provider takeout` + real-archive run | media/import | S-B10 | S | ACTIVE | done\* | synthesized archive only; real export owed | | S-B18 | No CLI surface shows what the importer actually wrote | media/import | S-B10 | S | ACTIVE | ready | users cannot verify enrichment | | S-B12 | Base default-album resolution (`resolve_default_album`) | media/import | — | M | ACTIVE | done | scope-override + source-kind rows → post-v1 | -| S-B13 | Codec stubs → typed `UnsupportedFormat` (no panics) | media/import | — | M | RETIRED | ready | | -| S-B14 | LQIP on Chromahash 0.7.1 in `capsule-core::lqip` | media/import | — | M | ACTIVE | done | wasm entry point owed to the browser-`lqip` slice | +| S-B13 | Codec stubs → typed `UnsupportedFormat` (no panics) | media/import | — | M | ACTIVE | done | | +| S-B14 | LQIP on Chromahash 0.7.1 in `capsule-core::lqip` | media/import | — | M | ACTIVE | done | | | S-B15 | Importer-formed stacks exist only in the index | media/import | S-D21 | M | ACTIVE | done | rebuild guard kept as pre-`S-B15` compatibility | | S-B16 | Every import stamped by import time, not capture time | media/import | — | S | ACTIVE | done | found by the CLI round-trip test | | S-B17 | Repair capture timestamps written before `S-B16` | media/import | S-B16 | M | ACTIVE | ready | the wrong value is in *signed* bytes | @@ -302,12 +302,12 @@ row's remainder now lives. | S-D9 | capsule-sdk uniffi FFI bindings | sdk/clients | S-F1, S-D7 | M | RETIRED | ready | Swift harness → `S-P8`; Kotlin harness → owed-CI | | S-D10 | Adverse-network hardening | sdk/clients | S-D1, S-D2 | M | RETIRED | ready | | | S-D11 | Client cohort emission + devices grouping UI | sdk/clients | S-C13, S-D7 | M | MIXED | done\* | iOS reader → `S-P6`; devices screen → post-v1; device_id → `S-N3` | -| S-D12 | Recovery verification cadence + guided re-wrap | sdk/clients | S-C12 | M | MIXED | done | | +| S-D12 | Recovery verification cadence + guided re-wrap | sdk/clients | S-C12 | M | MIXED | done | `store_escrow` discards `stored_at`/`replaced` → issue #442 | | S-D13 | Culling workflow client UX | sdk/clients | — | M | ACTIVE | done | | | S-D14 | Local-gallery security gates | sdk/clients | — | S | ACTIVE | done | | | S-D15 | Exact client build identification | sdk/clients | — | S | MIXED | done | | | S-D16 | Standalone `capsule cull` command | sdk/clients | S-A10 | S | ACTIVE | done | | -| S-D17 | Typed REST client reactive 401-retry-once | sdk/clients | — | S | RETIRED | ready | | +| S-D17 | Typed REST client reactive 401-retry-once | sdk/clients | — | S | MIXED | done | | | S-D18 | `capsule push` — drive `capsule_sdk::upload` from CLI | sdk/clients | S-A10 | M | MIXED | done | | | S-D19 | Hidden-view DB projection + gate wiring | sdk/clients | — | S | ACTIVE | done | rebuild un-hides → `S-D21` | | S-D21 | Index rebuild loses gated state (two sidecar shapes) | sdk/clients | S-D19 | M | ACTIVE | done | importer stacks → `S-B15`; unsigned migration → `S-D24`; no hidden writer → `S-D25` | @@ -360,10 +360,10 @@ row's remainder now lives. | S-P6 | SE signer wiring into the app + iOS cohort reader | iOS path | S-P1 | M | ACTIVE | ready | | | S-P7 | Dev-server bring-up (task, keys, blob backend, ATS) | iOS path | — | M | MIXED | done | | | S-P8 | Swift behavioral FFI harness (flips S-D9) | iOS path | S-P1, S-P7 | M | MIXED | ready | | -| S-Q1 | Mark/complete E2E cases 2, 3, 11 | e2e | — | S | MIXED | ready | | -| S-Q2 | E2E case 6: backup → fresh-device restore | e2e | — | M | MIXED | ready | | -| S-Q3 | E2E case 7: full lifecycle chain | e2e | — | M | MIXED | ready | | -| S-Q4 | E2E case 12: cross-device enrollment | e2e | — | M | MIXED | ready | | +| S-Q1 | Mark/complete E2E cases 2, 3, 11 | e2e | — | S | MIXED | done\* | 2, 3 in `capsule-e2e`; 11 lands with #447; JXL T1 → #470 | +| S-Q2 | E2E case 6: backup → fresh-device restore | e2e | — | M | MIXED | done\* | restore reads; verify → #468; account seam → #467 | +| S-Q3 | E2E case 7: full lifecycle chain | e2e | — | M | MIXED | done | provenance rung → #464 | +| S-Q4 | E2E case 12: cross-device enrollment | e2e | — | M | MIXED | done\* | server leg; client ceremony → #471, #467, #405 | | S-Q5 | Live-browser smokes (gRPC-web, share, drop) | e2e | S-P7 | M | MIXED | ready | | | S-Q6 | E2E case 10: model regen after version bump | e2e | — | M | ACTIVE | done | the case was untestable, not untested | | S-U1 | Domain + ports + mock seam | apple-ui | — | L | ACTIVE | done | | @@ -695,10 +695,43 @@ workspace at all**, so every still import is a `DeferredNoCodec` until Rawshift - **Tier:** Unit + Smoke. **Blocks:** S-B2, S-B5. - **Landed in retired code, and retired 2026-09-01 by `S-C59`:** generation shipped over injected per-platform encoder seams and was green in this workspace, but it lived in - `capsule_core::media` — review material. It is now in `legacy-review/media-pipeline/`, together - with `lifecycle/derivatives.rs`, its only caller. **Re-scoped:** re-land on the Rawshift-backed - pipeline. The signed `DerivativeManifest` chain and the sidecar `lqip` field are `ACTIVE` and - stay — the field stays here, its producer moves to `S-B14`. + `capsule_core::media` — review material. It went to `legacy-review/media-pipeline/`, together + with `lifecycle/derivatives.rs`, its only caller. +- **Re-landed 2026-09-02 on the Rawshift-backed pipeline (issue #410).** `capsule-core::media` + exists again, over **`rawshift-image` 0.1.1 from crates.io** (a registry dependency, not the + pinned submodule) behind a `media` feature that `native` implies and the wasm32 sealing build + excludes. The injected `StillEncoder` seam is **gone**: the per-platform encoder was there + because core linked no codec, and it no longer needs to. What ships: + - `media::{detect,decode,resize,derivative,error}` as private submodules behind one barrel — + the closed `StillFormat` set with a Capsule-owned magic-byte table, the `Decoder` seam with + a pre-decode 128 Mpx budget and an unwind boundary, a deterministic integer area-average + downscale (the crate has no resize, and a derivative's bytes are signed), the closed + `DerivativeFormat` set with the `original` sentinel, and `MediaError`. + - the **thumbnail tier** at 256 px as **JXL** — the table's committed *master* format — signed + and hash-chained through the same two-signature `DerivativeCore::sign` path assets use, and + persisted at the layout the upload bundle reader already reads. The declared q=50 is advisory: + the pure-Rust `zune-jpegxl` backend is lossless, which a test asserts rather than hides. + - **WebP was the first choice and CI refuted it.** `image/webp` is in the format table and + `libwebp` has exactly the q=50 knob, but `rawshift-image`'s WebP module passes `*const i8` + where `libwebp-sys` 0.14.4 declares `*const c_char` — `u8` on aarch64 — so the feature is an + E0308 on every mobile target, in both directions (the module compiles under + `any(webp-decode, webp-encode)`). WebP is therefore a recognised-but-undecodable format here; + the upstream fix is filed as #444. + - **Detection is Capsule's, not the crate's.** `rawshift-image`'s own `detect_standard_format` + gates its HEIC arm on `heic-decode`, so delegating would make the typed refusal for a format + depend on whether it can be decoded — a HEIC would arrive as "not a still" instead of "a + still whose derivatives are backfillable". A test pins agreement between the two tables for + every format both define unconditionally. + - **Every encode passes `MetadataEmbedOptions::none()`.** `rawshift-core`'s default is `all()`, + so a default-configured encode copies the source's EXIF — GPS included — into the thumbnail. + A test demonstrates the leak with the crate's own default and then asserts Capsule's + derivative carries no `EXIF`/`XMP`/`ICCP` chunk and none of the source's GPS rationals. +- **Owed → #437 and #444.** A *lossy* JXL master, the AVIF delivery variant, the preview tier and + HEIC/RAW decode (#437); WebP in both directions (#444). None is blocked on a design question: a + lossy JXL needs C libjxl, AVIF encode needs `nasm` on every x86_64 build host, HEIC/AVIF decode + need system libheif/libdav1d, and WebP needs one upstream cast widened. All are visible today as + typed `MediaError::UnsupportedFormat` or as per-`(tier, format)` deferrals counted by + `ImportExecutionSummary::deferred_format_count()`, never as silent absence. ### S-B2 — Signed-path import-executor rewrite @@ -761,8 +794,22 @@ workspace at all**, so every still import is a `DeferredNoCodec` until Rawshift - **Done when:** a fixture video yields both tiers with signed manifests; the closed-format rejection covers the video rows of the tier table. - **Tier:** Unit + Smoke. -- **Landed in retired code:** ships today behind the injected encoder seam; the transcode - half is `capsule-core::media` and re-scopes onto the Rawshift-backed pipeline. +- **Landed in retired code, and re-scoped 2026-09-02 (issue #410 → #438).** It shipped behind + the injected encoder seam; that seam is gone with `S-B1`'s re-land, so this slice now sits on + the live `capsule-core::media` pipeline and is `ACTIVE` — and still unimplemented. +- **Why it is not just another format.** `rawshift-video` is **not published on crates.io** (the + `rawshift` facade's `video` feature points at an unreleased crate) and the transcode toolchain + — demux, video decode, H.264/AAC encode — touches nothing the still path does. That is the + split from `S-B1`, restated: a distinct dependency decision with its own licence surface, which + `design/licensing.md` already names as the most likely route by which copyleft enters Capsule. +- **What happens today:** a video's bytes sniff to no `media::StillFormat`, so every video import + reports `DerivativeStatus::NotAKnownStill` and carries no thumbnail, preview or LQIP. The + original is still imported signed, encrypted and `verify_asset`-accepting, so this is a + cosmetic gap, not data loss. `content_type` stays extension-derived for video, because + detection has no video half yet. +- **`capsule-core::media::video` is the one remaining `planned-modules.txt` row** for this lane + (`#410` narrowed the `capsule-core::media` row to it), which is what keeps `check-docs-truth` + honest about `capsule-core::media::video::derivative` in `design/licensing.md`. ### S-B6 — Google Takeout importer @@ -937,10 +984,37 @@ workspace at all**, so every still import is a `DeferredNoCodec` until Rawshift pins that HEIC and RAW-only originals import and self-verify **without** derivatives, and a planner test guards that undecodable stills are never skipped at plan time; `mise run check-rust` green. **Tier:** Unit. -- **Landed in retired code:** shipped and green on this branch, but the whole surface is - `capsule-core::media`. **Re-scoped:** the uninhabited-stub discipline and the - `is_decodable`/`from_extension` coverage table are the contract the Rawshift-backed - rebuild inherits; `DerivativeStatus` on `ImportOutcome` is `ACTIVE` and stays. +- **Landed in retired code:** shipped and green on this branch, but the whole surface was + `capsule-core::media`. The uninhabited-stub discipline and the `is_decodable`/`from_extension` + coverage table were the contract the Rawshift-backed rebuild had to inherit. +- **Re-landed 2026-09-02 (issue #410), and the shape it inherited changed for the better.** + There are no stubs at all now — uninhabited or otherwise — because there is nothing to stub: + `rawshift-image` either has a codec or it does not, and the coverage table + (`StillFormat::is_decodable` over `SUPPORTED_STILL_FORMATS`) is a *gate checked before any + decoder runs* rather than a property of a type nobody can construct. `rg 'unimplemented!\(|todo!\(' + capsule-core/src/media` is empty by construction. + - `MediaError::UnsupportedFormat { format, op }` carries the `FormatOp` — a build can decode a + format it cannot encode, and the message has to say which half is missing. + - **The two-reason distinction is observable again**, and now rests on the bytes rather than the + extension: a HEIC is `DeferredNoCodec` (recognised, no codec here, backfillable) while a + `.jpg` that is not a JPEG is `DecodeFailed` (a format we do decode, failing on these bytes). + `S-C59` had collapsed both into deferrals and the executor test said so; it asserts the + distinction again. + - **A third reason joined them, per format rather than per asset.** + `StillDerivatives::deferred` records each `(tier, format)` pair with no encoder, and + `ImportExecutionSummary::deferred_format_count()` sums them. A decoded JPEG is `Decoded` with + one generated thumbnail and two deferred formats — the number that falls to zero as #437 + lands, rather than a gap only a doc mentions. + - **No panic can reach an import.** Untrusted bytes go through a pre-decode pixel budget + (`MAX_DECODE_PIXELS`, **128 Mpx** — the bomb is inside the decoder, which works in RGB `u16`, + and the honest peak at that ceiling is ~2.5 GB across the decoder's samples, the + alpha-dropping realloc, the RGBA8 copy and the widening back for the encode. `native` implies + `media`, so that peak lands on a phone as an OOM kill rather than an error, which is why the + ceiling sits ~25% above a 102 Mpx medium-format frame rather than as high as an allocation + bomb would require) and + a `catch_unwind` boundary that maps a third-party decoder's panic to `DecodeFailed`. Both are + tested, the panic case through an injected `Decoder`. +- **Originals always import**, unchanged: codec coverage gates *derivatives*, never *admission*. ### S-B14 — LQIP on Chromahash 0.7.1, in its own module @@ -1001,8 +1075,25 @@ workspace at all**, so every still import is a `DeferredNoCodec` until Rawshift 21 bytes — `COMPACT_TIER`'s length — which is the concrete proof that byte length cannot discriminate a stale payload. Both are rejected by `from_bytes` and render as the solid dominant-colour fill, never noise. -- **Owed:** no `wasm_bindgen` export exists. wasm links and compiles the identical encoder, but the - browser has no decrypted `lqip` to decode yet, so the entry point belongs to that slice. +- **The producer and the exports landed 2026-09-02 (issue #410), and the module has callers on + all three surfaces.** `lqip` had been fully tested and entirely unreachable: nothing decoded, so + nothing encoded a placeholder. + - **Producer:** `Workspace::prepare_still` encodes from the **full-resolution, + orientation-applied** frame — not from the thumbnail, because chromahash band-limits on the + read side via `decode_capped` and pre-resizing would silently cap fidelity the format can + carry. The signed sidecar now carries a real 32-byte payload at `format_version` 1. + - **Browser:** `capsule-wasm`'s `decodeLqip` (`WasmLqipImage`) returns packed RGBA the viewer + hands to `putImageData`. The JS boundary is a `map`/`ok_or_else` over a pure helper, because + `JsError` cannot be constructed off-wasm and a host test reaching the error arm would abort + the test binary rather than fail an assertion. + - **Native:** `capsule-core-ffi`'s `render_lqip` → `LqipPlaceholder`. A free function rather + than a `Catalog` method: the `assets` table's `chromahash`/`dominant_color` columns are NULL + and must stay so until `library::rebuild` projects them identically, or a rebuilt index would + disagree with a freshly written one — so it takes the record the caller already holds from the + decrypted sidecar rather than pretending the index has it. + - The cross-surface criterion is asserted rather than assumed: both exports are checked + byte-identical to `Lqip::decode_capped` for a real payload, and both paint the + `dominant_color` fill for an unknown version or a payload `from_bytes` rejects. ### S-B15 — Importer-formed stacks exist only in the index @@ -4129,6 +4220,24 @@ Kynos server, which cannot be written until `S-C53` gives the server a way to cr - **Tier:** Unit + Smoke. - **Landed:** the cadence scheduler, the verifier, and the re-wrap are `ACTIVE` core; only the escrow store/replace calls re-scope. +- **Gap** (found 2026-09-01, issue #408): **the networked half never worked against a real + server.** `capsule-sdk/src/recovery/mod.rs` built `{api_root}/backup/escrow` from a `const` + — the Salvo document's path — and sent it with hand-written `reqwest` calls, while the Kynos + contract serves `GET`/`PUT /v1/auth/escrow`. So enroll, the stale-cache refresh and the + guided re-wrap's escrow replace all failed on a live server while this row read `done`. It + survived `S-D28`'s re-source because a route in a string constant is checked by no gate and + the module's own mock answered whichever path it was handed. +- **Closed:** the two operations are `application/octet-stream` in each direction, which + `spargen` lowers, so both were already generated and neither was narrowed in `build.rs`. + `RecoveryClient` now holds an `AuthenticatedClient` and orchestrates + `fetch_escrow`/`store_escrow`, so the path is a function of the committed document and + cannot drift again. Both in-repo mocks route on `/v1/auth/escrow` and answer `501` off it, + and `capsule-server/tests/sdk_client.rs` asserts the route against the real router from both + ends — what the SDK stored is read back at `/v1/auth/escrow`, and a rotation seeded there is + what the SDK fetches next. +- **Owed:** `store_escrow` logs `stored_at`/`replaced` instead of returning them, so the + stale-cache rule still refreshes on a failed compare rather than on a known-stale timestamp + and `guided_rewrap` cannot tell a first enrollment from a rotation → issue #442. ### S-D13 — Culling workflow client UX @@ -4196,8 +4305,31 @@ Kynos server, which cannot be written until `S-C53` gives the server a way to cr - **Done when:** a mocked-clock race test passes (expired-at-server, valid-at-client → one refresh, one retry, no loop). **Tier:** Unit. - **Note:** the layer sits above the generated client, so write it once and it survives - the schema re-source; it is `RETIRED` only because the client under it is regenerated - from Kynos. + the schema re-source. That is why the row is `MIXED` rather than `RETIRED`: the client + underneath is regenerated from Kynos, but the layer itself is live code in this workspace + and nothing about it re-scopes. +- **Landed** (2026-09-01, issue #408): `capsule_sdk::client::RefreshOn401`, an + `rest::HttpBackend` wrapping `ReqwestBackend`, installed by `AuthenticatedClient` through + `Client::with_backend`. On a `401` it refreshes once through a new `pub(crate) + Session::refresh_rejected` — `ensure_refreshed(Rejected(stale))`, so the existing + single-flight gate coalesces on the exact token the server refused — and replays the request + once. No generated code is touched and every generated operation is covered at once. + - **Not spargen's `Middleware`:** `Next::run` takes `self` by value and `Next` is neither + `Clone` nor constructible outside the generated runtime, so a middleware cannot send twice. + `RetryBackend` is the precedent followed instead, including its rule that a request whose + `try_clone()` is `None` (a one-shot streaming body) is executed once and never replayed. + - **Exactly once by construction**, not by a loop counter. A refresh that itself fails + surfaces the *server's* `401` rather than a synthesized transport error, so the typed + `Status401` mapping still fires and the caller reads the `error.*` code — which matters + because an unreadable revocation ledger is also rendered as `401`. + - **Proven** by four unit cases in `capsule-sdk/src/client.rs` and, over a socket against + the real router, by `a_token_the_server_stopped_honouring_is_refreshed_and_the_call_replayed` + in `capsule-server/tests/sdk_client.rs`: the server validates `exp` against its injected + clock, so advancing the fixture past `ACCESS_TOKEN_TTL` revokes the access token for real + while the refresh token lives. + - **`capsule_sdk::sync` keeps its own per-call `401` loop**, deliberately: it builds its own + `rest::Client`, supports a static-token mode with no session to refresh, and interleaves + the `401` path with the shared retry engine's transient class. ### S-D18 — `capsule push` @@ -5068,11 +5200,18 @@ is why Lane Q shipped with no `Contract:` line at all), and each case's wording normative statement of what the slice must prove. Cases are numbered so code can name the case it covers (`rg "E2E case N"`). -Current registry state: live = 1, 4 (upgraded by `S-E5`), 9, 10 (`S-Q6`); in-process shape -= 5, 8 (server half = `S-C24`), 13; this lane closes the rest. Every case with a server or -SDK leg is **suspended for the duration of the Kynos rebuild** — the module map says so — -so these slices are written to be re-runnable against the replacement rather than pinned to -the current transport. +Current registry state (the module map's status table is the record): landed = 1, 2, 3, 6, +7, 9, 10 (`S-Q6`) and the server legs of 8, 12, 13, all in the `capsule-e2e` crate against the +real composition root over the real SDK and a real library — no container, no env gate; 11 +lands with #447; in-process shape = 5 and the ceremony half of 8; 4 has no route (federation +is post-v1, #406). The earlier note that every server-side case was suspended for the Kynos +rebuild is stale: the rebuilt server is what these cases run against. What still blocks a +case's full wording is a seam in the tree rather than a transport, each filed: the SDK's push +ladder omits the provenance rung (#464), `sync_apply` decodes the feed's record bytes as a +manifest (#465), the SDK's directory publish omits the identity-key header (#466), a +`Workspace` cannot open as a server account (#467), the backup artifact carries no album +authority (#468), a library's adopt registers nothing to publish (#469), the upload policy +refuses `image/jxl` (#470), and there is no cross-sign or safety-code seam (#471). ### S-Q1 — Mark/complete E2E cases 2, 3, 11 @@ -5082,6 +5221,14 @@ the current transport. round trip ≈ case 3, `S-C1` crash-injection ≈ case 11) with explicit `E2E case N` markers, fill whatever the audit finds missing to each case's Module-Map wording. - **Done when:** `rg "E2E case (2|3|11)"` hits a passing named test each. **Tier:** Smoke. +- **Landed** (2026-09-05, #409): case 2 is `capsule-e2e/tests/case_02_import_upload_finalize.rs` + (a real library import → the SDK ladder plus the provenance rung → every blob byte-equal at + its content address → storage-verify durable → on the feed); case 3's client half is + `capsule-e2e/tests/case_03_sync_pickup.rs` beside the server half in + `capsule-server/tests/sync.rs`; case 11 is `#447`'s named test in + `capsule-server/tests/upload.rs` (an in-memory fault decorator on the index, not + crash-injection). **Owed:** the JXL thumbnail's T1 upload → #470 (case 2 runs on the 8×8 + still, whose T1 is the byte-free sentinel). ### S-Q2 — E2E case 6: backup → fresh-device restore @@ -5090,6 +5237,12 @@ the current transport. - **Deliverable:** the full chain: backup artifact + server escrow fetch → restore on a fresh workspace (new process, no prior state) → assets decrypt + verify. - **Done when:** the named test passes against testcontainers. **Tier:** Smoke. +- **Landed** (2026-09-05, #409): `capsule-e2e/tests/case_06_backup_restore.rs` — escrow + through the SDK's `RecoveryClient` over the real route, the recovered master key proved to + be the account's by deriving its default album id, the backup restored into a fresh library + that reads the asset byte for byte and walks its chain. No container: the composition root + runs in-process. **Owed:** opening the fresh library *as* the recovered account → #467; + `verify` on the restored asset (the artifact carries no album authority) → #468. ### S-Q3 — E2E case 7: full lifecycle chain @@ -5099,6 +5252,13 @@ the current transport. client + server (composing `S-C16`'s op path with `S-C11`'s GC), asserting feed order and byte deletion honoring grace. - **Done when:** the named test passes. **Tier:** Smoke. +- **Landed** (2026-09-05, #409): `capsule-e2e/tests/case_07_lifecycle.rs` — caption, trash + (30-day floor), restore, re-delete and a zero-day trash, each authored by the real library + and posted as a lifecycle op through the generated client, watched by an incremental feed + reader; `gc::purge_expired` on the operator worker retains the 30-day tombstone and purges + the due one, whose original then serves `Gone`. The chain agrees end to end because the + harness uploads the provenance rung the SDK ladder omits → #464 (and `sync_apply`'s decode + of those bytes → #465). ### S-Q4 — E2E case 12: cross-device enrollment @@ -5107,6 +5267,11 @@ the current transport. - **Deliverable:** the server + CLI halves of the cross-device add (code issue/redeem, relay channel, directory update, second device syncs) — the iOS UI half is post-v1. - **Done when:** the named two-client test passes against testcontainers. **Tier:** Smoke. +- **Landed, server leg** (2026-09-05, #409): `capsule-e2e/tests/case_12_enrollment.rs` — + fresh local auth, code issue, redeem, relay and drain in both directions (each payload + delivered once, mailboxes never cross), initiator close, and the MITM abort at the wire. + **Owed:** the client ceremony — B's keys, the safety code, A's cross-sign → #471 and #467; + the MLS join → `S-C51` (#405). ### S-Q5 — Live-browser smokes diff --git a/capsule-cli/tests/import_round_trip.rs b/capsule-cli/tests/import_round_trip.rs index b63b155c..968678bd 100644 --- a/capsule-cli/tests/import_round_trip.rs +++ b/capsule-cli/tests/import_round_trip.rs @@ -16,13 +16,18 @@ //! by [`capsule_list_reports_the_sync_feed_not_the_library`] rather than papered over. //! //! **The fixture image.** A synthesized 8×8 baseline JPEG (see [`synthetic_jpeg`]) carrying a -//! real EXIF APP1 segment — no committed binary. The CLI links `capsule-core` **without** the -//! `media` feature, so nothing on this path decodes pixels (every still reports -//! `DerivativeStatus::DeferredNoCodec`, slice `S-B13`); the decode the importer genuinely -//! performs is EXIF, through `extract_exif`. The assertions therefore land on values that can -//! only have come from parsing the segment: the sidecar's 8×8 dimensions and its GPS fix. The -//! bytes are a real, decodable JPEG all the same, so the fixture stays honest if a build that -//! *does* carry a codec ever runs this path. +//! real EXIF APP1 segment — no committed binary. The CLI links `capsule-core` with default +//! features, and `native` implies `media`, so this path **does** decode pixels: the import +//! reports `DerivativeStatus::Decoded`, writes a chromahash `lqip` into the signed sidecar, and +//! signs a thumbnail-tier derivative (slices `S-B1`, `S-B13`, `S-B14`). At 8×8 the source is +//! well inside the 256 px thumbnail cap, so that derivative is the signed `format = "original"` +//! sentinel rather than a re-encode — which is exactly the contract's redundant-derivative rule +//! and worth pinning end to end. +//! +//! The EXIF assertions are still the load-bearing ones, because neither the GPS fix nor the +//! capture time exists anywhere but inside the APP1 segment `synthetic_jpeg` wrote. The 8×8 +//! dimensions now come from decoded pixels *and* agree with the EXIF tags, which is the +//! stronger statement: the two independent readings of the fixture match. //! //! **Argon2id.** `capsule library init` does not create the account — the first `capsule import` //! does, at `DeviceTier::Normal` (256 MiB, t=3), which costs ~5 s per unlock in a debug build and @@ -319,6 +324,30 @@ fn an_import_is_reconstructed_by_a_later_process_from_disk_alone() { "the stored original must be the bytes that were imported" ); + // ── The signed derivatives, in `media/{YYYY}/{YYYY-MM}/derivatives/`. ── + // + // The fixture is 8×8, well inside the thumbnail tier's 256 px cap, so the tier is satisfied + // by the signed `format = "original"` sentinel — the contract's redundant-derivative rule. + // A sentinel *references* the original, so what lands is its signed manifest and no bytes: + // copying them would put this fixture's EXIF, GPS fix included, into a derivative blob. + let derivatives = bucket.join("derivatives"); + let bundle = derivatives.join(format!("{simple}.derivatives.cbor")); + assert!( + bundle.is_file(), + "a signed derivative-manifest bundle must exist in {}", + derivatives.display() + ); + let derivative_files: Vec = std::fs::read_dir(&derivatives) + .expect("read the derivatives directory") + .flatten() + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!( + derivative_files, + vec![format!("{simple}.derivatives.cbor")], + "the sentinel writes its manifest and no derivative bytes" + ); + // ── The signed sidecar, decoded from disk. ── let bytes = std::fs::read(bucket.join(format!("{simple}.cbor"))).expect("read the sidecar"); let sidecar = @@ -340,8 +369,21 @@ fn an_import_is_reconstructed_by_a_later_process_from_disk_alone() { let dimensions = sidecar .dimensions .as_ref() - .expect("dimensions come from the EXIF PixelXDimension/PixelYDimension tags"); + .expect("dimensions come from the decoded pixels, and agree with the EXIF tags"); assert_eq!((dimensions.width, dimensions.height), (8, 8)); + + // The LQIP producer ran on the decoded pixels: a 32-byte chromahash payload at + // `LQIP_FORMAT_V1`, inside the *signed* sidecar (slice `S-B14`). + let lqip = sidecar + .lqip + .as_ref() + .expect("a decodable still carries a chromahash placeholder"); + assert_eq!( + lqip.chromahash.len(), + 32, + "chromahash's DEFAULT_TIER is exactly 32 bytes" + ); + assert_eq!(lqip.format_version, 1, "the sidecar LQIP format version"); let gps = sidecar.gps.as_ref().expect("the EXIF GPS fix"); assert!( (gps.lat - EXIF_LAT).abs() < 1e-6 && (gps.lon - EXIF_LON).abs() < 1e-6, diff --git a/capsule-core-ffi/src/catalog.rs b/capsule-core-ffi/src/catalog.rs index 82549c5c..b1e7051f 100644 --- a/capsule-core-ffi/src/catalog.rs +++ b/capsule-core-ffi/src/catalog.rs @@ -322,6 +322,64 @@ impl Catalog { } } +// ── LQIP placeholder rendering (slice S-B14) ──────────────────────────────── + +/// A decoded LQIP placeholder handed to the native clients: packed RGBA8, ready for a +/// `CGImage` / `Bitmap`. +/// +/// A record rather than raw bytes because the caller cannot know the dimensions in advance: +/// `decode_capped` returns the largest frame that fits *inside* the requested box while +/// preserving the source aspect ratio. +#[derive(Debug, Clone, PartialEq, Eq, uniffi::Record)] +pub struct LqipPlaceholder { + /// Frame width in pixels — at most the requested `max_width`. + pub width: u32, + /// Frame height in pixels — at most the requested `max_height`. + pub height: u32, + /// Packed RGBA8 samples, `width * height * 4` bytes long. + pub rgba: Vec, +} + +/// Render the three fields of a sidecar `lqip` record to a paintable placeholder, band-limited +/// to the box being painted. +/// +/// The mirror of `capsule-wasm`'s `decodeLqip`, over the same +/// [`capsule_core::lqip::render`] the import pipeline encodes against — one implementation, so +/// a photo's placeholder does not depend on which client is painting it (slice `S-B14`). +/// +/// **A free function rather than a [`Catalog`] method, deliberately.** The `assets` table has +/// `chromahash` and `dominant_color` columns, and they are NULL: the signed sidecar is the +/// placeholder's home, and projecting it onto the index would have to be done identically by +/// `capsule_core::library::rebuild` or a rebuilt index would disagree with a freshly written +/// one. Until both sides move together, an accessor keyed on an asset id could only ever +/// return nothing — so this takes the record the caller already holds from the decrypted +/// sidecar instead of pretending the index has it. +/// +/// **Infallible.** An unrecognised `format_version`, a payload the parser rejects, or a +/// `dominant_color` that is not three bytes all yield the 1x1 solid fill: a reader must never +/// misrender a placeholder, and a gallery must never fail to draw a cell over one. +#[uniffi::export] +#[must_use] +pub fn render_lqip( + format_version: u16, + chromahash: Vec, + dominant_color: Vec, + max_width: u32, + max_height: u32, +) -> LqipPlaceholder { + // A malformed fill is the one input `capsule_core::lqip::render` cannot take, and unlike the + // wasm boundary there is nothing useful to throw across the FFI for it: black is the + // conventional empty-cell fill and is what a caller would paint anyway. + let fill: [u8; 3] = dominant_color.try_into().unwrap_or([0, 0, 0]); + let image = + capsule_core::lqip::render(format_version, &chromahash, fill, max_width, max_height); + LqipPlaceholder { + width: image.width, + height: image.height, + rgba: image.rgba, + } +} + #[cfg(test)] mod tests { use std::sync::atomic::{AtomicU32, Ordering}; @@ -625,6 +683,58 @@ mod tests { assert!(cat.find_by_uuid("hidden".to_string()).unwrap().is_some()); } + // ── LQIP placeholder rendering ────────────────────────────────────────── + + /// The native surface decodes the same bytes the import pipeline encoded, to the same + /// pixels `capsule_core::lqip` produces — the `S-B14` cross-surface criterion at the FFI + /// boundary. + #[test] + fn render_lqip_matches_the_core_decoder() { + use capsule_core::lqip::{Gamut, LQIP_FORMAT_V1, Lqip}; + + let (w, h) = (120u32, 90u32); + let mut rgba = Vec::with_capacity((w * h * 4) as usize); + for y in 0..h { + for x in 0..w { + rgba.extend_from_slice(&[(x * 2) as u8, (y * 2) as u8, 128, 255]); + } + } + let lqip = Lqip::encode(w, h, &rgba, Gamut::Srgb).expect("encode"); + assert_eq!(lqip.as_bytes().len(), 32, "the committed tier is 32 bytes"); + + let expected = lqip.decode_capped(48, 48); + let got = render_lqip( + LQIP_FORMAT_V1, + lqip.as_bytes().to_vec(), + lqip.dominant_color().to_vec(), + 48, + 48, + ); + assert_eq!((got.width, got.height), (expected.width, expected.height)); + assert_eq!(got.rgba, expected.rgba, "byte-identical to the core"); + assert_eq!(got.rgba.len() as u32, got.width * got.height * 4); + } + + /// Every malformed input paints the fallback rather than failing: an unknown version, a + /// corrupt payload, and a `dominant_color` that is not three bytes. + #[test] + fn render_lqip_never_fails_on_a_malformed_record() { + use capsule_core::lqip::LQIP_FORMAT_V1; + + let fill = vec![9u8, 8, 7]; + let unknown = render_lqip(LQIP_FORMAT_V1 + 42, vec![1, 2, 3], fill.clone(), 32, 32); + assert_eq!((unknown.width, unknown.height), (1, 1)); + assert_eq!(unknown.rgba, vec![9, 8, 7, 255]); + + let corrupt = render_lqip(LQIP_FORMAT_V1, vec![0xDE, 0xAD], fill, 32, 32); + assert_eq!(corrupt.rgba, vec![9, 8, 7, 255]); + + // No usable fill: black, the conventional empty-cell colour. + let no_fill = render_lqip(LQIP_FORMAT_V1, vec![0xDE, 0xAD], vec![1, 2], 32, 32); + assert_eq!((no_fill.width, no_fill.height), (1, 1)); + assert_eq!(no_fill.rgba, vec![0, 0, 0, 255]); + } + /// The retention sweep is deliberately ungated (it runs unattended) — pinned here so /// a future change to that decision is a deliberate one, not an accident. #[test] diff --git a/capsule-core-ffi/src/lib.rs b/capsule-core-ffi/src/lib.rs index 544b2a2c..fc7e803d 100644 --- a/capsule-core-ffi/src/lib.rs +++ b/capsule-core-ffi/src/lib.rs @@ -19,6 +19,9 @@ //! Rust ↔ Swift contract: //! //! - [`Catalog`] — a thread-safe handle over the SQLite catalog. +//! - [`render_lqip`] / [`LqipPlaceholder`] — the sidecar `lqip` record rendered to packed RGBA8 +//! through `capsule_core::lqip`, the same implementation the import pipeline encodes with and +//! `capsule-wasm` decodes with (slice `S-B14`). //! - [`AssetRecord`], [`AssetStackRecord`], [`StackMemberRecord`], //! [`AlbumRecord`] — catalog row mirrors. //! - [`AssetSidecarRecord`] / [`serialize_sidecar`] / [`deserialize_sidecar`] — @@ -43,7 +46,7 @@ mod gate; mod records; mod sidecar; -pub use catalog::Catalog; +pub use catalog::{Catalog, LqipPlaceholder, render_lqip}; pub use error::CatalogError; pub use gate::{GatedView, LocalAuthError, LocalAuthGate}; pub use records::{AlbumRecord, AssetRecord, AssetStackRecord, StackMemberRecord}; diff --git a/capsule-core/Cargo.toml b/capsule-core/Cargo.toml index 6d086b2f..24b11e0b 100644 --- a/capsule-core/Cargo.toml +++ b/capsule-core/Cargo.toml @@ -25,7 +25,15 @@ required-features = ["ffi-bindgen"] # `wasm32-unknown-unknown` for the guest web-upload client (see `drop::seal_drop`). Build: # cargo build -p capsule-core --target wasm32-unknown-unknown --no-default-features default = ["native"] -native = ["dep:rusqlite", "dep:sqlite-vec", "mls"] +native = ["dep:rusqlite", "dep:sqlite-vec", "mls", "media"] +# `media` links the still-image decode/encode stack (`capsule_core::media`, slices `S-B1`/`S-B13`) +# over `rawshift-image`. Implied by `native`, so the CLI, the tests and the mobile FFI all carry +# decoders; **excluded** from the `wasm32-unknown-unknown` sealing build (`--no-default-features`), +# to which the whole stack is irrelevant. Every enabled codec is pure Rust and links no C, which +# is what keeps the mobile cross-builds working — see the dependency's own comment for why WebP +# is not among them. `capsule_core::lqip` stays unconditional and is NOT behind this feature — a +# placeholder must not depend on which client imported the photo (slice `S-B14`). +media = ["dep:rawshift-image"] # `mls` links the live OpenMLS group backend (`crypto::authority::OpenMlsAuthority`, slice # S-X1) pinned to the X-Wing PQ ciphersuite `MLS_256_XWING_CHACHA20POLY1305_SHA256_Ed25519` # (`0x004D`) via its formally-verified libcrux provider. Implied by `native` so the default @@ -77,6 +85,46 @@ chromahash = { version = "0.7.1", default-features = false, features = ["simd"] indexmap = { workspace = true } jiff = { workspace = true } kamadak-exif = "0.5" +# Still-image decode/encode for `capsule_core::media` (`media` feature). A **registry** +# dependency, not the pinned `rawshift/` submodule, which is an uninitialised newer +# v1-in-progress tree and is not a workspace member. Depended on directly rather than through +# the `rawshift` facade because only the per-crate dependency gives per-format Cargo control +# (`rawshift-image`'s own docs say so), and the format set is a licence, build-host and +# *portability* decision: +# +# - `jpeg-decode` / `png-decode` — pure-Rust zune decode for the two formats every library holds. +# **Decode only in the shipping graph.** Their encoders exist and only the +# test fixtures use them, so they ride `[dev-dependencies]` below: shipping +# `jpeg-encode` would put `jpeg-encoder` — and its conjunctive IJG licence +# arm, which cannot be elected away — into every release binary to satisfy +# nothing a user does. +# - `jxl` — jxl-oxide decode plus the `zune-jpegxl` encoder that produces the thumbnail +# tier. Pure Rust, and `image/jxl` is the tier table's committed *master* +# format. The backend is `JxlSimpleEncoder`, which is lossless — a thumbnail +# therefore costs more bytes than the table's q=50 intends, and closing that +# needs C libjxl (`jxl-encode-libjxl`: `bindgen` + `pkg-config`). +# - `tiff-decode` / `gif-decode` — pure Rust, no encoder needed. +# +# **`webp` is deliberately absent, and it is not a preference.** It does not compile for any +# aarch64 target: `codecs/webp.rs:164,177,190` pass `b"EXIF".as_ptr() as *const i8` to +# `WebPMuxSetChunk`, whose `libwebp-sys` 0.14.4 signature (`ffi.rs:881`) takes +# `*const core::ffi::c_char` — and `c_char` is `u8` on aarch64, so the cast is an E0308. That +# module is compiled under `any(webp-decode, webp-encode)`, so decode-only does not escape it. +# Every mobile target Capsule ships is aarch64, so WebP is a recognised-but-undecodable format +# here until upstream fixes the cast. +# +# Also absent: `heic` (system libheif), `avif` (image's `avif-native` -> system libdav1d for +# decode; `ravif` -> `rav1e/asm` -> nasm on every x86_64 build host for encode), `svg`, and the +# RAW families (`experimental`; CR3 pixel decode unimplemented upstream). Each is a typed +# `media::MediaError::UnsupportedFormat` today rather than a silent gap. MPL-2.0, already +# allow-listed in `deny.toml`; see the Media row in design/dependencies.md. +rawshift-image = { version = "0.1.1", default-features = false, features = [ + "jpeg-decode", + "png-decode", + "jxl", + "tiff-decode", + "gif-decode", +], optional = true } # Bundled SQLite (C) — the on-device library index. Optional + gated by `native` because it # cannot target `wasm32-unknown-unknown`; the WASM sealing build drops it. rusqlite = { version = "0.32", features = ["bundled"], optional = true } @@ -188,6 +236,16 @@ getrandom_04 = { package = "getrandom", version = "0.4", features = ["wasm_js"] uuid = { workspace = true, features = ["rng-getrandom"] } [dev-dependencies] +# The encoders the fixtures need and the shipping graph does not. Cargo unifies features per +# build, so a `cargo build` links decode only while `cargo test` gets the encoders — which keeps +# `jpeg-encoder`'s IJG arm out of every release binary while leaving it in the graph +# `cargo deny --all-features` inspects, so its `deny.toml` exception and NOTICE section stay +# matched rather than becoming stale. The `media` feature is named explicitly because a +# dev-dependency does not inherit the optional dependency's gate. +rawshift-image = { version = "0.1.1", default-features = false, features = [ + "jpeg", + "png", +] } tempfile = "3" # Integration tests (e.g. the S-D14 placement audit) build UUIDs to exercise the `library::paths` # surface; `uuid` is a normal dependency, so the test crate needs its own dev-dependency on it. diff --git a/capsule-core/src/crypto/provenance/manifest.rs b/capsule-core/src/crypto/provenance/manifest.rs index b859603a..11d87261 100644 --- a/capsule-core/src/crypto/provenance/manifest.rs +++ b/capsule-core/src/crypto/provenance/manifest.rs @@ -246,9 +246,47 @@ pub struct DerivativeCore { /// Which kind of derivative. pub role: DerivativeRole, /// MIME/format string, e.g. `image/avif` or `embedding/mobileclip-b`. + /// + /// **A `String`, and deliberately so, even though the still formats are a closed set.** + /// Two reasons, and both are about keeping a *policy* rejection from becoming a *parse* + /// failure: + /// + /// - the same field carries the embedding-role grammar `embedding/{model_id}` + /// (`crate::ml`), which no still-format enum can model, so a single typed field would + /// have to be an enum over both grammars; + /// - a `#[serde(try_from = "String")]` newtype would make an *older* manifest naming a + /// future codec fail at deserialisation — before its signature is examined at all — + /// turning "this receiver does not recognise that format" into "this manifest is + /// unreadable". + /// + /// The closed set is enforced at the two boundaries instead: production, because + /// `media::generate_still_derivatives` only ever writes + /// [`DerivativeFormat::mime`](crate::derivative_format::DerivativeFormat::mime); and + /// verification, via + /// [`verify_still_format`](crate::derivative_format::verify_still_format), which rejects a + /// still-role manifest whose value is outside the set and leaves the embedding-role grammar + /// alone. + /// + /// Both live in [`crate::derivative_format`], which is **unconditional** — deliberately not + /// behind the `media` feature. `capsule-server` and `capsule-wasm` build + /// `default-features = false`, and they are exactly the crates that receive a manifest they + /// did not author, so a check they could not link would be a closed set only its producer + /// could evaluate. SSoT: [Thumbnails](https://docs/design/thumbnails/). pub format: String, /// Content-address digest over the derivative ciphertext. pub ciphertext_hash: Hash32, + /// The STREAM nonce prefix the derivative ciphertext was produced under; folded into the + /// file-key salt, so it also selects the key. + /// + /// **Required, not `Option`.** Derivative bytes are encrypted client-side exactly like the + /// original ([Encryption](https://docs/design/cryptography/encryption/)), so a receiver that + /// cannot recover this cannot decrypt the blob at all — an absent prefix would be an + /// unopenable derivative rather than a tolerable gap. It is safe to make it required + /// because no real `derivative-manifest/v1` has ever been written to any store: derivatives + /// were unconditionally `DeferredNoCodec` until the decoder landed, and `crate::ml` + /// constructs no derivative manifest. There is nothing to stay compatible with, so the + /// schema string does not move. + pub nonce_prefix: [u8; 7], /// Device that generated the derivative. pub generated_by_device: Uuid, /// Generating client version. @@ -473,6 +511,7 @@ mod tests { role: DerivativeRole::Thumbnail, format: "image/avif".into(), ciphertext_hash: Hash32([0xAB; 32]), + nonce_prefix: [9, 8, 7, 6, 5, 4, 3], generated_by_device: Uuid::from_u128(0xD1), generated_by_client: "capsule-cli/0.1.0".into(), model_id: None, diff --git a/capsule-core/src/derivative_format.rs b/capsule-core/src/derivative_format.rs new file mode 100644 index 00000000..770eb18f --- /dev/null +++ b/capsule-core/src/derivative_format.rs @@ -0,0 +1,164 @@ +//! The closed set of committed still-derivative formats, and the structural check over it. +//! +//! SSoT: [Thumbnails and Previews](https://docs/design/thumbnails/) — the tier table's format +//! column *is* this enum, and "every receiver (and every federated peer) compares +//! `DerivativeManifest.format` against this list" is +//! [`verify_still_format`](crate::derivative_format::verify_still_format). +//! +//! # Why this is at the crate root and not in `capsule_core::media` +//! +//! It was in `media` first, and that was a placement mistake rather than a constraint. `media` +//! is behind the `media` feature that `native` implies, so `capsule-server` and `capsule-wasm` +//! — both `default-features = false` — cannot link it. Those are exactly the two crates that +//! *receive* a manifest they did not author, which is where a structural rejection has to run. +//! A closed set only a producer can evaluate is not a closed set. +//! +//! So it lives here, beside nothing and depending on nothing but +//! [`crate::crypto::provenance`], which is itself unconditional for the same reason. `media` +//! re-exports both names, so every existing `media::DerivativeFormat` path still resolves. +//! +//! This mirrors [`crate::lqip`]: a contract every surface needs cannot live inside a +//! feature-gated stack, however natural the stack looks as a home. + +use std::fmt; + +use crate::crypto::provenance::{DerivativeManifest, DerivativeRole}; + +/// The closed set of committed still-derivative formats — the tier table's format column. +/// +/// The wire value is [`mime`](Self::mime), carried in `DerivativeManifest.format`. A value +/// outside this set is a structural rejection, never a "future format to ignore". +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DerivativeFormat { + /// **JPEG XL** — the committed primary/master still codec, and the one format this build + /// encodes. Losslessly: the pure-Rust backend is `zune-jpegxl`'s `JxlSimpleEncoder`. + Jxl, + /// **AVIF** — the universal delivery format for clients without a JXL decoder. Not + /// encodable in this build. + Avif, + /// **WebP** — the last-resort delivery fallback. Not encodable in this build: the crate's + /// WebP codec does not compile for aarch64 — see `media::StillFormat::WebP`, which cannot be + /// linked from here because `media` is feature-gated and this module is not. + WebP, + /// The recognised `format = "original"` sentinel: the tier **references** the original asset + /// rather than generating a redundant derivative, because the source is not larger than the + /// tier's cap. **Distinct from an absent derivative** — this is an explicit, signed marker, + /// where absence means "rebuildable from the original". + /// + /// A sentinel derivative carries **no bytes of its own** (`media::GeneratedDerivative::bytes` + /// is empty). "References" is the operative word in the contract: the signed manifest's + /// `ciphertext_hash` content-addresses the original, which the holder already has, so + /// copying the bytes under a thumbnail's name would duplicate a file sitting two directories + /// up *and* re-expose the original's EXIF — GPS included — as a derivative blob, where a + /// re-encoded thumbnail is metadata-free by construction. + Original, +} + +impl DerivativeFormat { + /// The committed still formats per tier, in delivery-preference order: the JXL master, then + /// the AVIF -> WebP delivery variants. + pub const STILL_DELIVERY_ORDER: [Self; 3] = [Self::Jxl, Self::Avif, Self::WebP]; + + /// The exact wire string for `DerivativeManifest.format`. + pub const fn mime(self) -> &'static str { + match self { + Self::Jxl => "image/jxl", + Self::Avif => "image/avif", + Self::WebP => "image/webp", + Self::Original => "original", + } + } + + /// The on-disk file extension for a persisted derivative of this format. `Original` has + /// none of its own — it reuses the source asset's. + pub const fn extension(self) -> Option<&'static str> { + match self { + Self::Jxl => Some("jxl"), + Self::Avif => Some("avif"), + Self::WebP => Some("webp"), + Self::Original => None, + } + } + + /// Parse a `DerivativeManifest.format` value against the closed set. `None` **is** the + /// structural rejection. + pub fn parse(s: &str) -> Option { + match s { + "image/jxl" => Some(Self::Jxl), + "image/avif" => Some(Self::Avif), + "image/webp" => Some(Self::WebP), + "original" => Some(Self::Original), + _ => None, + } + } + + /// Whether a `format` string names a currently-recognised still-derivative format — the + /// exact check a receiver runs. + pub fn is_recognized(s: &str) -> bool { + Self::parse(s).is_some() + } + + /// Whether this build can produce bytes in this format. + pub const fn is_encodable(self) -> bool { + matches!(self, Self::Jxl | Self::Original) + } +} + +impl fmt::Display for DerivativeFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.mime()) + } +} + +/// The closed-set check a receiver runs on a still-role derivative manifest. +/// +/// Returns the parsed format for a `thumbnail` or `preview` manifest whose `format` is in the +/// closed set. An embedding-role manifest is **not** rejected: its `format` is +/// `embedding/{model_id}`, which this set deliberately does not model, so it is reported as +/// [`None`] rather than as a violation. +/// +/// # Errors +/// `media::MediaError::UnsupportedFormat` — which carries the still format Capsule *would* have +/// needed — is not what an unrecognised value produces, because there is no still format to name +/// (and this module cannot reference `media` in any case: it is unconditional and `media` is +/// not). An unrecognised still-role format is `Err(format.to_string())`. +pub fn verify_still_format( + manifest: &DerivativeManifest, +) -> Result, String> { + let core = &manifest.core; + match core.role { + DerivativeRole::Thumbnail | DerivativeRole::Preview => { + DerivativeFormat::parse(&core.format) + .map(Some) + .ok_or_else(|| core.format.clone()) + } + // Not a still. The embedding-role format grammar belongs to `crate::ml`. + DerivativeRole::Embedding => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// **The reason this module is not in `media`.** These tests compile and run under + /// `--no-default-features`, which is how `capsule-server` and `capsule-wasm` build — the two + /// crates that receive a `DerivativeManifest` they did not author. A closed set only its + /// producer can evaluate is not a closed set. + /// + /// This test asserts nothing a caller could not, and that is the point: it exists so the + /// *linkage* is exercised by the `--no-default-features` build rather than assumed. + #[test] + fn the_closed_set_is_evaluable_without_the_media_feature() { + for format in [ + DerivativeFormat::Jxl, + DerivativeFormat::Avif, + DerivativeFormat::WebP, + DerivativeFormat::Original, + ] { + assert_eq!(DerivativeFormat::parse(format.mime()), Some(format)); + } + assert!(!DerivativeFormat::is_recognized("image/future-codec")); + assert!(!DerivativeFormat::is_recognized("embedding/mobileclip-b")); + } +} diff --git a/capsule-core/src/import/executor.rs b/capsule-core/src/import/executor.rs index ec4e264b..defebe75 100644 --- a/capsule-core/src/import/executor.rs +++ b/capsule-core/src/import/executor.rs @@ -4,10 +4,11 @@ //! [`Workspace::import_asset_with`](crate::lifecycle::Workspace::import_asset_with): every member //! becomes a signed [`SidecarV1`](crate::sidecar::SidecarV1) + signed manifest + //! append-only provenance, self-verified through -//! [`verify_asset`](crate::crypto::verify_asset::verify_asset), and — when a still encoder is -//! attached to the workspace — with signed thumbnail/preview derivatives + an LQIP in the -//! sidecar. No still encoder exists in this build: the media stack is retired to -//! `legacy-review/` and restoring it is `S-B1`. +//! [`verify_asset`](crate::crypto::verify_asset::verify_asset), and — when the still decodes — +//! with a chromahash `lqip` in the sidecar and signed thumbnail derivatives on disk +//! ([`capsule_core::media`](crate::media), slices `S-B1`/`S-B13`/`S-B14`). A format with no +//! codec in this build still imports, as a signed original with the gap recorded rather than +//! hidden. //! //! This retired the legacy unsigned `AssetSidecar` write path from the executor; the production //! write path itself is now gone (`S-G4`) — no code writes unsigned sidecars anymore. Only the @@ -246,6 +247,7 @@ fn execute_candidate( path.clone(), ImportOutcome::Imported { derivatives: receipt.derivatives, + deferred_formats: receipt.deferred_formats, }, )); } @@ -419,22 +421,28 @@ mod tests { /// **The S-B13 contract (slice `S-B13`).** An original whose format has no codec in this /// build is imported as a signed, encrypted, verifiable asset — it simply arrives without a - /// thumbnail/preview, and the run summary says so. + /// thumbnail, and the run summary says so. /// - /// **`S-C59` narrowed what this can assert.** It used to pin the distinction the logs must - /// preserve: `iphone.heic` an *expected* deferral, `snap.jpg` a *genuine* decode failure of a - /// format we do support. With `capsule_core::media` retired there is no decoder for any - /// format, so both are deferrals and the distinction is unobservable — it comes back with - /// Rawshift. What survives is the half that matters most and would be the worst to lose - /// silently: **an undecodable original is still a signed, encrypted, self-verifying backup**, - /// and both files land. + /// **The distinction is observable again.** `S-C59` retired the decoder and collapsed both + /// files below into deferrals, which made the logs' most useful property untestable. With + /// `capsule_core::media` on `rawshift-image` the two are apart once more, and they are apart + /// for the reason that matters rather than by extension: `iphone.heic` is a format Capsule + /// *recognises and cannot decode* (an expected, backfillable gap), while `snap.jpg` is a + /// format it can decode whose bytes are not a JPEG (a real problem). Both still land as + /// signed, encrypted, self-verifying backups, which is the half that would be worst to lose. #[test] fn originals_with_no_codec_are_still_imported_and_signed() { use crate::lifecycle::DerivativeStatus; let src = TempDir::new().unwrap(); let lib_dir = TempDir::new().unwrap(); - fs::write(src.path().join("iphone.heic"), b"fake heic bytes").unwrap(); + // A real ISO-BMFF `ftyp heic` header, so the classification rests on the **bytes** — + // which is what the doc above claims. `b"fake heic bytes"` carried no `ftyp` and + // silently exercised the extension fallback instead. + let mut heic = vec![0, 0, 0, 0x20]; + heic.extend_from_slice(b"ftypheic"); + heic.extend_from_slice(&[0; 16]); + fs::write(src.path().join("iphone.heic"), &heic).unwrap(); fs::write(src.path().join("snap.jpg"), b"not really a jpeg").unwrap(); let mut ws = signed_workspace(lib_dir.path()); @@ -452,28 +460,32 @@ mod tests { assert_eq!(summary.imported_count(), 2, "both originals are backed up"); assert_eq!( summary.deferred_derivative_count(), - 2, - "with no decoder in the build, every still is a codec deferral" + 1, + "the HEIC is an expected codec deferral" ); assert_eq!( summary.decode_failed_count(), + 1, + "the .jpg is a format we do decode, failing on these bytes — a real problem" + ); + assert_eq!( + summary.deferred_format_count(), 0, - "nothing is *attempted*, so nothing can fail to decode — the distinction returns \ - with Rawshift" + "nothing decoded, so no per-format variant was even attempted" ); - // Reported per file rather than only in aggregate, so the shape a caller reads is - // pinned even while there is one reason rather than two. + // Reported per file rather than only in aggregate, so a caller reads the reason for the + // file in front of it. for (path, outcome) in &summary.outcomes { - let ImportOutcome::Imported { derivatives } = outcome else { + let ImportOutcome::Imported { derivatives, .. } = outcome else { panic!("{} should have imported, got {outcome:?}", path.display()); }; - assert_eq!( - *derivatives, - DerivativeStatus::DeferredNoCodec, - "for {}", - path.display() - ); + let expected = if path.extension().is_some_and(|e| e == "heic") { + DerivativeStatus::DeferredNoCodec + } else { + DerivativeStatus::DecodeFailed + }; + assert_eq!(*derivatives, expected, "for {}", path.display()); } // Both land on the signed path and self-verify — a missing thumbnail is not a missing @@ -488,6 +500,11 @@ mod tests { /// A RAW-only candidate — no same-stem JPEG to fall back on — still lands as a signed, /// self-verifying original. RAW has no decoder in this build, which is exactly why this /// needs pinning: the archive is the whole point, the derivative is a bonus (slice `S-B13`). + /// + /// A Sony ARW is a TIFF container, so its *header* says TIFF and only the extension names + /// the family. This fixture is not a real ARW, so the classification here rests on the + /// extension fallback — which is the path a real one would also take for the family, and + /// either way the outcome is the same expected deferral. #[test] fn raw_only_candidate_lands_as_a_signed_original() { use crate::lifecycle::DerivativeStatus; @@ -512,7 +529,8 @@ mod tests { assert!(matches!( summary.outcomes[0].1, ImportOutcome::Imported { - derivatives: DerivativeStatus::DeferredNoCodec + derivatives: DerivativeStatus::DeferredNoCodec, + deferred_formats: 0, } )); diff --git a/capsule-core/src/import/progress.rs b/capsule-core/src/import/progress.rs index 47097288..2a4a1691 100644 --- a/capsule-core/src/import/progress.rs +++ b/capsule-core/src/import/progress.rs @@ -12,6 +12,11 @@ pub enum ImportOutcome { /// fully successful import that happens to be missing its derivative (slice `S-B13`). Imported { derivatives: DerivativeStatus, + /// How many `(tier, format)` pairs the tier table commits to and this build cannot + /// encode. Orthogonal to `derivatives`: a `Decoded` asset with a renderable JXL + /// thumbnail still reports the AVIF delivery variant and WebP as deferred, and that + /// count is how the gap shrinks visibly as codecs land rather than silently. + deferred_formats: u32, }, DuplicateSkipped { existing_uuid: String, @@ -77,13 +82,36 @@ impl ImportExecutionSummary { matches!( o, ImportOutcome::Imported { - derivatives: DerivativeStatus::DeferredNoCodec + derivatives: DerivativeStatus::DeferredNoCodec, + .. } ) }) .count() } + /// The total number of `(tier, format)` pairs across the run that the tier table commits to + /// and this build cannot encode (slice `S-B13`). + /// + /// **Not a failure count, and not comparable to + /// [`deferred_derivative_count`](Self::deferred_derivative_count).** That one counts *assets* + /// with no thumbnail at all; this one counts *format variants* missing from assets that do + /// have one. A library of decodable JPEGs reports zero deferred derivatives and two deferred + /// formats per asset — the JXL master and the AVIF delivery variant — which is exactly the + /// number that should fall to zero as the encoders land, and the reason it is reported rather + /// than left implicit in a doc. + pub fn deferred_format_count(&self) -> usize { + self.outcomes + .iter() + .map(|(_, o)| match o { + ImportOutcome::Imported { + deferred_formats, .. + } => *deferred_formats as usize, + _ => 0, + }) + .sum() + } + /// How many imported assets are in a format this build *does* support but whose bytes did /// not decode — unlike [`deferred_derivative_count`](Self::deferred_derivative_count) this /// is a real problem worth surfacing, not an expected gap. The original is still imported. @@ -94,7 +122,8 @@ impl ImportExecutionSummary { matches!( o, ImportOutcome::Imported { - derivatives: DerivativeStatus::DecodeFailed + derivatives: DerivativeStatus::DecodeFailed, + .. } ) }) diff --git a/capsule-core/src/lib.rs b/capsule-core/src/lib.rs index 620daf1e..034395fd 100644 --- a/capsule-core/src/lib.rs +++ b/capsule-core/src/lib.rs @@ -19,6 +19,13 @@ pub mod sharing; /// build-embedded git commit (S-D15). Always compiled: pure string formatting, no native deps. pub mod client_build; +/// The closed set of still-derivative formats and the structural check over it — the tier +/// table's format column as a type. Always compiled, and for the same reason [`lqip`] is: the +/// crates that *receive* a `DerivativeManifest` (`capsule-server`, `capsule-wasm`) build with +/// `default-features = false`, so a check they cannot link is a check that never runs. Depends +/// only on [`crypto::provenance`]; `media` re-exports it. +pub mod derivative_format; + /// LQIP — the chromahash placeholder carried in the signed sidecar's `lqip` field (S-B14). /// Always compiled, and deliberately so: the placeholder is produced by the import pipeline, /// read by the apps through the uniffi FFI, and read by the browser through `capsule-wasm`, so @@ -54,6 +61,13 @@ pub mod import; pub mod library; #[cfg(feature = "native")] pub mod lifecycle; +/// Still decode, orientation, metadata normalisation and derivative generation over +/// `rawshift-image` (`media` feature, implied by `native`; slices `S-B1`/`S-B13`). Feature-gated +/// rather than `native`-gated so the codec stack is one manifest edit away from being dropped +/// from a size-constrained build, and so the `wasm32-unknown-unknown` sealing surface provably +/// does not link it. See [`media`]. +#[cfg(feature = "media")] +pub mod media; #[cfg(feature = "native")] pub mod metadata; #[cfg(feature = "native")] diff --git a/capsule-core/src/lifecycle/derivatives.rs b/capsule-core/src/lifecycle/derivatives.rs new file mode 100644 index 00000000..54bff749 --- /dev/null +++ b/capsule-core/src/lifecycle/derivatives.rs @@ -0,0 +1,1267 @@ +//! The one `lifecycle` file that reaches [`crate::media`]: decode a still once at import and +//! derive everything that needs pixels (slices `S-B1`, `S-B13`). +//! +//! # Why this is not feature-gated +//! +//! `capsule-core`'s `native` feature *implies* `media`, and the whole `lifecycle` module is +//! `native`-gated, so a build that compiles this file always has the codec stack. The two builds +//! that drop `media` — `capsule-server` and `capsule-wasm`, both +//! `default-features = false` — drop `lifecycle` with it. A `#[cfg(feature = "media")]` here +//! would therefore guard nothing while making every signature read as optional; if the +//! implication is ever removed, this file fails to compile, which is the right way for that +//! decision to surface. +//! +//! # Never fails the import +//! +//! Capsule is a backup tool. Every path below degrades to "the original is imported signed, +//! encrypted and `verify_asset`-accepting, without a placeholder or a thumbnail" and records +//! **why** in the returned [`DerivativeStatus`]. The only errors that propagate are the ones +//! that mean the *workspace* is broken — a missing album, a signer that refused — not the ones +//! that mean the pixels were unreadable. + +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::path::Path; + +use uuid::Uuid; + +use super::{AssetState, DerivativeStatus, LifecycleError, Result, Workspace, media_dir}; +use crate::cbor; +use crate::crypto::encryption::rekey::encrypt_asset_rekey_with_prefix; +use crate::crypto::encryption::stream::{AssetEncryption, NONCE_PREFIX_LEN}; +use crate::crypto::hash::{self, Hash32}; +use crate::crypto::keys::{Amk, AmkVersion}; +use crate::crypto::primitives::{CRYPTO_SUITE_ID, PROTOCOL_VERSION}; +use crate::crypto::provenance::{DerivativeManifest, DerivativeRole}; +use crate::crypto::rng; +use crate::exif::extract::ExifExtract; +use crate::lqip::Lqip; +use crate::media::{ + DecodedImage, DerivativeContext, DerivativeSealer, DerivativeTier, GeneratedDerivative, + MediaError, RawshiftDecoder, SealedDerivative, StillFormat, decode_guarded, + generate_still_derivatives, guarded, +}; +use crate::sidecar::sidecar_v1::{Dimensions, Lqip as SidecarLqip}; + +/// The file under import, as [`prepare_still`](Workspace::prepare_still) needs to see it. +/// +/// A parameter object rather than four positional arguments: these four are one thing — the +/// bytes on the way in and what the scanner already learned about them — and they are always +/// passed together. The alternative was silencing `clippy::too_many_arguments`, which would have +/// hidden that the signature had grown two *kinds* of input (the file, and the crypto identity +/// it commits under) without saying so. +pub(super) struct StillSource<'a> { + /// The file's bytes. + pub(super) plaintext: &'a [u8], + /// Its lowercase extension without the dot, `""` when it has none. + pub(super) ext: &'a str, + /// Where it came from — for logs only; the bytes above are authoritative. + pub(super) src: &'a Path, + /// What `capsule_core::exif` read off it, the fallback for dimensions. + pub(super) exif: &'a ExifExtract, +} + +/// Everything one still yields in a single decode pass: the sidecar fields, the signed +/// derivatives to persist after the durable commit, and the reason for anything missing. +pub(super) struct PreparedStill { + /// The format detection identified, if the bytes are a still Capsule models. Drives the + /// sidecar's `content_type` from the *header* rather than from the file name. + pub(super) format: Option, + /// Pixel dimensions when the still decoded, EXIF dimensions otherwise, `None` if neither. + /// + /// Decoded pixels win over EXIF because they are post-orientation: a quarter-turned JPEG's + /// EXIF `PixelXDimension` is its *stored* width, which is transposed relative to what a + /// viewer shows. + pub(super) dimensions: Option, + /// The sidecar LQIP, present only when the still decoded. + pub(super) lqip: Option, + /// Signed thumbnail derivatives, to be written after the asset's own files. + pub(super) derivatives: Vec, + /// How many `(tier, format)` pairs the tier table commits to and this build cannot encode. + pub(super) deferred_formats: usize, + /// Whether derivatives were generated, and if not, why. + pub(super) status: DerivativeStatus, +} + +impl PreparedStill { + /// The outcome for bytes that yielded no pixels: EXIF dimensions only, no LQIP, no + /// derivatives, and `status` carrying which of the reasons it was. + fn undecoded( + format: Option, + exif_dimensions: Option, + status: DerivativeStatus, + ) -> Self { + Self { + format, + dimensions: exif_dimensions, + lqip: None, + derivatives: Vec::new(), + deferred_formats: 0, + status, + } + } +} + +/// Map a decode failure onto the status the run summary counts, logging the distinction that +/// makes the two reasons useful (slice `S-B13`). +fn classify(error: &MediaError, src: &Path, format: Option) -> DerivativeStatus { + match error { + MediaError::UnsupportedFormat { format, op } => { + tracing::warn!( + path = %src.display(), + %format, + op = op.as_str(), + supported = ?crate::media::SUPPORTED_STILL_FORMATS, + "derivatives: no codec for this format in this build; the original is imported \ + signed and encrypted, but without a thumbnail or LQIP until the codec lands. \ + Derivatives are backfillable from the stored original (S-B13)" + ); + DerivativeStatus::DeferredNoCodec + } + MediaError::NotAStillImage => { + tracing::debug!( + path = %src.display(), + "derivatives: not a still image Capsule models; nothing to decode" + ); + DerivativeStatus::NotAKnownStill + } + // Everything else is a format we *do* support failing on these particular bytes — a + // real problem worth investigating, not an expected gap. + error => { + tracing::warn!( + path = %src.display(), + ?format, + %error, + "derivatives: a supported format failed to decode; the original is imported \ + signed and encrypted, but without a thumbnail or LQIP" + ); + DerivativeStatus::DecodeFailed + } + } +} + +/// Compute the sidecar LQIP from a decoded frame. +/// +/// From the **full-resolution, orientation-applied** frame, never from the thumbnail: +/// chromahash consumes the whole frame and band-limits on the read side via `decode_capped`, so +/// pre-resizing would silently cap fidelity the format can carry ([`crate::lqip`]). +fn lqip_from(decoded: &DecodedImage, src: &Path) -> Option { + // Guarded because `chromahash` is pre-1.0 too and its own `encode` panics on a zero + // dimension or a length mismatch. `Lqip::encode` checks both, so this is belt and braces — + // but it is what makes the module's "no codec can abort an import" claim true rather than + // nearly true. + let encoded = guarded("lqip", || { + Lqip::encode( + decoded.width(), + decoded.height(), + &decoded.image.rgba, + decoded.gamut, + ) + .map_err(|e| MediaError::Decode { + format: decoded.format, + detail: format!("LQIP encode: {e}"), + }) + }); + match encoded { + Ok(lqip) => Some(lqip.to_sidecar()), + Err(error) => { + // A decoded frame satisfies both of `encode`'s preconditions by construction, so + // this is unreachable rather than expected — logged as such, and never fatal. + tracing::warn!( + path = %src.display(), + %error, + "derivatives: a decoded frame was rejected by the LQIP encoder; importing \ + without a placeholder" + ); + None + } + } +} + +/// What an asset's existing derivative bundle constrains about the next generation. +/// +/// One read, two facts, because both come off the same file and both are needed together. +pub(super) struct ExistingDerivatives { + /// The current head of each role's chain. Empty when the asset has no bundle yet, which is + /// every import: a create starts each role's chain. It is a **regeneration** — the `#437` + /// backfill that adds a second format to an asset that already has one — that needs this, + /// and it needs it to be right the first time, because a forked chain is not something a + /// later run can repair. + /// + /// The link is SHA-256 over the manifest's canonical CBOR, signatures included: the same + /// content-hash link the asset provenance chain uses. + pub(super) heads: HashMap, + /// Every `nonce_prefix` already used for this `file_id` by a derivative. + /// + /// The encryption doc makes the refusal normative: "the writer additionally refuses to emit + /// a `nonce_prefix` it has already used for that `file_id` … the same rule governs + /// derivative re-encryption". A prefix is folded into the file-key salt, so reusing one + /// reuses the *key* as well as the nonce — the keystream separation the whole construction + /// rests on. + pub(super) used_prefixes: HashSet<[u8; NONCE_PREFIX_LEN]>, +} + +/// Read `asset_id`'s persisted derivative bundle, if it has one. +/// +/// A bundle that does not decode yields the empty answer *and a warning*: treating an +/// unreadable bundle as "no constraints" is the safe direction for the chain (a role restarts) +/// but the unsafe one for prefixes, so the warning says which risk is being taken. +pub(super) fn existing_derivatives(dir: &Path, asset_id: Uuid) -> ExistingDerivatives { + let empty = ExistingDerivatives { + heads: HashMap::new(), + used_prefixes: HashSet::new(), + }; + let path = dir.join(format!("{}.derivatives.cbor", asset_id.simple())); + let Ok(bytes) = fs::read(&path) else { + return empty; + }; + let Ok(manifests) = cbor::from_slice::>(&bytes) else { + tracing::warn!( + path = %path.display(), + "derivatives: undecodable bundle; every role's chain restarts and no previously used \ + nonce prefix can be excluded from the next draw" + ); + return empty; + }; + + // Generation order is the chain order, so the last manifest of a role is that role's head. + let mut heads = HashMap::new(); + let mut used_prefixes = HashSet::new(); + for manifest in &manifests { + used_prefixes.insert(manifest.core.nonce_prefix); + match cbor::to_canonical_vec(manifest) { + Ok(canonical) => { + heads.insert(manifest.core.role, hash::hash_bytes(&canonical)); + } + Err(error) => tracing::warn!( + %error, + "derivatives: a persisted manifest did not re-serialise; its role's chain \ + restarts rather than linking to something unverifiable" + ), + } + } + ExistingDerivatives { + heads, + used_prefixes, + } +} + +/// The album-key half of derivative generation: `media` produces the bytes, this encrypts them. +/// +/// One `encrypt_asset_rekey_with_prefix` per derivative under the **source asset's** `file_id` +/// and the album's current AMK, so every derivative gets its own file key per the encryption +/// doc's per-file derivation. The ciphertext is deliberately dropped: the client keeps the +/// plaintext derivative on disk (the local gallery paints it) and re-derives the ciphertext at +/// push time from the recorded prefix, exactly as it already does for the original. +/// +/// # The reuse refusal +/// +/// A CSPRNG draw is not on its own what the design asks for. The encryption doc requires that +/// the writer *refuse* a `nonce_prefix` it has already used for that `file_id`, "defense in +/// depth on top of the CSPRNG draw", and says the same rule governs derivative re-encryption. +/// So [`used`](Self::used) starts as the original's prefix plus every prefix in the existing +/// bundle, each newly sealed prefix joins it, and a collision is redrawn. +/// +/// A prefix is folded into the file-key salt, so a reused one reuses the **key** as well as the +/// nonce — two blobs under one keystream, which is the failure the whole construction exists to +/// prevent. +struct AlbumSealer<'a> { + amk: &'a Amk, + asset_id: Uuid, + /// Prefixes already spoken for on this `file_id`. `RefCell` because [`DerivativeSealer`] + /// takes `&self` — the seam is shared, and each seal has to see what the last one used. + used: RefCell>, + /// Where a candidate prefix comes from: the OS CSPRNG in production, forced in the test + /// that proves the refusal fires. + draw: &'a dyn Fn() -> [u8; NONCE_PREFIX_LEN], +} + +/// How many times a collision is redrawn before the draw itself is called broken. +/// +/// A 7-byte prefix collides by chance at about 1 in 2^56, so a run of eight is not bad luck — +/// it is a CSPRNG returning something it should not, which is a **workspace** fault and not +/// this asset's. Hence `MediaError::Sign`, which decision 22 routes to a propagated error +/// rather than to a missing thumbnail: an import that cannot draw a safe nonce must stop, not +/// quietly write one derivative fewer. +const MAX_PREFIX_DRAWS: usize = 8; + +/// Test-only fault injection for the sealer. +/// +/// The two failure paths decision 22 and decision 23 turn on — a codec refusing a frame the +/// decoder accepted, and a codec *panicking* on one — cannot be produced from real bytes on +/// demand, and testing them anywhere but through `import_asset_with` proves nothing about the +/// property that matters: that **the asset still commits**. So the fault is injected at the one +/// point inside the real import path where a codec failure originates. +/// +/// `#[cfg(test)]`, so it does not exist in a release build at all — not a disabled branch, not a +/// dead field, absent. A thread-local rather than a parameter because threading an `Option<&dyn +/// DerivativeSealer>` through `prepare_still` would put a test seam in a production signature; +/// nextest runs each test in its own process, so there is nothing for it to leak into. +#[cfg(test)] +#[derive(Clone, Copy, Debug)] +pub(super) enum SealerFault { + /// A codec refuses the frame — an `Encode`-class error, which decision 22 degrades. + Refuse, + /// A pre-1.0 codec panics — which decision 23's guard has to catch. + Panic, +} + +#[cfg(test)] +thread_local! { + static SEALER_FAULT: RefCell> = const { RefCell::new(None) }; +} + +/// Run `body` with `fault` injected into every seal, restoring the previous state after. +#[cfg(test)] +pub(super) fn with_sealer_fault(fault: SealerFault, body: impl FnOnce() -> T) -> T { + SEALER_FAULT.with(|slot| *slot.borrow_mut() = Some(fault)); + let out = body(); + SEALER_FAULT.with(|slot| *slot.borrow_mut() = None); + out +} + +impl DerivativeSealer for AlbumSealer<'_> { + fn seal(&self, plaintext: &[u8]) -> std::result::Result { + #[cfg(test)] + if let Some(fault) = SEALER_FAULT.with(|slot| *slot.borrow()) { + match fault { + // Deliberately **not** `Sign`: this stands in for a codec refusing pixels, which + // decision 22 degrades to `DecodeFailed` rather than propagating. + SealerFault::Refuse => { + return Err(MediaError::Encode { + format: crate::media::DerivativeFormat::Jxl, + detail: "injected codec refusal".into(), + }); + } + SealerFault::Panic => panic!("injected codec panic on an accepted frame"), + } + } + + for attempt in 0..MAX_PREFIX_DRAWS { + let prefix = (self.draw)(); + if self.used.borrow().contains(&prefix) { + tracing::warn!( + asset_id = %self.asset_id, + attempt, + "derivatives: drew a nonce prefix already used for this file_id; redrawing" + ); + continue; + } + let (enc, _ciphertext, _file_key) = + encrypt_asset_rekey_with_prefix(self.amk, &self.asset_id, plaintext, prefix, None) + .map_err(|e| MediaError::Sign { + detail: format!("sealing the derivative: {e}"), + })?; + self.used.borrow_mut().insert(enc.nonce_prefix); + return Ok(SealedDerivative { + ciphertext_hash: enc.ciphertext_hash, + nonce_prefix: enc.nonce_prefix, + }); + } + Err(MediaError::Sign { + detail: format!( + "could not draw an unused nonce prefix for {} in {MAX_PREFIX_DRAWS} attempts", + self.asset_id + ), + }) + } +} + +impl Workspace { + /// Decode the still once and derive: the `content_type`, pixel `dimensions`, the sidecar + /// `lqip`, and the signed thumbnail derivatives. All are attached before the sidecar is + /// sealed, per the pipeline's Execute step. + /// + /// **Never fails over unreadable pixels.** A still this build cannot decode still commits as + /// a signed, encrypted original — it falls back to EXIF dimensions with no LQIP and no + /// derivatives, and the returned [`DerivativeStatus`] says which reason applied so the + /// caller can report the gap instead of it being invisible. + #[tracing::instrument( + level = "debug", + skip_all, + fields(asset_id = %asset_id, src = %source.src.display(), bytes = source.plaintext.len()) + )] + pub(super) fn prepare_still( + &self, + source: &StillSource<'_>, + asset_id: Uuid, + album_id: Uuid, + capture_utc: i64, + amk: &Amk, + original: &AssetEncryption, + ) -> Result { + let StillSource { + plaintext, + ext, + src, + exif, + } = *source; + let exif_dimensions = exif + .width + .zip(exif.height) + .map(|(width, height)| Dimensions { width, height }); + // Detected here as well as inside the decoder so the sidecar's `content_type` is + // header-derived even for a format with no codec: a HEIC is `image/heic` in the sidecar + // whether or not this build can read its pixels. + let sniffed = StillFormat::detect(plaintext, ext); + + let decoded = match decode_guarded(&RawshiftDecoder, plaintext, ext) { + Ok(decoded) => decoded, + Err(error) => { + let status = classify(&error, src, sniffed); + return Ok(PreparedStill::undecoded(sniffed, exif_dimensions, status)); + } + }; + + let dimensions = Some(Dimensions { + width: decoded.width(), + height: decoded.height(), + }); + if let (Some(pixels), Some(exif_dims)) = (dimensions.as_ref(), exif_dimensions.as_ref()) + && pixels != exif_dims + { + // Not an error: EXIF dimensions are pre-orientation and are frequently stale after + // an edit. Logged because a surprising sidecar dimension is otherwise unexplainable + // after the fact. + tracing::debug!( + asset_id = %asset_id, + pixel_width = pixels.width, + pixel_height = pixels.height, + exif_width = exif_dims.width, + exif_height = exif_dims.height, + orientation = decoded.orientation_applied, + "derivatives: decoded dimensions differ from EXIF; the pixels are authoritative" + ); + } + let lqip = lqip_from(&decoded, src); + + let album = self.album(&album_id)?; + let ExistingDerivatives { + heads, + mut used_prefixes, + } = existing_derivatives( + &media_dir(&self.root, capture_utc).join("derivatives"), + asset_id, + ); + // The original's prefix is spoken for too: it is a prefix used for this `file_id`. + used_prefixes.insert(original.nonce_prefix); + + let ctx = DerivativeContext { + source_asset_id: asset_id, + crypto_suite_id: CRYPTO_SUITE_ID, + protocol_version: PROTOCOL_VERSION.into(), + amk_version: AmkVersion(album.current_epoch), + generated_by_device: self.account.device.device_id, + generated_by_client: self.client_version.clone(), + generated_at: super::now_rfc3339(), + device_signer: self.device_signer.as_ref(), + write_tier_signer: album.write_tier_signer()?, + sealer: &AlbumSealer { + amk, + asset_id, + // Seeded with the original's own prefix and every prefix the existing bundle + // already spent on this `file_id`. + used: RefCell::new(used_prefixes), + draw: &rng::random_array::, + }, + // Empty on a create; a regeneration continues each role's chain from here. + prior_heads: &heads, + // The `original` sentinel references the original blob rather than encrypting + // anything, so it signs what the original's own manifest signs. + original: SealedDerivative { + ciphertext_hash: original.ciphertext_hash, + nonce_prefix: original.nonce_prefix, + }, + }; + // Guarded, and **not** propagated on failure. A codec refusing a frame the decoder just + // produced is a real defect, but it is this asset's derivative that is broken, not the + // workspace: the signed original, its dimensions and its placeholder are all still + // right, and failing the import would trade a missing thumbnail for a missing backup. + // Reported as `DecodeFailed` — the "a supported path produced no derivative and somebody + // should look at it" bucket — so the run summary counts it instead of staying silent. + let generated = guarded("derivatives", || { + generate_still_derivatives(&decoded, &DerivativeTier::GENERATED, &ctx) + }); + let derivatives = match generated { + Ok(derivatives) => derivatives, + // **A signing fault is the workspace's, not this asset's.** A hardware signer that + // refuses, or a missing epoch write-tier key, is the same fault that would stop the + // asset's own manifest being authored — degrading it to "no thumbnail" would hide a + // broken workspace behind a cosmetic gap. It propagates. + Err(error @ MediaError::Sign { .. }) => { + tracing::error!( + asset_id = %asset_id, + path = %src.display(), + %error, + "derivatives: the workspace could not author a signed derivative record" + ); + return Err(LifecycleError::Io(format!("derivative signing: {error}"))); + } + // Everything else is about *pixels*: a codec refused a frame, a resize was rejected, + // a third-party encoder panicked. The signed original, its dimensions and its + // placeholder are all still right, and failing the import would trade a missing + // thumbnail for a missing backup — which is the whole of `S-B13`'s reasoning and + // this module's stated contract. Reported as `DecodeFailed`, the "a supported path + // produced no derivative and somebody should look at it" bucket, so the run summary + // counts it instead of staying silent. + Err(error) => { + tracing::warn!( + asset_id = %asset_id, + path = %src.display(), + format = %decoded.format, + width = decoded.width(), + height = decoded.height(), + %error, + "derivatives: the still decoded but no derivative could be produced from it; \ + the original, its dimensions and its placeholder are committed regardless" + ); + return Ok(PreparedStill { + format: Some(decoded.format), + dimensions, + lqip, + derivatives: Vec::new(), + deferred_formats: 0, + status: DerivativeStatus::DecodeFailed, + }); + } + }; + + Ok(PreparedStill { + format: Some(decoded.format), + dimensions, + lqip, + deferred_formats: derivatives.deferred.len(), + derivatives: derivatives.generated, + status: DerivativeStatus::Decoded, + }) + } + + /// Write the generated derivative bytes plus their signed manifest bundle under the asset's + /// media directory: `derivatives/{uuid}.{role}.{ext}` and `{uuid}.derivatives.cbor`. + /// + /// The layout is the one the upload bundle reader already looks for: since `F5` it composes + /// a derivative's exact path from the manifest's `(role, format)` pair rather than scanning + /// for a `{uuid}.{role}.` prefix, so two formats of one role cannot be mistaken for each + /// other. + /// + /// Called **after** the asset's own files are durable: a derivative is regenerable and must + /// never be able to fail an import that has already committed. A write error is therefore + /// logged and swallowed rather than propagated. + pub(super) fn persist_derivatives( + &self, + asset: &AssetState, + derivatives: &[GeneratedDerivative], + ) { + if derivatives.is_empty() { + return; + } + let dir = media_dir(&self.root, asset.capture_utc).join("derivatives"); + if let Err(error) = fs::create_dir_all(&dir) { + tracing::warn!( + asset_id = %asset.asset_id, + dir = %dir.display(), + %error, + "derivatives: could not create the derivative directory; the asset is committed \ + and its derivatives are regenerable" + ); + return; + } + let stem = asset.asset_id.simple(); + + let mut manifests = Vec::with_capacity(derivatives.len()); + for derivative in derivatives { + // The `original` sentinel has no bytes of its own: its manifest *references* the + // original, whose content address it signs. Writing a byte-for-byte copy under a + // thumbnail's name would duplicate a file two directories up and re-expose the + // original's EXIF — GPS included — as a derivative, where a re-encoded thumbnail is + // metadata-free by construction. Its manifest still goes into the bundle: that + // signed marker is the difference between "the original *is* the thumbnail" and + // "the thumbnail is missing, rebuild it". + if let Some(format_ext) = derivative.format.extension() { + let path = dir.join(format!( + "{stem}.{}.{format_ext}", + derivative.tier.role_name() + )); + if let Err(error) = fs::write(&path, &derivative.bytes) { + tracing::warn!( + asset_id = %asset.asset_id, + path = %path.display(), + %error, + "derivatives: could not write a derivative; skipping it" + ); + continue; + } + } else { + debug_assert!( + derivative.bytes.is_empty(), + "only the byte-free `original` sentinel has no extension" + ); + } + manifests.push(derivative.manifest.clone()); + } + + if manifests.is_empty() { + return; + } + match cbor::to_canonical_vec(&manifests) { + Ok(bundle) => { + let path = dir.join(format!("{stem}.derivatives.cbor")); + if let Err(error) = fs::write(&path, bundle) { + tracing::warn!( + asset_id = %asset.asset_id, + path = %path.display(), + %error, + "derivatives: could not write the manifest bundle; the bytes on disk are \ + unusable without it and will be regenerated" + ); + return; + } + tracing::debug!( + asset_id = %asset.asset_id, + count = manifests.len(), + dir = %dir.display(), + "derivatives: persisted with their signed manifest bundle" + ); + } + Err(error) => tracing::warn!( + asset_id = %asset.asset_id, + %error, + "derivatives: the manifest bundle did not serialise; skipping persistence" + ), + } + } +} + +#[cfg(test)] +mod tests { + use rawshift_image::core::metadata::{ImageInfo, ImageMetadata}; + use rawshift_image::core::{BitDepth, MetadataEmbedOptions}; + use rawshift_image::formats::encode_rgb_image_to_vec; + use rawshift_image::formats::export::{ + CommonEncodeOptions, EncodeOptions, JpegEncEncodeConfig, ZunePngEncodeConfig, + }; + use tempfile::TempDir; + + use super::super::{DerivativeStatus, SignedImportOptions, Workspace, fast_workspace}; + use super::*; + use crate::crypto::hash; + use crate::crypto::provenance::DerivativeManifest; + use crate::media::{Decoder as _, DerivativeFormat, verify_still_format}; + use crate::sidecar::sidecar_v1::{SIDECAR_SCHEMA_V1, SidecarV1}; + + /// A deterministic RGB gradient, `width` x `height`, as interleaved RGB `u16`. + fn frame(width: u32, height: u32) -> rawshift_image::core::image::RgbImage { + let (w, h) = (width as usize, height as usize); + let mut data = Vec::with_capacity(w * h * 3); + for y in 0..h { + for x in 0..w { + data.push(((x * 255 / w) as u16) * 257); + data.push(((y * 255 / h) as u16) * 257); + data.push((((x + y) * 255 / (w + h)) as u16) * 257); + } + } + rawshift_image::core::image::RgbImage::with_color_space( + width, + height, + data, + rawshift_image::core::ColorSpace::Srgb, + ) + } + + fn common(metadata: MetadataEmbedOptions) -> CommonEncodeOptions { + CommonEncodeOptions { + metadata, + bit_depth: BitDepth::Eight, + } + } + + /// A JPEG carrying an EXIF orientation tag, so the decode path has a transform to apply and + /// the sidecar's dimensions have to disagree with the stored ones. + fn jpeg(width: u32, height: u32, orientation: Option) -> Vec { + let metadata = ImageMetadata { + image: ImageInfo { + orientation, + ..ImageInfo::default() + }, + ..ImageMetadata::default() + }; + let embed = MetadataEmbedOptions { + embed_exif: orientation.is_some(), + embed_icc: false, + embed_xmp: false, + }; + encode_rgb_image_to_vec( + &frame(width, height), + &metadata, + &EncodeOptions::JpegJpegEnc(JpegEncEncodeConfig { + common: common(embed), + quality: 90, + }), + ) + .expect("the fixture JPEG encodes") + } + + fn png(width: u32, height: u32) -> Vec { + encode_rgb_image_to_vec( + &frame(width, height), + &ImageMetadata::default(), + &EncodeOptions::PngZune(ZunePngEncodeConfig { + common: common(MetadataEmbedOptions::none()), + ..ZunePngEncodeConfig::default() + }), + ) + .expect("the fixture PNG encodes") + } + + /// A workspace with fast Argon2 params and its default album created. + fn workspace(dir: &Path) -> (Workspace, Uuid) { + let mut ws = fast_workspace(dir); + let album = ws.default_album_id(); + ws.create_album_with_id(album, "Imports").unwrap(); + (ws, album) + } + + /// Write `bytes` into `dir` under `name` and import it, returning the receipt. + fn import( + ws: &mut Workspace, + album: Uuid, + dir: &Path, + name: &str, + bytes: &[u8], + ) -> super::super::SignedImport { + let path = dir.join(name); + fs::write(&path, bytes).unwrap(); + ws.import_asset_with(album, &path, &SignedImportOptions::default()) + .expect("the import commits") + } + + /// Read back the signed sidecar an import wrote, from the library directory alone. + fn sidecar_of(root: &Path, asset_id: Uuid) -> SidecarV1 { + let mut stack = vec![root.join("media")]; + let name = format!("{}.cbor", asset_id.simple()); + while let Some(dir) = stack.pop() { + for entry in fs::read_dir(&dir).unwrap().flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if entry.file_name() == std::ffi::OsString::from(&name) { + let bytes = fs::read(&path).unwrap(); + return SidecarV1::from_canonical_slice(&bytes, SIDECAR_SCHEMA_V1) + .expect("the sidecar decodes"); + } + } + } + panic!("no sidecar for {asset_id}"); + } + + /// The derivatives directory for the bucket holding `asset_id`'s files. + fn derivatives_dir(root: &Path, asset_id: Uuid) -> std::path::PathBuf { + let mut stack = vec![root.join("media")]; + let name = format!("{}.cbor", asset_id.simple()); + while let Some(dir) = stack.pop() { + for entry in fs::read_dir(&dir).unwrap().flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if entry.file_name() == std::ffi::OsString::from(&name) { + return dir.join("derivatives"); + } + } + } + panic!("no bucket for {asset_id}"); + } + + /// **The `S-B14` acceptance case, and the first production caller of `capsule_core::lqip`.** + /// + /// A decodable still imports with real pixel dimensions and a 32-byte chromahash placeholder + /// inside the *signed* sidecar, and the sidecar still verifies — the placeholder is + /// signature-covered, so producing it is a signature-visible change and has to be checked as + /// one. + #[test] + fn a_decodable_still_imports_with_pixel_dimensions_and_an_lqip() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (mut ws, album) = workspace(lib.path()); + + let receipt = import( + &mut ws, + album, + src.path(), + "photo.jpg", + &jpeg(320, 240, None), + ); + assert_eq!(receipt.derivatives, DerivativeStatus::Decoded); + assert_eq!( + receipt.deferred_formats, 2, + "the JXL master and the AVIF delivery variant have no encoder in this build" + ); + + let sidecar = sidecar_of(lib.path(), receipt.asset_id); + let dimensions = sidecar + .dimensions + .as_ref() + .expect("dimensions from decoded pixels"); + assert_eq!((dimensions.width, dimensions.height), (320, 240)); + assert_eq!( + sidecar.content_type, "image/jpeg", + "the content type is header-derived" + ); + + let lqip = sidecar.lqip.as_ref().expect("the LQIP producer ran"); + assert_eq!(lqip.chromahash.len(), 32, "DEFAULT_TIER is 32 bytes"); + assert_eq!(lqip.format_version, crate::lqip::LQIP_FORMAT_V1); + assert!( + Lqip::from_bytes(&lqip.chromahash).is_ok(), + "the stored payload is a structurally valid chromahash" + ); + + assert!( + sidecar.verify(&ws.user_ik_public()), + "the sidecar signature covers the placeholder it now carries" + ); + } + + /// The sidecar's dimensions are the **upright** ones. A quarter-turned JPEG's stored width + /// is its EXIF `PixelXDimension`, transposed relative to what a viewer shows, so taking the + /// decoded pixels rather than the tag is the whole point. + #[test] + fn a_rotated_still_records_upright_dimensions() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (mut ws, album) = workspace(lib.path()); + + let receipt = import( + &mut ws, + album, + src.path(), + "portrait.jpg", + &jpeg(320, 240, Some(6)), + ); + assert_eq!(receipt.derivatives, DerivativeStatus::Decoded); + + let sidecar = sidecar_of(lib.path(), receipt.asset_id); + let dimensions = sidecar.dimensions.as_ref().expect("dimensions"); + assert_eq!( + (dimensions.width, dimensions.height), + (240, 320), + "orientation 6 is a quarter-turn, so the sidecar records the transposed pair" + ); + } + + /// Thumbnail bytes and a signed manifest bundle land on disk, at the layout the upload + /// bundle reader already looks for, and the bundle re-verifies against the bytes beside it. + #[test] + fn an_import_persists_thumbnail_bytes_and_a_verifying_manifest_bundle() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (mut ws, album) = workspace(lib.path()); + + let receipt = import(&mut ws, album, src.path(), "big.png", &png(512, 384)); + assert_eq!(receipt.derivatives, DerivativeStatus::Decoded); + + let dir = derivatives_dir(lib.path(), receipt.asset_id); + let stem = receipt.asset_id.simple().to_string(); + let thumb = dir.join(format!("{stem}.thumbnail.jxl")); + let bundle_path = dir.join(format!("{stem}.derivatives.cbor")); + assert!(thumb.is_file(), "thumbnail bytes at {}", thumb.display()); + assert!(bundle_path.is_file(), "a manifest bundle beside them"); + + let bytes = fs::read(&thumb).unwrap(); + let manifests: Vec = + cbor::from_slice(&fs::read(&bundle_path).unwrap()).expect("the bundle decodes"); + assert_eq!(manifests.len(), 1); + let core = &manifests[0].core; + assert_ne!( + core.ciphertext_hash, + hash::hash_bytes(&bytes), + "the manifest addresses the ciphertext, never the plaintext on disk" + ); + assert_ne!( + core.nonce_prefix, [0u8; 7], + "and it records the prefix that ciphertext was produced under" + ); + assert_eq!(core.source_asset_id, receipt.asset_id); + assert_eq!( + verify_still_format(&manifests[0]), + Ok(Some(DerivativeFormat::Jxl)), + "the persisted format is inside the closed set" + ); + + // The bytes really are a 256 px JXL. + let decoded = crate::media::RawshiftDecoder + .decode(&bytes, "jxl") + .expect("the persisted thumbnail decodes"); + assert_eq!((decoded.width(), decoded.height()), (256, 192)); + } + + /// A still already inside the tier cap gets the signed `original` sentinel: a manifest that + /// says `original` and content-addresses the source, and **no derivative bytes on disk**. + /// + /// The absent bytes are the point, and they are what "the tier *references* the original" + /// means. Writing a copy would put the original — EXIF and GPS intact — in + /// `derivatives/{uuid}.thumbnail.{ext}` and therefore into the derivative blob of the upload + /// bundle, which is the one place a re-encoded thumbnail is metadata-free by construction. + /// The signed marker is still there, so this stays distinct from an absent derivative, which + /// means "rebuild me". + #[test] + fn a_small_still_persists_the_original_sentinel_without_copying_it() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (mut ws, album) = workspace(lib.path()); + + let original = png(128, 96); + let receipt = import(&mut ws, album, src.path(), "small.png", &original); + assert_eq!(receipt.derivatives, DerivativeStatus::Decoded); + assert_eq!( + receipt.deferred_formats, 0, + "the sentinel satisfies the tier, so nothing was deferred" + ); + + let dir = derivatives_dir(lib.path(), receipt.asset_id); + let stem = receipt.asset_id.simple().to_string(); + + let files: Vec = fs::read_dir(&dir) + .unwrap() + .flatten() + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!( + files, + vec![format!("{stem}.derivatives.cbor")], + "the sentinel writes its manifest and no derivative bytes" + ); + + let manifests: Vec = + cbor::from_slice(&fs::read(dir.join(format!("{stem}.derivatives.cbor"))).unwrap()) + .expect("the bundle decodes"); + assert_eq!(manifests.len(), 1); + assert_eq!(manifests[0].core.format, "original"); + // The sentinel references the original *blob*: it signs the original manifest's own + // ciphertext address and nonce prefix, not the plaintext's. + let state = ws.asset(&receipt.asset_id).expect("the asset is held"); + let original_core = &state + .chain + .records() + .last() + .expect("a create record") + .manifest + .core; + assert_eq!( + manifests[0].core.ciphertext_hash, original_core.ciphertext_hash, + "the sentinel points at the blob a receiver already holds" + ); + assert_eq!( + manifests[0].core.nonce_prefix, original_core.nonce_prefix, + "under the same key the original was encrypted with" + ); + assert_ne!( + manifests[0].core.ciphertext_hash, + hash::hash_bytes(&original), + "which is not the plaintext's address" + ); + assert_eq!( + verify_still_format(&manifests[0]), + Ok(Some(DerivativeFormat::Original)), + "the sentinel is inside the closed set" + ); + } + + /// **Decision 22's degradation path, through the real import.** A codec that refuses a frame + /// the decoder accepted costs this asset its thumbnail and **nothing else**: the original + /// commits, signed and self-verifying, with real pixel dimensions and a real placeholder, + /// and the run reports `DecodeFailed` so somebody can look at it. + /// + /// Reverting `prepare_still`'s match arm to a bare `?` must fail this test — that is what it + /// is for. Before the review round the code did exactly that, and because the failure + /// happened *before* `write_asset_files`, an encoder refusal lost the original from the + /// backup outright. + #[test] + fn a_codec_refusal_costs_the_thumbnail_and_not_the_backup() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (mut ws, album) = workspace(lib.path()); + let path = src.path().join("photo.png"); + fs::write(&path, png(512, 384)).unwrap(); + + let receipt = super::with_sealer_fault(super::SealerFault::Refuse, || { + ws.import_asset_with(album, &path, &SignedImportOptions::default()) + .expect("a codec refusal must never fail the import") + }); + + assert_eq!( + receipt.derivatives, + DerivativeStatus::DecodeFailed, + "reported as a real problem rather than an expected gap" + ); + assert_eq!(receipt.deferred_formats, 0); + + // The half that must not be lost: a signed, encrypted, self-verifying original. + assert_eq!( + ws.verify(&receipt.asset_id).unwrap(), + crate::crypto::verify_asset::VerifyOutcome::Accept + ); + let sidecar = sidecar_of(lib.path(), receipt.asset_id); + let dimensions = sidecar.dimensions.as_ref().expect("real pixel dimensions"); + assert_eq!((dimensions.width, dimensions.height), (512, 384)); + assert_eq!( + sidecar.lqip.as_ref().map(|l| l.chromahash.len()), + Some(32), + "the placeholder came from the decode, which succeeded" + ); + assert!( + !derivatives_dir(lib.path(), receipt.asset_id).exists(), + "and no derivative was written" + ); + } + + /// **Decision 23's guard, through the real import.** A codec that *panics* on a frame the + /// decoder accepted is caught, and the import still commits. + /// + /// Driven through `import_asset_with` rather than by calling `guarded` directly: a test that + /// calls the guard itself stays green even if every production call site is deleted, which + /// is precisely the hole this replaces. Deleting the guard around generation makes this test + /// abort the process rather than fail — which is the failure mode it exists to prevent. + #[test] + fn a_codec_panic_is_caught_and_the_import_still_commits() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (mut ws, album) = workspace(lib.path()); + let path = src.path().join("photo.png"); + fs::write(&path, png(512, 384)).unwrap(); + + let previous = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let receipt = super::with_sealer_fault(super::SealerFault::Panic, || { + ws.import_asset_with(album, &path, &SignedImportOptions::default()) + .expect("a codec panic must never fail the import") + }); + std::panic::set_hook(previous); + + assert_eq!(receipt.derivatives, DerivativeStatus::DecodeFailed); + assert_eq!( + ws.verify(&receipt.asset_id).unwrap(), + crate::crypto::verify_asset::VerifyOutcome::Accept, + "one panicking photo does not cost the asset, let alone the rest of the run" + ); + let sidecar = sidecar_of(lib.path(), receipt.asset_id); + assert!(sidecar.lqip.is_some(), "the placeholder still landed"); + } + + /// A format with no codec here, and bytes that are no still at all: both import as signed, + /// verifiable originals, with EXIF-or-nothing dimensions, no placeholder, no derivative + /// files, and the reason recorded (slice `S-B13`). + #[test] + fn an_undecodable_original_still_imports_with_the_reason_recorded() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (mut ws, album) = workspace(lib.path()); + + // A real HEIC header (ISO-BMFF `ftyp heic`) with no payload: recognised, no codec. + let mut heic = vec![0, 0, 0, 0x20]; + heic.extend_from_slice(b"ftypheic"); + heic.extend_from_slice(&[0; 16]); + let deferred = import(&mut ws, album, src.path(), "shot.heic", &heic); + assert_eq!(deferred.derivatives, DerivativeStatus::DeferredNoCodec); + assert_eq!(deferred.deferred_formats, 0, "nothing was attempted"); + + let sidecar = sidecar_of(lib.path(), deferred.asset_id); + assert!(sidecar.lqip.is_none(), "no pixels, no placeholder"); + assert!( + sidecar.dimensions.is_none(), + "no pixels and no EXIF dimensions" + ); + assert_eq!( + sidecar.content_type, "image/heic", + "the header still names the format, codec or not" + ); + assert!( + !derivatives_dir(lib.path(), deferred.asset_id).exists(), + "no derivative directory is created for an asset with no derivatives" + ); + + // Not a still at all. + let video = import( + &mut ws, + album, + src.path(), + "clip.mp4", + b"\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom", + ); + assert_eq!(video.derivatives, DerivativeStatus::NotAKnownStill); + assert_eq!( + sidecar_of(lib.path(), video.asset_id).content_type, + "video/mp4", + "the extension table still types a video" + ); + + // Both are signed, encrypted, self-verifying backups regardless. + for id in [deferred.asset_id, video.asset_id] { + assert_eq!( + ws.verify(&id).unwrap(), + crate::crypto::verify_asset::VerifyOutcome::Accept + ); + } + } +} + +// ── The nonce-prefix reuse refusal (encryption.md, "Re-keying on Rewrite") ─── + +#[cfg(test)] +mod sealer_tests { + use super::*; + + /// A draw that hands back a fixed sequence, so a collision can be *forced* rather than + /// waited for — a real 7-byte collision is a 1-in-2^56 event. + struct ScriptedDraw { + prefixes: RefCell>, + } + + impl ScriptedDraw { + fn next(&self) -> [u8; NONCE_PREFIX_LEN] { + let mut queue = self.prefixes.borrow_mut(); + if queue.len() == 1 { + queue[0] + } else { + queue.remove(0) + } + } + } + + fn sealer<'a>( + amk: &'a Amk, + used: HashSet<[u8; NONCE_PREFIX_LEN]>, + draw: &'a dyn Fn() -> [u8; NONCE_PREFIX_LEN], + ) -> AlbumSealer<'a> { + AlbumSealer { + amk, + asset_id: Uuid::from_u128(0xDEF), + used: RefCell::new(used), + draw, + } + } + + /// **The normative refusal.** A draw that keeps returning a prefix already used for this + /// `file_id` is refused rather than accepted, and the refusal is a `Sign`-class fault so + /// decision 22 propagates it instead of silently writing one derivative fewer. + /// + /// A prefix is folded into the file-key salt, so reusing one reuses the **key**: two blobs + /// under one keystream, which is exactly what the encryption doc's "defense in depth on top + /// of the CSPRNG draw" exists to prevent. + #[test] + fn a_prefix_already_used_for_this_file_id_is_refused() { + let amk = Amk::from_bytes([0x11; 32]); + let original = [1, 2, 3, 4, 5, 6, 7]; + let mut used = HashSet::new(); + used.insert(original); + + // The RNG is forced to keep offering the original's prefix. + let scripted = ScriptedDraw { + prefixes: RefCell::new(vec![original]), + }; + let draw = || scripted.next(); + let error = sealer(&amk, used, &draw) + .seal(b"derivative plaintext") + .expect_err("a reused prefix is refused"); + assert!( + matches!(error, MediaError::Sign { .. }), + "an exhausted draw is a workspace fault, not a missing thumbnail: {error:?}" + ); + } + + /// A collision is **redrawn**, not fatal: the first candidate is taken, the second is used. + #[test] + fn a_collision_is_redrawn_and_the_next_candidate_is_accepted() { + let amk = Amk::from_bytes([0x22; 32]); + let taken = [9, 9, 9, 9, 9, 9, 9]; + let fresh = [8, 7, 6, 5, 4, 3, 2]; + let mut used = HashSet::new(); + used.insert(taken); + + let scripted = ScriptedDraw { + prefixes: RefCell::new(vec![taken, fresh]), + }; + let draw = || scripted.next(); + let sealed = sealer(&amk, used, &draw) + .seal(b"derivative plaintext") + .expect("the redraw succeeds"); + assert_eq!( + sealed.nonce_prefix, fresh, + "the colliding candidate is skipped and the next one is used" + ); + } + + /// Each sealed prefix **joins** the set, so two derivatives of one asset cannot collide with + /// each other either — not only with what was already on disk. + #[test] + fn a_freshly_sealed_prefix_is_spoken_for_by_the_next_seal() { + let amk = Amk::from_bytes([0x33; 32]); + let first = [1, 1, 1, 1, 1, 1, 1]; + let second = [2, 2, 2, 2, 2, 2, 2]; + + // The draw offers `first`, then `first` again (a collision with what was just sealed), + // then `second`. + let scripted = ScriptedDraw { + prefixes: RefCell::new(vec![first, first, second]), + }; + let draw = || scripted.next(); + let sealer = sealer(&amk, HashSet::new(), &draw); + + assert_eq!(sealer.seal(b"one").expect("first seal").nonce_prefix, first); + assert_eq!( + sealer.seal(b"two").expect("second seal").nonce_prefix, + second, + "the prefix the first seal used is refused for the second" + ); + } + + /// The bundle reader hands the sealer every prefix already spent on this `file_id`. + #[test] + fn existing_derivatives_reports_every_persisted_prefix() { + use crate::crypto::keys::HybridSigningKey; + use crate::crypto::provenance::manifest::{DERIVATIVE_MANIFEST_VERSION, DerivativeCore}; + + let dir = tempfile::tempdir().expect("scratch"); + let asset_id = Uuid::from_u128(0xFEED); + let device = HybridSigningKey::from_seed_bytes(&[31; 32], &[32; 32]); + let write = HybridSigningKey::from_seed_bytes(&[33; 32], &[34; 32]); + + let manifest = |role, prefix: [u8; NONCE_PREFIX_LEN]| { + DerivativeCore { + version: DERIVATIVE_MANIFEST_VERSION.into(), + crypto_suite_id: CRYPTO_SUITE_ID, + protocol_version: Some(PROTOCOL_VERSION.into()), + amk_version: Some(AmkVersion(1)), + source_asset_id: asset_id, + role, + format: "image/jxl".into(), + ciphertext_hash: hash::hash_bytes(b"bytes"), + nonce_prefix: prefix, + generated_by_device: Uuid::from_u128(0xD1), + generated_by_client: "capsule-core/test".into(), + model_id: None, + model_version: None, + generated_at: "2026-09-02T00:00:00Z".into(), + prior_provenance_hash: None, + } + .sign(&device, &write) + .expect("signing") + }; + let manifests = vec![ + manifest(DerivativeRole::Thumbnail, [1, 1, 1, 1, 1, 1, 1]), + manifest(DerivativeRole::Preview, [2, 2, 2, 2, 2, 2, 2]), + ]; + fs::write( + dir.path() + .join(format!("{}.derivatives.cbor", asset_id.simple())), + cbor::to_canonical_vec(&manifests).unwrap(), + ) + .unwrap(); + + let existing = existing_derivatives(dir.path(), asset_id); + assert!(existing.used_prefixes.contains(&[1, 1, 1, 1, 1, 1, 1])); + assert!(existing.used_prefixes.contains(&[2, 2, 2, 2, 2, 2, 2])); + assert_eq!(existing.used_prefixes.len(), 2); + assert_eq!(existing.heads.len(), 2, "and both roles have a chain head"); + } +} diff --git a/capsule-core/src/lifecycle/import.rs b/capsule-core/src/lifecycle/import.rs index 216a95ed..58aee402 100644 --- a/capsule-core/src/lifecycle/import.rs +++ b/capsule-core/src/lifecycle/import.rs @@ -8,6 +8,7 @@ use std::path::Path; use jiff::Timestamp; use uuid::Uuid; +use super::derivatives::{PreparedStill, StillSource}; use super::{ AssetState, LifecycleError, Result, SidecarEnrichment, SignedImport, SignedImportOptions, StackPlacement, StreamedImport, Workspace, asset_is_deleted, media_dir, now_rfc3339, @@ -28,7 +29,7 @@ use crate::exif::extract::extract_exif; use crate::exif::timezone::resolve_timezone; use crate::metadata::crdt::{Lww, OrSet}; use crate::sidecar::sidecar_v1::{ - Dimensions, Gps, GpsSource, SIDECAR_SCHEMA_V1, SidecarV1, StackMembership, StackRole, + Gps, GpsSource, SIDECAR_SCHEMA_V1, SidecarV1, StackMembership, StackRole, }; /// Render a Unix-second capture time as the sidecar's RFC 3339 `capture_timestamp`. @@ -97,13 +98,19 @@ fn folded_gps(embedded: Option, folded: Option<&Gps>) -> Option { embedded.or_else(|| folded.cloned()) } +/// The sidecar `content_type` for a file whose bytes named no still image Capsule models. +/// +/// The **fallback only**: [`Workspace::prepare_still`] sniffs the header first, so a still's +/// media type comes from [`StillFormat::mime`](crate::media::StillFormat::mime) and a `.jpg` +/// that is really a HEIC is typed `image/heic`. What is left for this table is the non-still +/// suffixes — video above all, which has no detection path until slice `S-B5`. fn content_type_for(ext: &str) -> String { match ext { - "jpg" | "jpeg" => "image/jpeg", - "png" => "image/png", - "heic" => "image/heic", - "webp" => "image/webp", - "mp4" => "video/mp4", + "mp4" | "m4v" => "video/mp4", + "mov" | "qt" => "video/quicktime", + "mkv" => "video/x-matroska", + "webm" => "video/webm", + "avi" => "video/x-msvideo", _ => "application/octet-stream", } .to_string() @@ -303,12 +310,13 @@ impl Workspace { /// As [`import_asset`](Self::import_asset) but with executor-supplied [`SignedImportOptions`] /// (Move-mode source release + stack placement). This is the single signed write path the /// import executor drives (S-B2): every imported member lands as a signed `SidecarV1` + - /// manifest + append-only provenance, self-verified through [`verify_asset`], and — when a - /// still encoder is attached — with signed thumbnail/preview derivatives + an LQIP in the - /// sidecar. No still encoder exists in this build; the media stack is retired (`S-B1`). + /// manifest + append-only provenance, self-verified through [`verify_asset`], and — when the + /// still decodes — with a chromahash `lqip` in the sidecar and signed thumbnail derivatives + /// on disk (the private `prepare_still`, slices `S-B1`/`S-B14`). /// - /// Returns a [`SignedImport`]: the asset id, plus the [`DerivativeStatus`](super::DerivativeStatus) - /// saying whether derivatives were generated and, if not, why. A format this build has no + /// Returns a [`SignedImport`]: the asset id, the + /// [`DerivativeStatus`](super::DerivativeStatus) saying whether derivatives were generated + /// and, if not, why, and the per-format deferral count. A format this build has no /// codec for **still imports** — the original is the backup, the thumbnail is a bonus — so /// the status is a report, never a rejection (slice `S-B13`). #[tracing::instrument(skip_all, fields(album_id = %album_id, src = %src.display()))] @@ -382,27 +390,48 @@ impl Workspace { "import: sidecar metadata resolved" ); - // Still-derived sidecar metadata. Dimensions come from EXIF; there is **no decoder in - // this build** since `S-C59` retired `capsule_core::media`, so no still is decoded, no - // LQIP is computed and no derivatives are generated. The import proceeds regardless: the - // original is still backed up as a signed, encrypted blob, and `derivative_status` - // records the gap so it is reportable rather than silent (`S-B13`). Rawshift's - // replacement is what closes it. - let (dimensions, lqip, derivative_status) = ( - exif.width - .zip(exif.height) - .map(|(width, height)| Dimensions { width, height }), - None::, - super::DerivativeStatus::DeferredNoCodec, - ); - let album = self.album(&album_id)?; let epoch = album.current_epoch; let amk = Amk::from_bytes(album.amks[&epoch]); // First write: draw a fresh nonce prefix and derive the folded file key together // (nothing to replace on a create). + // + // **Before** the derivatives, and that ordering is load-bearing: the `original` + // sentinel is a signed *reference* to this blob, so it commits to this ciphertext's + // address and this nonce prefix, neither of which exists until now. let (enc, ciphertext, _file_key) = encrypt_asset_rekey(&amk, &asset_id, &plaintext, None)?; + // Still-derived sidecar metadata, from one decode pass over the plaintext: the + // header-derived `content_type`, pixel `dimensions`, the chromahash `lqip`, and the + // signed thumbnail derivatives to persist once the asset's own files are durable. Each + // generated derivative is encrypted under its own fresh nonce prefix as it is signed — + // derivative bytes cross the network encrypted exactly like the original. + // + // Never fatal. A still this build cannot decode — or cannot decode *these bytes* of — + // commits exactly as before: EXIF dimensions, no LQIP, no derivatives, and a + // `DerivativeStatus` recording which reason applied so the gap is reportable rather + // than silent (`S-B13`). + let PreparedStill { + format, + dimensions, + lqip, + derivatives, + deferred_formats, + status: derivative_status, + } = self.prepare_still( + &StillSource { + plaintext: &plaintext, + ext: &ext, + src, + exif: &exif, + }, + asset_id, + album_id, + capture_utc, + &amk, + &enc, + )?; + // Sealing order (1) the prior head `H` is `None` on a create; (2) author + sign the // sidecar with `provenance_chain_hash = H`. let mut sidecar = SidecarV1 { @@ -412,7 +441,9 @@ impl Workspace { hash: hash::hash_bytes(&plaintext), capture_timestamp: capture_rfc3339(capture_utc), import_timestamp: now_rfc3339(), - content_type: content_type_for(&ext), + // Header-derived wherever the bytes name a still Capsule models; the extension + // table is the fallback for everything else (video, unknown suffixes). + content_type: format.map_or_else(|| content_type_for(&ext), |f| f.mime().to_string()), dimensions, lqip, tags_user, @@ -512,6 +543,10 @@ impl Workspace { stack: opts.stack.as_ref().map(StackPlacement::from_membership), }; self.write_asset_files(&asset, &plaintext)?; + // After the asset's own files, and deliberately: a derivative is regenerable, so a + // failure to write one must never fail an import whose signed original is already + // durable. `persist_derivatives` logs and continues rather than returning. + self.persist_derivatives(&asset, &derivatives); self.index_asset_row(&asset)?; self.index_original_representation(&asset, plaintext.len())?; @@ -526,6 +561,7 @@ impl Workspace { Ok(SignedImport { asset_id, derivatives: derivative_status, + deferred_formats: deferred_formats as u32, }) } diff --git a/capsule-core/src/lifecycle/mod.rs b/capsule-core/src/lifecycle/mod.rs index 4b3d7da9..981bc534 100644 --- a/capsule-core/src/lifecycle/mod.rs +++ b/capsule-core/src/lifecycle/mod.rs @@ -26,6 +26,7 @@ mod album; mod backup; +mod derivatives; mod drops; mod groups; mod import; @@ -295,26 +296,34 @@ pub struct SignedImportOptions { /// is a real problem someone should look at. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum DerivativeStatus { - /// The still decoded: dimensions and LQIP came from real pixels, and signed derivatives - /// were generated if a still encoder is attached to the workspace. + /// The still decoded: `dimensions` and `lqip` came from real pixels, and the derivatives + /// this build can encode were generated and signed. **Independent of how many *formats* + /// deferred** — a decoded still whose AVIF and WebP variants have no encoder here is still + /// `Decoded`, because it has a renderable JXL thumbnail. The per-format gap is counted + /// separately by + /// [`ImportExecutionSummary::deferred_format_count`](crate::import::ImportExecutionSummary::deferred_format_count). Decoded, - /// **Expected deferral.** This build links no codec for the asset's format — the - /// supported-image-format table lives in the retired media stack. The original is safely - /// backed up; dimensions fall back to EXIF and there is no LQIP or preview until the codec - /// lands, at which point derivatives can be backfilled from the stored original. Counted by + /// **Expected deferral.** This build links no codec for the asset's format — see + /// [`SUPPORTED_STILL_FORMATS`](crate::media::SUPPORTED_STILL_FORMATS) for what it does + /// link, and [`StillFormat`](crate::media::StillFormat) for what it recognises. The + /// original is safely backed up; dimensions fall back to EXIF and there is no LQIP or + /// thumbnail until the codec lands, at which point derivatives can be backfilled from the + /// stored original. Counted by /// [`ImportExecutionSummary::deferred_derivative_count`](crate::import::ImportExecutionSummary::deferred_derivative_count). /// - /// This build links no codecs at all — the media stack is retired to `legacy-review/` - /// (`S-B1`) — so every still it imports reports this. + /// What reaches here today: HEIC, AVIF and the RAW families, each needing a system library + /// (libheif, libdav1d) or an assembler the cross and cargo-ndk builds do not carry. DeferredNoCodec, /// **A real problem.** The format *is* one this build can decode, but these particular /// bytes did not decode — truncation, corruption, or a decoder bug. The original is still /// imported (the bytes are backed up verbatim, whatever they are), but this is worth /// investigating rather than shrugging at. DecodeFailed, - /// Nothing to decode: the extension names no still image this build models — a video, an - /// XMP sidecar, an unknown suffix, or an exotic RAW flavour the raw-image-format table has - /// no variant for. Video derivatives are generated on their own path. + /// Nothing to decode: neither the bytes' header nor the extension names a still image + /// Capsule models — a video, an XMP sidecar, an SVG, or an unknown suffix. Distinct from + /// [`DeferredNoCodec`](Self::DeferredNoCodec), which is a still whose codec is merely + /// absent and whose derivatives are therefore backfillable. Video derivatives are generated + /// on their own path (slice `S-B5`). NotAKnownStill, } @@ -339,6 +348,11 @@ pub struct SignedImport { pub asset_id: Uuid, /// Whether thumbnail/preview derivatives were generated, and if not, why. pub derivatives: DerivativeStatus, + /// How many `(tier, format)` pairs the tier table commits to and this build cannot encode + /// — the per-format half of the `S-B13` gap, which is orthogonal to + /// [`derivatives`](Self::derivatives): a `Decoded` asset can still carry deferred formats. + /// Zero when the still did not decode at all, because nothing was attempted. + pub deferred_formats: u32, } /// A streamed import: everything the [streaming window](crate::import::execute_streaming) needs diff --git a/capsule-core/src/lifecycle/upload.rs b/capsule-core/src/lifecycle/upload.rs index f724991f..0ae3da29 100644 --- a/capsule-core/src/lifecycle/upload.rs +++ b/capsule-core/src/lifecycle/upload.rs @@ -18,16 +18,17 @@ use std::fs; use uuid::Uuid; -use super::{AssetState, LifecycleError, Result, Workspace, media_dir}; +use super::{AlbumKeys, AssetState, LifecycleError, Result, Workspace, media_dir}; use crate::cbor; use crate::crypto::encryption::stream; use crate::crypto::hash::{self, Hash32}; use crate::crypto::provenance::DerivativeManifest; use crate::crypto::provenance::action::{Action, DerivativeRole}; use crate::crypto::provenance::manifest::KeyMode; +use crate::media::{DerivativeFormat, verify_still_format}; -/// One derivative blob of an asset bundle: the bytes plus the content address its signed -/// [`DerivativeManifest`] committed to. +/// One derivative blob of an asset bundle: the **ciphertext** plus the content address its +/// signed [`DerivativeManifest`] committed to. #[derive(Debug, Clone)] pub struct DerivativeBlob { /// Which derivative this is (`thumbnail` / `preview` / `embedding`). @@ -36,7 +37,10 @@ pub struct DerivativeBlob { pub format: String, /// The AMK epoch the derivative manifest was authorized under, when it recorded one. pub amk_version: Option, - /// The derivative's transferable bytes. + /// The derivative's transferable bytes — **ciphertext**, re-derived from the plaintext the + /// library holds using the nonce prefix the manifest recorded. Derivative bytes are + /// encrypted client-side exactly like the original + /// ([Encryption](https://docs/design/cryptography/encryption/)). pub bytes: Vec, /// The content address the signed derivative manifest committed to. pub ciphertext_hash: Hash32, @@ -147,7 +151,7 @@ impl Workspace { return Err(LifecycleError::CiphertextMismatch(asset.asset_id)); } - let derivatives = self.derivative_blobs(asset); + let derivatives = self.derivative_blobs(asset, album, epoch); tracing::debug!( album_id = %asset.album_id, amk_version = epoch, @@ -184,11 +188,40 @@ impl Workspace { } /// The asset's persisted derivative blobs, read back from - /// `media/{YYYY}/{YYYY-MM}/derivatives/`. A derivative whose bytes are missing or no - /// longer content-address to its signed manifest is **skipped with a warning** rather - /// than failing the bundle: the original and its metadata are what a backup must not + /// `media/{YYYY}/{YYYY-MM}/derivatives/` and **encrypted** for the wire. + /// + /// The library holds derivatives as plaintext, exactly as it holds the original: the local + /// gallery paints them, and the ciphertext is re-derived here from the nonce prefix the + /// signed manifest recorded — the same round trip + /// [`upload_bundle`](Self::upload_bundle) performs for the original. So what leaves this + /// function in [`DerivativeBlob::bytes`] is ciphertext, and the field's `ciphertext_hash` + /// name is now true of it. A thumbnail is a recognisable low-resolution copy of a private + /// photo; the encryption doc admits no exception for it. + /// + /// **Still roles only.** An embedding-role manifest is skipped at `debug!` and named as out + /// of scope: its `embedding/{model_id}` grammar is not in the still format set, and + /// `crate::ml` produces no derivative manifest for this reader to have an opinion about. + /// + /// Five reasons a manifest is skipped rather than shipped, and two of them are quiet: + /// + /// - the `original` sentinel, which references the original blob and has no bytes of its + /// own — an **expected** absence, logged at `debug!`; + /// - an embedding-role manifest, out of scope as above — also `debug!`; + /// - a still-role `format` outside the closed set, which is the structural rejection the + /// tier table specifies; + /// - an `amk_version` naming an epoch this album does not hold, which would otherwise panic + /// on the key lookup; + /// - bytes missing on disk, or bytes whose re-derived ciphertext does not match the signed + /// content address. + /// + /// None of them fails the bundle: the original and its metadata are what a backup must not /// lose, and a stale thumbnail is regenerable. - fn derivative_blobs(&self, asset: &AssetState) -> Vec { + fn derivative_blobs( + &self, + asset: &AssetState, + album: &AlbumKeys, + epoch: u32, + ) -> Vec { let dir = media_dir(&self.root, asset.capture_utc).join("derivatives"); let stem = asset.asset_id.simple().to_string(); let bundle_path = dir.join(format!("{stem}.derivatives.cbor")); @@ -209,10 +242,54 @@ impl Workspace { let mut blobs = Vec::with_capacity(manifests.len()); for manifest in manifests { + let role_name = derivative_role_name(manifest.core.role); + + // The closed-format check the tier table calls a structural rejection. It answers + // `Ok(None)` for an embedding-role manifest, whose `embedding/{model_id}` grammar + // this set deliberately does not model, so those pass through untouched. + match verify_still_format(&manifest) { + Ok(Some(DerivativeFormat::Original)) => { + // A reference to the original blob, not a missing file: the receiver + // resolves it against the original it already has. `debug!`, because an + // expected absence logged as a warning trains people to ignore warnings. + tracing::debug!( + asset_id = %asset.asset_id, + role = role_name, + "upload bundle: `original` sentinel references the original blob; \ + nothing to upload for this tier" + ); + continue; + } + Ok(None) => { + // An embedding-role manifest. Its `embedding/{model_id}` grammar is not in + // the still format set, and nothing produces one yet: `crate::ml` writes no + // derivative manifest at all. Rather than invent a reader for an artefact + // with no writer, this reader says plainly that embeddings are out of its + // scope — at `debug!`, because encountering one is not a fault. + tracing::debug!( + asset_id = %asset.asset_id, + role = role_name, + "upload bundle: embedding-role derivatives are outside this reader's \ + scope until `crate::ml` produces manifests for them; skipping" + ); + continue; + } + Ok(Some(_)) => {} + Err(format) => { + tracing::warn!( + asset_id = %asset.asset_id, + role = role_name, + %format, + "upload bundle: still-role derivative names a format outside the closed \ + set; skipping" + ); + continue; + } + } + let core = manifest.core; - let role_name = derivative_role_name(core.role); - let prefix = format!("{stem}.{role_name}."); - let Some(bytes) = read_derivative_bytes(&dir, &prefix) else { + let Some(plaintext) = read_derivative_bytes(&dir, &stem, role_name, &core.format) + else { tracing::warn!( asset_id = %asset.asset_id, role = role_name, @@ -220,7 +297,32 @@ impl Workspace { ); continue; }; - let observed = hash::hash_bytes(&bytes); + + // Re-derive the ciphertext from the prefix the manifest signed. The prefix is + // folded into the file-key salt, so it selects the key as well as the nonces — + // there is exactly one ciphertext this manifest can be describing. + // + // The epoch is **checked**, not indexed. It comes off an unverified `.cbor` on + // disk, and `file_key` reaches `album.amks[&epoch]`, which panics on a missing key + // — inside a function whose whole contract is that it never fails the bundle. An + // album recovered from a backup holds only the epochs it escrowed, so a manifest + // naming one it does not hold is reachable without any tampering at all. + let key_epoch = core.amk_version.map_or(epoch, |v| v.0); + if !album.amks.contains_key(&key_epoch) { + tracing::warn!( + asset_id = %asset.asset_id, + role = role_name, + epoch = key_epoch, + "upload bundle: derivative manifest names an AMK epoch this album does not \ + hold; skipping" + ); + continue; + } + let file_key = self.file_key(album, key_epoch, &asset.asset_id, &core.nonce_prefix); + let (_, ciphertext) = + stream::encrypt_asset_vec_with_prefix(&file_key, core.nonce_prefix, &plaintext); + + let observed = hash::hash_bytes(&ciphertext); if observed != core.ciphertext_hash { tracing::warn!( asset_id = %asset.asset_id, @@ -233,7 +335,7 @@ impl Workspace { role: core.role, format: core.format, amk_version: core.amk_version.map(|v| v.0), - bytes, + bytes: ciphertext, ciphertext_hash: observed, }); } @@ -250,17 +352,28 @@ fn derivative_role_name(role: DerivativeRole) -> &'static str { } } -/// The first file in `dir` whose name starts with `prefix` — the derivative's bytes, whose -/// extension varies with the encoder's chosen format. -fn read_derivative_bytes(dir: &std::path::Path, prefix: &str) -> Option> { - let entries = fs::read_dir(dir).ok()?; - let mut names: Vec<_> = entries - .filter_map(std::result::Result::ok) - .map(|e| e.file_name().to_string_lossy().into_owned()) - .filter(|name| name.starts_with(prefix)) - .collect(); - names.sort(); - fs::read(dir.join(names.first()?)).ok() +/// The persisted bytes of one derivative, addressed by **(role, format)** rather than by role +/// alone. +/// +/// Role alone was ambiguous the moment a tier could carry more than one format: with a JXL and +/// an AVIF thumbnail side by side, a prefix match would take whichever sorted first and then +/// content-address it against the *other* manifest, so both would be skipped as mismatched. +/// `#437` lands exactly that pair, so this is a latent break rather than a hypothetical one. +/// +/// Returns `None` for any `format` outside the closed set — including the embedding-role +/// grammar, which has no still extension. The caller has already skipped both cases, so reaching +/// this with one is not expected; it answers `None` rather than asserting, because a reader that +/// panics on a manifest it merely does not understand is worse than one that ships nothing. A stale file left by a +/// retired format is simply never read — nothing enumerates the directory any more, so an +/// orphan is inert rather than a candidate, and it is regenerable by design. +fn read_derivative_bytes( + dir: &std::path::Path, + stem: &str, + role_name: &str, + format: &str, +) -> Option> { + let extension = DerivativeFormat::parse(format)?.extension()?; + fs::read(dir.join(format!("{stem}.{role_name}.{extension}"))).ok() } #[cfg(test)] diff --git a/capsule-core/src/lifecycle/upload/tests.rs b/capsule-core/src/lifecycle/upload/tests.rs index b2fad48a..59cc9200 100644 --- a/capsule-core/src/lifecycle/upload/tests.rs +++ b/capsule-core/src/lifecycle/upload/tests.rs @@ -173,3 +173,527 @@ fn walk_find(root: &std::path::Path, name: &str) -> bool { .filter_map(std::result::Result::ok) .any(|e| e.file_name().to_string_lossy() == name) } + +// ── Derivative blobs cross the network encrypted (S-B1 / encryption.md) ────── + +/// A library with one **decodable** asset large enough to earn a real derivative, so the +/// derivative path is exercised rather than the `original` sentinel. +/// +/// A 512x384 PNG, built here rather than committed: the repository carries no binary fixtures. +fn library_with_a_thumbnailed_asset(lib: &TempDir, src: &TempDir) -> (Uuid, Uuid) { + use rawshift_image::core::metadata::ImageMetadata; + use rawshift_image::core::{BitDepth, MetadataEmbedOptions}; + use rawshift_image::formats::encode_rgb_image_to_vec; + use rawshift_image::formats::export::{ + CommonEncodeOptions, EncodeOptions, ZunePngEncodeConfig, + }; + + let (w, h) = (512u32, 384u32); + let mut data = Vec::with_capacity((w * h * 3) as usize); + for y in 0..h { + for x in 0..w { + data.push(((x * 255 / w) as u16) * 257); + data.push(((y * 255 / h) as u16) * 257); + data.push((((x + y) * 255 / (w + h)) as u16) * 257); + } + } + let frame = rawshift_image::core::image::RgbImage::with_color_space( + w, + h, + data, + rawshift_image::core::ColorSpace::Srgb, + ); + let png = encode_rgb_image_to_vec( + &frame, + &ImageMetadata::default(), + &EncodeOptions::PngZune(ZunePngEncodeConfig { + common: CommonEncodeOptions { + metadata: MetadataEmbedOptions::none(), + bit_depth: BitDepth::Eight, + }, + ..ZunePngEncodeConfig::default() + }), + ) + .expect("the fixture PNG encodes"); + + let img = src.path().join("photo.png"); + fs::write(&img, &png).unwrap(); + let mut ws = Workspace::create_with_params(lib.path(), b"passphrase", FAST).unwrap(); + let album = ws.default_album_id(); + ws.ensure_album(album, "Imports").unwrap(); + let asset = ws.import_asset(album, &img).unwrap(); + (album, asset) +} + +/// The derivative round trip, end to end: the plaintext the library holds is **not** what the +/// bundle ships, the bundle's bytes content-address to the signed `ciphertext_hash`, and +/// decrypting them with the manifest's recorded `nonce_prefix` yields the plaintext back. +/// +/// This is the property the encryption doc states without qualification — "every asset — +/// original bytes, derivative bytes, metadata blob — is encrypted client-side" — and a thumbnail +/// is a recognisable low-resolution copy of a private photo, so shipping one in the clear would +/// hand the server the picture it is not allowed to see. +#[test] +fn derivative_blobs_ship_ciphertext_that_decrypts_to_the_bytes_on_disk() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + let bundle = ws.upload_bundle(&asset_id).unwrap(); + assert_eq!(bundle.derivatives.len(), 1, "one encodable format today"); + let blob = &bundle.derivatives[0]; + assert_eq!(blob.format, "image/jxl"); + + // The plaintext the local gallery paints, straight off disk. + let asset = ws.asset(&asset_id).expect("the asset is held"); + let dir = media_dir(lib.path(), asset.capture_utc).join("derivatives"); + let stem = asset_id.simple().to_string(); + let plaintext = fs::read(dir.join(format!("{stem}.thumbnail.jxl"))).unwrap(); + assert_eq!( + &plaintext[..2], + b"\xFF\x0A", + "the on-disk derivative is a bare JXL codestream" + ); + + assert_ne!( + blob.bytes, plaintext, + "what crosses the network is not what sits on disk" + ); + assert_eq!( + hash::hash_bytes(&blob.bytes), + blob.ciphertext_hash, + "the blob's declared address is its own content address" + ); + + // And that address is the one the *signed* manifest committed to. + let bundle_path = dir.join(format!("{stem}.derivatives.cbor")); + let manifests: Vec = + cbor::from_slice(&fs::read(&bundle_path).unwrap()).expect("the bundle decodes"); + let core = &manifests[0].core; + assert_eq!(core.ciphertext_hash, blob.ciphertext_hash); + + // The receiver's half: the recorded prefix selects the key and the nonces. + let album_keys = ws.album(&asset.album_id).unwrap(); + let file_key = ws.file_key( + album_keys, + core.amk_version.unwrap().0, + &asset_id, + &core.nonce_prefix, + ); + let recovered = + stream::decrypt_asset_vec(&file_key, &core.nonce_prefix, &blob.bytes).expect("it opens"); + assert_eq!( + recovered, plaintext, + "and it decrypts to exactly the derivative the client holds" + ); +} + +/// A derivative whose on-disk bytes have been altered no longer re-derives to the address its +/// manifest signed, so it is skipped rather than shipped. The bundle still carries the original +/// and its metadata — a stale thumbnail is regenerable, a missing backup is not. +#[test] +fn a_tampered_derivative_is_skipped_rather_than_shipped() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + let asset = ws.asset(&asset_id).expect("the asset is held"); + let dir = media_dir(lib.path(), asset.capture_utc).join("derivatives"); + let path = dir.join(format!("{}.thumbnail.jxl", asset_id.simple())); + + let mut bytes = fs::read(&path).unwrap(); + let last = bytes.len() - 1; + bytes[last] ^= 0x01; + fs::write(&path, &bytes).unwrap(); + + let bundle = ws.upload_bundle(&asset_id).unwrap(); + assert!( + bundle.derivatives.is_empty(), + "a derivative that does not match its signed manifest is not uploaded" + ); + assert!( + !bundle.ciphertext.is_empty(), + "and the original is still shipped — the backup is what must not be lost" + ); +} + +/// The `original` sentinel carries no bytes **by design**, so the bundle simply has no +/// derivative blob for that tier — and the skip is not a warning, because an expected absence +/// logged as a problem is how people learn to ignore warnings. +#[test] +fn the_original_sentinel_contributes_no_blob_and_is_not_an_error() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + // The 8x8 JPEG fixture is far inside the 256 px thumbnail cap, so its tier is the sentinel. + let (_album, asset_id) = library_with_one_asset(&lib, &src); + + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + let bundle = ws.upload_bundle(&asset_id).unwrap(); + assert!( + bundle.derivatives.is_empty(), + "the sentinel references the original blob; there is nothing extra to upload" + ); +} + +/// Rewrite an asset's derivative bundle with `manifests`, returning the directory it lives in. +fn rewrite_bundle(lib: &TempDir, ws: &Workspace, asset_id: Uuid, manifests: &[DerivativeManifest]) { + let asset = ws.asset(&asset_id).expect("the asset is held"); + let dir = media_dir(lib.path(), asset.capture_utc).join("derivatives"); + fs::create_dir_all(&dir).unwrap(); + fs::write( + dir.join(format!("{}.derivatives.cbor", asset_id.simple())), + cbor::to_canonical_vec(&manifests.to_vec()).unwrap(), + ) + .unwrap(); +} + +/// Sign a derivative manifest with the given role and wire `format`, over `ciphertext_hash`. +fn signed_derivative( + asset_id: Uuid, + role: DerivativeRole, + format: &str, + ciphertext_hash: crate::crypto::hash::Hash32, +) -> DerivativeManifest { + signed_derivative_at_epoch(asset_id, role, format, ciphertext_hash, 1) +} + +/// As [`signed_derivative`], with an explicit `amk_version` — so a manifest can name an epoch +/// the album does not hold. +fn signed_derivative_at_epoch( + asset_id: Uuid, + role: DerivativeRole, + format: &str, + ciphertext_hash: crate::crypto::hash::Hash32, + epoch: u32, +) -> DerivativeManifest { + use crate::crypto::keys::{AmkVersion, HybridSigningKey}; + use crate::crypto::primitives::{CRYPTO_SUITE_ID, PROTOCOL_VERSION}; + use crate::crypto::provenance::manifest::{DERIVATIVE_MANIFEST_VERSION, DerivativeCore}; + + let device = HybridSigningKey::from_seed_bytes(&[21; 32], &[22; 32]); + let write = HybridSigningKey::from_seed_bytes(&[23; 32], &[24; 32]); + DerivativeCore { + version: DERIVATIVE_MANIFEST_VERSION.into(), + crypto_suite_id: CRYPTO_SUITE_ID, + protocol_version: Some(PROTOCOL_VERSION.into()), + amk_version: Some(AmkVersion(epoch)), + source_asset_id: asset_id, + role, + format: format.into(), + ciphertext_hash, + nonce_prefix: [7, 6, 5, 4, 3, 2, 1], + generated_by_device: Uuid::from_u128(0xD1), + generated_by_client: "capsule-core/test".into(), + model_id: None, + model_version: None, + generated_at: "2026-09-02T00:00:00Z".into(), + prior_provenance_hash: None, + } + .sign(&device, &write) + .expect("signing") +} + +/// **The closed-format rule, at the boundary that ships bytes.** A still-role manifest naming a +/// format outside the committed set is a structural rejection, so its bytes never reach the +/// network — even though they are sitting on disk and hash correctly. +/// +/// The embedding role is deliberately exempt: it writes `embedding/{model_id}` into the same +/// field, a grammar this set does not model, so it must not be caught in the crossfire. +#[test] +fn a_still_role_derivative_outside_the_closed_set_is_skipped() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + + // The bytes on disk stay exactly as the import wrote them; only the manifest's format moves. + let asset = ws.asset(&asset_id).expect("held"); + let dir = media_dir(lib.path(), asset.capture_utc).join("derivatives"); + let plaintext = fs::read(dir.join(format!("{}.thumbnail.jxl", asset_id.simple()))).unwrap(); + let album_keys = ws.album(&asset.album_id).unwrap(); + let file_key = ws.file_key(album_keys, 1, &asset_id, &[7, 6, 5, 4, 3, 2, 1]); + let (_, ciphertext) = + stream::encrypt_asset_vec_with_prefix(&file_key, [7, 6, 5, 4, 3, 2, 1], &plaintext); + let address = hash::hash_bytes(&ciphertext); + + // A recognised format ships... + rewrite_bundle( + &lib, + &ws, + asset_id, + &[signed_derivative( + asset_id, + DerivativeRole::Thumbnail, + "image/jxl", + address, + )], + ); + assert_eq!( + ws.upload_bundle(&asset_id).unwrap().derivatives.len(), + 1, + "a format inside the closed set is uploaded" + ); + + // ...and an unrecognised one does not, with everything else held equal. + rewrite_bundle( + &lib, + &ws, + asset_id, + &[signed_derivative( + asset_id, + DerivativeRole::Thumbnail, + "image/future-codec", + address, + )], + ); + assert!( + ws.upload_bundle(&asset_id).unwrap().derivatives.is_empty(), + "an unrecognised still format is a structural rejection, not a blob" + ); +} + +/// A manifest with a **recognised** format and no bytes on disk is a genuine problem and is +/// skipped, which is what keeps the sentinel's quiet skip from being a blanket amnesty for +/// missing files. +#[test] +fn a_non_sentinel_manifest_with_no_bytes_is_still_skipped() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + + let asset = ws.asset(&asset_id).expect("held"); + let dir = media_dir(lib.path(), asset.capture_utc).join("derivatives"); + fs::remove_file(dir.join(format!("{}.thumbnail.jxl", asset_id.simple()))).unwrap(); + + assert!( + ws.upload_bundle(&asset_id).unwrap().derivatives.is_empty(), + "a thumbnail manifest whose bytes have gone is not shipped" + ); +} + +/// The `capsule import` acceptance case, at the bundle boundary: the bytes a push would send for +/// the thumbnail tier are **not** the bytes on disk, and their magic differs — the on-disk file +/// is a bare JXL codestream (`FF 0A`), the wire blob is STREAM ciphertext. +/// +/// The magic check is the cheap, legible version of the round trip above: it is what someone +/// eyeballing a packet capture would look for. +#[test] +fn the_pushed_thumbnail_is_not_the_jxl_on_disk() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + let asset = ws.asset(&asset_id).expect("held"); + let dir = media_dir(lib.path(), asset.capture_utc).join("derivatives"); + let disk = fs::read(dir.join(format!("{}.thumbnail.jxl", asset_id.simple()))).unwrap(); + assert_eq!(&disk[..2], b"\xFF\x0A", "on disk: a bare JXL codestream"); + + let mut bundle = ws.upload_bundle(&asset_id).unwrap(); + let blob = bundle.derivatives.remove(0); + assert_ne!( + &blob.bytes[..2], + b"\xFF\x0A", + "on the wire: ciphertext, so it does not begin with the JXL magic" + ); + assert_ne!(blob.bytes, disk); +} + +/// **The `F1` contract at the import boundary.** A derivative that cannot be produced must never +/// cost the original: the asset still lands signed, encrypted and `verify_asset`-accepting, and +/// the run reports `DecodeFailed` rather than failing. +/// +/// Exercised through a *decodable* still whose derivative directory is then made unwritable, so +/// the failure happens after a successful decode — the exact shape that used to propagate as +/// `LifecycleError::Io` and lose the import. +#[test] +fn a_derivative_that_cannot_be_persisted_never_costs_the_original() { + use crate::crypto::verify_asset::VerifyOutcome; + + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + + // The asset committed, derivatives or not. + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + assert_eq!(ws.verify(&asset_id).unwrap(), VerifyOutcome::Accept); + + // And the original's own bytes are on disk and re-derive to their signed address, which is + // the property that makes this a backup rather than a thumbnail service. + let bundle = ws.upload_bundle(&asset_id).unwrap(); + assert!(!bundle.ciphertext.is_empty()); + assert_eq!(hash::hash_bytes(&bundle.ciphertext), bundle.ciphertext_hash); +} + +/// A persisted derivative survives a `Workspace` reopen and still reaches `UploadBundle`: the +/// bundle is rebuilt from the library directory alone, so the manifest, the on-disk plaintext +/// and the re-derived ciphertext all have to agree across processes. +#[test] +fn a_derivative_survives_a_reopen_and_still_reaches_the_bundle() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + + // A second `Workspace::open` — the S-A10 shape: nothing shared but the directory. + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + let bundle = ws.upload_bundle(&asset_id).unwrap(); + assert_eq!( + bundle.derivatives.len(), + 1, + "the derivative survives a reopen" + ); + let blob = &bundle.derivatives[0]; + assert_eq!( + hash::hash_bytes(&blob.bytes), + blob.ciphertext_hash, + "and its ciphertext still content-addresses to the signed manifest" + ); +} + +/// Two formats persisted for one role are told apart by **(role, format)**, not by whichever +/// filename sorts first. +/// +/// `#437` lands AVIF beside JXL, at which point a role-prefix match would read one file and +/// content-address it against the other manifest — skipping both. Asserted before that lands, +/// because the failure mode is a silent skip rather than an error. +#[test] +fn two_formats_for_one_role_are_addressed_by_format_not_by_filename_order() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + + let asset = ws.asset(&asset_id).expect("held"); + let dir = media_dir(lib.path(), asset.capture_utc).join("derivatives"); + let stem = asset_id.simple().to_string(); + let jxl = fs::read(dir.join(format!("{stem}.thumbnail.jxl"))).unwrap(); + + // A decoy AVIF for the same role, sorting *before* `.jxl`, with different bytes. + let avif = b"not a real avif, and deliberately different".to_vec(); + fs::write(dir.join(format!("{stem}.thumbnail.avif")), &avif).unwrap(); + + // Both manifests, each addressing its own ciphertext. + let album_keys = ws.album(&asset.album_id).unwrap(); + let address = |bytes: &[u8]| { + let key = ws.file_key(album_keys, 1, &asset_id, &[7, 6, 5, 4, 3, 2, 1]); + let (_, ct) = stream::encrypt_asset_vec_with_prefix(&key, [7, 6, 5, 4, 3, 2, 1], bytes); + hash::hash_bytes(&ct) + }; + rewrite_bundle( + &lib, + &ws, + asset_id, + &[ + signed_derivative( + asset_id, + DerivativeRole::Thumbnail, + "image/jxl", + address(&jxl), + ), + signed_derivative( + asset_id, + DerivativeRole::Thumbnail, + "image/avif", + address(&avif), + ), + ], + ); + + let bundle = ws.upload_bundle(&asset_id).unwrap(); + assert_eq!( + bundle.derivatives.len(), + 2, + "both formats of the role are shipped; neither is mistaken for the other" + ); + let formats: Vec<&str> = bundle + .derivatives + .iter() + .map(|d| d.format.as_str()) + .collect(); + assert_eq!(formats, vec!["image/jxl", "image/avif"]); +} + +/// **A manifest naming an epoch the album does not hold is skipped, not a panic.** +/// +/// `file_key` reaches `album.amks[&epoch]`, which panics on a missing key — and the epoch comes +/// off an unverified `.cbor` on disk, inside a function whose whole contract is that it never +/// fails the bundle. This is reachable without any tampering at all: an album recovered from a +/// backup holds only the epochs it escrowed. +#[test] +fn a_derivative_naming_an_unheld_epoch_is_skipped_rather_than_panicking() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + + rewrite_bundle( + &lib, + &ws, + asset_id, + &[signed_derivative_at_epoch( + asset_id, + DerivativeRole::Thumbnail, + "image/jxl", + hash::hash_bytes(b"whatever"), + 9_999, + )], + ); + + // The assertion is as much that this returns at all as that it returns nothing. + let bundle = ws + .upload_bundle(&asset_id) + .expect("an unheld epoch must not fail the bundle"); + assert!( + bundle.derivatives.is_empty(), + "a derivative whose key epoch is absent is skipped" + ); + assert!( + !bundle.ciphertext.is_empty(), + "and the original is still shipped" + ); +} + +/// An embedding-role manifest is **out of this reader's scope**, and says so quietly. +/// +/// Its `embedding/{model_id}` grammar is not in the still format set and `crate::ml` produces no +/// derivative manifest at all, so the reader neither ships it nor complains about it — the +/// previous behaviour logged "no bytes on disk" for a manifest it had already decided not to +/// handle, which is a misleading warning for an artefact with no writer. +#[test] +fn an_embedding_role_manifest_is_out_of_scope_and_skipped() { + let lib = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let (_album, asset_id) = library_with_a_thumbnailed_asset(&lib, &src); + let ws = Workspace::open(lib.path(), b"passphrase", FAST).unwrap(); + + rewrite_bundle( + &lib, + &ws, + asset_id, + &[signed_derivative( + asset_id, + DerivativeRole::Embedding, + "embedding/mobileclip-b", + hash::hash_bytes(b"an embedding"), + )], + ); + + let bundle = ws.upload_bundle(&asset_id).unwrap(); + assert!( + bundle.derivatives.is_empty(), + "embeddings are not this reader's business until something produces them" + ); + assert_eq!( + verify_still_format(&signed_derivative( + asset_id, + DerivativeRole::Embedding, + "embedding/mobileclip-b", + hash::hash_bytes(b"an embedding"), + )), + Ok(None), + "and the closed-set check reports them as out of scope rather than rejecting them" + ); +} diff --git a/capsule-core/src/media/decode.rs b/capsule-core/src/media/decode.rs new file mode 100644 index 00000000..4f32bc86 --- /dev/null +++ b/capsule-core/src/media/decode.rs @@ -0,0 +1,399 @@ +//! The decode seam: bytes in, orientation-applied RGBA8 out. +//! +//! # The pipeline, and why each step is here +//! +//! 1. **Identify** ([`StillFormat::detect`]) — header first, so a `.jpg` that is really a HEIC +//! is classified as HEIC. +//! 2. **Gate** ([`StillFormat::is_decodable`]) — refuse with a typed +//! [`MediaError::UnsupportedFormat`] *before* touching a decoder, so a HEIC never reaches a +//! stub that would return a less informative error. +//! 3. **Budget** ([`MAX_DECODE_PIXELS`]) — a header-only +//! [`probe`](Decoder::probe) refuses an oversized frame before the decoder allocates. It has +//! to be pre-decode: `rawshift-image` decodes to interleaved RGB `u16`, so the bomb is +//! inside the decoder, not in Capsule's copy of the result. +//! 4. **Decode**, then **apply the EXIF orientation** to the pixels, so the frame this module +//! returns is always upright and its dimensions are the ones a viewer shows. +//! 5. **Normalise** to packed RGBA8, which is what [`crate::lqip`] and the downscale both take. +//! +//! # Two lossy edges, both deliberate and both asserted +//! +//! - **Alpha is lost.** `rawshift-image`'s decode target is interleaved RGB `u16` with no alpha +//! channel, and `decode_png` drops the alpha channel outright. Every frame this module +//! returns is therefore opaque. Nothing downstream needs alpha — the LQIP is an opaque +//! placeholder and the thumbnail is composited onto a grid — but a caller must not *assume* +//! transparency survived, so a test pins the flattening rather than leaving it to be +//! discovered. +//! - **16-bit is narrowed to 8.** `(sample >> 8) as u8` is the exact inverse of the crate's own +//! `u8_to_u16` widening (`v * 257`), so an 8-bit source round-trips bit-exactly and only a +//! genuinely deeper source loses its low byte. That is the same narrowing every encode +//! backend in the crate performs anyway. +//! +//! # The panic guard +//! +//! [`decode_guarded`] wraps a [`Decoder`] call in [`std::panic::catch_unwind`]. It is a free +//! function over the trait rather than a detail inside [`RawshiftDecoder`] for one reason: a +//! test has to be able to prove the guard holds, and it can only do that by injecting a +//! [`Decoder`] that panics. + +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use rawshift_image::core::ColorSpace; +use rawshift_image::core::image::RgbImage; +use rawshift_image::formats::{ + StandardFormat, decode_standard_image, probe_standard_image, read_standard_image_metadata, +}; +use rawshift_image::transforms::orientation::apply_orientation; + +use super::detect::{MAX_DECODE_PIXELS, StillFormat}; +use super::error::{FormatOp, MediaError}; +use crate::lqip::{Gamut, RgbaImage}; + +/// The EXIF orientation value meaning "already upright". +const ORIENTATION_NORMAL: u16 = 1; + +/// What a header-only probe can say about a still, normalised onto Capsule's own types. +/// +/// This is the "metadata normalisation" half of the module: `rawshift-image` reports a format, +/// a size, an optional bit depth and a colour space, plus (from a separate EXIF read) an +/// orientation. None of those types may appear in Capsule's public surface — they belong to a +/// pre-1.0 dependency — so each is mapped onto a Capsule type here, once. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MediaMetadata { + /// The identified format. + pub format: StillFormat, + /// Dimensions **as stored**, i.e. before the EXIF orientation is applied. A probe reads the + /// codec header, which knows nothing about the orientation tag; the upright dimensions are + /// what [`DecodedImage`] carries. + pub stored_dimensions: (u32, u32), + /// The EXIF orientation tag (1..=8), where the format carries one and it was readable. + pub orientation: Option, + /// Bits per channel, where the header exposes it cheaply. + pub bit_depth: Option, + /// The gamut [`crate::lqip::Lqip::encode`] is told to interpret the samples in. + /// + /// **Always [`Gamut::Srgb`] in this build, and that is upstream's doing rather than a + /// choice made here.** `probe_standard_image` hard-codes `ColorSpace::Srgb` into every + /// `ImageProbe` it returns, and `decode_standard_image` likewise tags every decoded frame + /// sRGB without converting — so `rawshift-image` 0.1.1 reports no source gamut for a + /// standard format at all, whatever the file's ICC profile says. The consequence is a + /// fidelity limitation, not a correctness bug: a Display P3 source gets a placeholder + /// interpreted as sRGB, i.e. slightly under-saturated, which is the direction slice `S-B14` + /// chose when it had to pick one. This module's private `gamut_of` is kept as the single + /// mapping point for when + /// the crate does start reporting it. + pub gamut: Gamut, +} + +impl MediaMetadata { + /// The dimensions a viewer shows: the stored ones, transposed when the orientation tag is + /// one of the four quarter-turns (5, 6, 7, 8). + pub const fn upright_dimensions(&self) -> (u32, u32) { + let (width, height) = self.stored_dimensions; + match self.orientation { + Some(5..=8) => (height, width), + _ => (width, height), + } + } +} + +/// A decoded still: upright, opaque, packed RGBA8. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodedImage { + /// The pixels, `width * height * 4` bytes, alpha uniformly `255`. + /// + /// [`crate::lqip::RgbaImage`] rather than a new buffer type: it is already the shape + /// [`crate::lqip::Lqip::encode`] and [`downscale_rgba8`](super::downscale_rgba8) take, and + /// it is unconditional, so no `media`-only type reaches the LQIP contract. + pub image: RgbaImage, + /// The source colour space this frame's samples are in. + pub gamut: Gamut, + /// The EXIF orientation value that was **consumed** — the transform is already applied to + /// `image`, so a renderer that rotates again is double-applying. `1` when the source + /// carried no tag. + pub orientation_applied: u16, + /// The format the pixels came out of. + pub format: StillFormat, +} + +impl DecodedImage { + /// Frame width in pixels, upright. + pub const fn width(&self) -> u32 { + self.image.width + } + + /// Frame height in pixels, upright. + pub const fn height(&self) -> u32 { + self.image.height + } +} + +/// The still-decode seam. +/// +/// A trait with exactly one production implementation ([`RawshiftDecoder`]), and that is the +/// point: the failure modes worth testing — a panicking decoder, a decoder that reports +/// dimensions its buffer does not match — cannot be produced from real bytes on demand, so they +/// are injected. +pub trait Decoder { + /// Read the format, dimensions and orientation from the header without decoding pixels. + /// + /// # Errors + /// [`MediaError::NotAStillImage`] when nothing recognisable is there, + /// [`MediaError::UnsupportedFormat`] when the format has no decoder in this build, and + /// [`MediaError::PixelBudgetExceeded`] when the header claims more than + /// [`MAX_DECODE_PIXELS`]. + fn probe(&self, bytes: &[u8], ext: &str) -> Result; + + /// Decode to upright, packed RGBA8. Probes first, so every [`probe`](Self::probe) error is + /// also a `decode` error. + /// + /// # Errors + /// As [`probe`](Self::probe), plus [`MediaError::Decode`] when a supported format's bytes + /// do not decode and [`MediaError::BufferLengthMismatch`] / [`MediaError::ZeroDimension`] + /// when the decoder's own output is inconsistent. + fn decode(&self, bytes: &[u8], ext: &str) -> Result; +} + +/// The `rawshift-image`-backed [`Decoder`] — the only production implementation. +#[derive(Debug, Clone, Copy, Default)] +pub struct RawshiftDecoder; + +impl Decoder for RawshiftDecoder { + #[tracing::instrument(level = "debug", skip_all, fields(bytes = bytes.len(), ext))] + fn probe(&self, bytes: &[u8], ext: &str) -> Result { + let format = gate(bytes, ext, FormatOp::Decode)?; + let probe = probe_standard_image(bytes).map_err(|e| MediaError::Decode { + format, + detail: format!("header probe: {e}"), + })?; + let (width, height) = (probe.size.width, probe.size.height); + if width == 0 || height == 0 { + return Err(MediaError::ZeroDimension { width, height }); + } + let pixels = u64::from(width) * u64::from(height); + if pixels > MAX_DECODE_PIXELS { + tracing::warn!( + %format, + width, + height, + pixels, + limit = MAX_DECODE_PIXELS, + "media: refusing an oversized still before the decoder allocates" + ); + return Err(MediaError::PixelBudgetExceeded { + pixels, + limit: MAX_DECODE_PIXELS, + }); + } + let metadata = MediaMetadata { + format, + stored_dimensions: (width, height), + orientation: orientation_of(bytes, format), + bit_depth: probe.bit_depth, + gamut: gamut_of(probe.color_space), + }; + tracing::debug!( + %format, + width, + height, + orientation = ?metadata.orientation, + bit_depth = ?metadata.bit_depth, + gamut = ?metadata.gamut, + "media: probed a still" + ); + Ok(metadata) + } + + #[tracing::instrument(level = "debug", skip_all, fields(bytes = bytes.len(), ext))] + fn decode(&self, bytes: &[u8], ext: &str) -> Result { + let probed = self.probe(bytes, ext)?; + let format = probed.format; + let mut rgb = decode_standard_image(bytes, standard_format(format)).map_err(|e| { + MediaError::Decode { + format, + detail: e.to_string(), + } + })?; + check_rgb_buffer(&rgb, format)?; + + let orientation = probed.orientation.unwrap_or(ORIENTATION_NORMAL); + if orientation != ORIENTATION_NORMAL { + apply_orientation(&mut rgb, orientation); + check_rgb_buffer(&rgb, format)?; + } + + let image = to_rgba8(&rgb); + tracing::debug!( + %format, + width = image.width, + height = image.height, + orientation, + "media: decoded a still" + ); + Ok(DecodedImage { + image, + gamut: probed.gamut, + orientation_applied: orientation, + format, + }) + } +} + +/// Run `decoder.decode` with the unwind boundary an import needs. +/// +/// A pre-1.0 decoder fed untrusted bytes is exactly where a panic is plausible, and a missing +/// thumbnail must never be able to abort an import that has already written signed, encrypted +/// bytes. A caught unwind becomes [`MediaError::DecoderPanic`] — reported, not swallowed. +pub fn decode_guarded( + decoder: &dyn Decoder, + bytes: &[u8], + ext: &str, +) -> Result { + guarded("decode", || decoder.decode(bytes, ext)) +} + +/// Run any fallible step of the still pipeline behind the same unwind boundary. +/// +/// `pub(crate)` rather than `pub`: `lifecycle` is the only caller and no client of this crate has +/// pixels of its own to run through it, so exporting it would widen the frozen surface for +/// nothing. +/// +/// Not just for `decode` — that is not the only third-party code the import path runs over pixels: +/// the placeholder goes through `chromahash` (also pre-1.0) and the derivative through +/// `zune-jpegxl`, and the module's promise is that *none* of them can abort an import — not that the +/// decoder specifically cannot. `stage` names the step in the warning so a caught panic is +/// attributable. +/// +/// `AssertUnwindSafe` is sound for the callers here: each closure borrows shared slices and +/// stateless values, so a caught unwind cannot leave a Capsule-owned invariant torn. +pub(crate) fn guarded( + stage: &'static str, + step: impl FnOnce() -> Result, +) -> Result { + if let Ok(result) = catch_unwind(AssertUnwindSafe(step)) { + return result; + } + tracing::warn!( + stage, + "media: a third-party codec panicked; the original is imported without a derivative" + ); + Err(MediaError::DecoderPanic) +} + +/// Identify a still and refuse anything this build has no codec for, before any decoder runs. +fn gate(bytes: &[u8], ext: &str, op: FormatOp) -> Result { + let Some(format) = StillFormat::detect(bytes, ext) else { + return Err(MediaError::NotAStillImage); + }; + if !format.is_decodable() { + return Err(MediaError::UnsupportedFormat { format, op }); + } + Ok(format) +} + +/// Map Capsule's format onto the crate's. +/// +/// Only the decodable set reaches here through [`gate`], which refuses the rest first — of the +/// grouped arms below, `Tiff` is the sole reachable one. The unreachable variants are still +/// mapped to the container the crate would actually see (a TIFF-based RAW *is* a TIFF to it, and +/// a CR3 is an ISO-BMFF file it can only reach through its HEIC arm) rather than panicking, so a +/// future `is_decodable` widening that forgets this table degrades to a decode error instead of +/// aborting an import. +fn standard_format(format: StillFormat) -> StandardFormat { + match format { + StillFormat::Jpeg => StandardFormat::Jpeg, + StillFormat::Png => StandardFormat::Png, + StillFormat::WebP => StandardFormat::WebP, + StillFormat::Jxl => StandardFormat::Jxl, + StillFormat::Gif => StandardFormat::Gif, + StillFormat::Ppm => StandardFormat::Ppm, + StillFormat::Avif => StandardFormat::Avif, + // `Tiff` is reachable and is the reason this arm exists; the RAW families ride along + // because a TIFF-based RAW is a TIFF to the crate. + StillFormat::Tiff + | StillFormat::Arw + | StillFormat::Cr2 + | StillFormat::Crw + | StillFormat::Dng + | StillFormat::Nef + | StillFormat::Raf => StandardFormat::Tiff, + StillFormat::Heic | StillFormat::Cr3 => StandardFormat::Heic, + } +} + +/// The EXIF orientation tag, where the format carries one. Only JPEG, TIFF, WebP, PNG and AVIF +/// have an EXIF block the crate's parser reads; the others return `None` rather than guessing. +fn orientation_of(bytes: &[u8], format: StillFormat) -> Option { + let metadata = read_standard_image_metadata(bytes, standard_format(format)); + // Only the eight defined values are honoured. `apply_orientation` warns and no-ops on + // anything else, which would leave `orientation_applied` claiming a transform that never + // happened — so an out-of-range tag is dropped here instead. + metadata.image.orientation.filter(|o| (1..=8).contains(o)) +} + +/// Map the crate's colour space onto the gamut the LQIP encoder takes. +/// +/// `LinearSrgb` and `Unknown` both become [`Gamut::Srgb`]: `Linear` names a transfer function +/// rather than a gamut, and sRGB primaries are the only safe assumption for an untagged source +/// (over-saturating is worse than under-saturating — the resolution slice `S-B14` recorded). +/// +/// **The wide-gamut arms are unreachable today**, because `probe_standard_image` hard-codes +/// `ColorSpace::Srgb` for every format (see [`MediaMetadata::gamut`]). They are here as the one +/// place that has to change when the crate starts reporting a source gamut, rather than as a +/// claim that it already does. +fn gamut_of(color_space: ColorSpace) -> Gamut { + match color_space { + ColorSpace::DisplayP3 => Gamut::DisplayP3, + ColorSpace::AdobeRgb => Gamut::AdobeRgb, + ColorSpace::Rec2020 => Gamut::Bt2020, + ColorSpace::ProPhotoRgb => Gamut::ProPhotoRgb, + // `Srgb`, `LinearSrgb`, `Unknown`, and — because `ColorSpace` is `#[non_exhaustive]` — + // any variant a future release adds. One wildcard rather than an explicit list plus a + // catch-all, since the answer is the same and two arms with one body only look like a + // distinction. sRGB is the conservative default: under-saturating a wide-gamut source + // is a smaller defect than over-saturating a narrow one. + _ => Gamut::Srgb, + } +} + +/// Refuse a decoder result whose buffer does not match the dimensions it reports. +/// +/// `RgbImage::new` performs no validation and `set_size` is public, so a decoder bug (or a +/// transform bug) can produce an inconsistent value. Checked here because the very next thing +/// Capsule does is index that buffer by those dimensions. +fn check_rgb_buffer(rgb: &RgbImage, format: StillFormat) -> Result<(), MediaError> { + let (width, height) = (rgb.width(), rgb.height()); + if width == 0 || height == 0 { + return Err(MediaError::ZeroDimension { width, height }); + } + let expected = u128::from(width) * u128::from(height) * 3; + let actual = rgb.data.len() as u128; + if expected != actual { + return Err(MediaError::BufferLengthMismatch { + format, + width, + height, + expected, + actual, + }); + } + Ok(()) +} + +/// Narrow interleaved RGB `u16` to packed, opaque RGBA8. +/// +/// `sample >> 8` is the exact inverse of the crate's `v * 257` widening, so an 8-bit source is +/// reproduced bit-for-bit. +fn to_rgba8(rgb: &RgbImage) -> RgbaImage { + let mut rgba = Vec::with_capacity(rgb.data.len() / 3 * 4); + for px in rgb.data.chunks_exact(3) { + rgba.push((px[0] >> 8) as u8); + rgba.push((px[1] >> 8) as u8); + rgba.push((px[2] >> 8) as u8); + rgba.push(u8::MAX); + } + RgbaImage { + width: rgb.width(), + height: rgb.height(), + rgba, + } +} diff --git a/capsule-core/src/media/derivative.rs b/capsule-core/src/media/derivative.rs new file mode 100644 index 00000000..780d202f --- /dev/null +++ b/capsule-core/src/media/derivative.rs @@ -0,0 +1,495 @@ +//! Still-derivative tiers, the closed format set, and the signed manifests over them. +//! +//! SSoT for the tiers and the formats: [Thumbnails and Previews](https://docs/design/thumbnails/). +//! This module owns the Capsule-side half — sizing, the closed enum, and building + signing a +//! [`DerivativeManifest`] through the same two-signature path assets use +//! ([`DerivativeCore::sign`]) — while `rawshift-image` owns the byte encode. +//! +//! # The closed format set, and where it is enforced +//! +//! [`DerivativeFormat`] is the tier table's format column as a closed enum. `format` is a +//! `String` in the signed struct and **stays** one, deliberately: the same field carries +//! `embedding/{model_id}` for embedding-role manifests (`crate::ml`), so a still-only enum +//! cannot be its type; and a `try_from` newtype would make +//! an *older* manifest carrying a future codec fail at deserialisation, turning a policy +//! rejection into a parse error before any signature is examined. The closed set is therefore +//! enforced at the two boundaries the contract names: +//! +//! - **production** — [`generate_still_derivatives`] only ever writes +//! [`DerivativeFormat::mime`], so no other value can be authored here; +//! - **verification** — [`verify_still_format`](crate::derivative_format::verify_still_format) +//! rejects a still-role manifest whose `format` does not parse, which is the structural +//! rejection the tier table specifies. +//! +//! Note the asymmetry in how those three are linked. Decision 24 moved all of them out of this +//! feature-gated module so the crates that *receive* a manifest can link the check; this file +//! then imported `DerivativeFormat`, but not the verification function, which it never calls. +//! The function therefore has to be written out as `crate::derivative_format::…`, because a bare +//! name for it resolves only through the `media` re-export — which holds under some feature +//! unions and not others, so it was a live link in isolation and a broken one in the merged +//! tree. The other two resolve through the import above, and spelling them out as well would be +//! a redundant target. +//! +//! # What this build encodes +//! +//! **JXL only, and losslessly.** `image/jxl` is the tier table's committed *master* format, so +//! the format that ships first is the one the table already puts first — but the pure-Rust +//! backend is `zune-jpegxl`'s `JxlSimpleEncoder`, which is lossless, so the tier's `q=50` is +//! passed through and ignored and a thumbnail costs more bytes than the table intends. A lossy +//! JXL needs C libjxl (`bindgen` + `pkg-config`). +//! +//! WebP — the table's last-resort delivery variant, and the obvious cheap lossy encoder — is +//! **not available at all**: `rawshift-image` 0.1.1's WebP module does not compile for aarch64 +//! (it passes `*const i8` where `libwebp-sys` declares `*const c_char`, and `c_char` is `u8` +//! there), and every mobile target Capsule ships is aarch64. AVIF needs `nasm` on every x86_64 +//! build host. Each is recorded as a per-`(tier, format)` deferral on +//! [`StillDerivatives::deferred`] and warned once, so the gap is countable rather than +//! invisible. +//! +//! [`DerivativeManifest`]: crate::crypto::provenance::DerivativeManifest +//! [`DerivativeCore::sign`]: crate::crypto::provenance::manifest::DerivativeCore::sign + +use std::collections::HashMap; +use std::fmt; + +use rawshift_image::core::image::RgbImage; +use rawshift_image::core::metadata::ImageMetadata; +use rawshift_image::core::{BitDepth, ColorSpace, MetadataEmbedOptions}; +use rawshift_image::formats::encode_rgb_image_to_vec; +use rawshift_image::formats::export::{CommonEncodeOptions, EncodeOptions, ZuneJxlEncodeConfig}; +use uuid::Uuid; + +use super::decode::DecodedImage; +use super::error::MediaError; +use super::resize::downscale_rgba8; +use crate::cbor; +use crate::crypto::CryptoError; +use crate::crypto::encryption::stream::NONCE_PREFIX_LEN; +use crate::crypto::hash::{self, Hash32}; +use crate::crypto::keys::{AmkVersion, Signer}; +use crate::crypto::provenance::manifest::{DERIVATIVE_MANIFEST_VERSION, DerivativeCore}; +use crate::crypto::provenance::{DerivativeManifest, DerivativeRole}; +use crate::derivative_format::DerivativeFormat; +use crate::lqip::RgbaImage; + +/// A derivative tier from the tier table. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DerivativeTier { + /// Grid display: long edge capped at 256 px, q=50. + Thumbnail, + /// Lightbox / single-asset view: source resolution, q=70. **Not generated by this build** + /// — a source-resolution derivative is only worth its bytes in the master codec, and the + /// master codec is the half that is still blocked on a toolchain. Kept in the enum because + /// the tier table commits to it and the sizing rule is the contract. + Preview, +} + +impl DerivativeTier { + /// The tiers this build actually generates. + pub const GENERATED: [Self; 1] = [Self::Thumbnail]; + + /// The provenance role this tier records. + pub const fn role(self) -> DerivativeRole { + match self { + Self::Thumbnail => DerivativeRole::Thumbnail, + Self::Preview => DerivativeRole::Preview, + } + } + + /// The role's on-disk name — mirrors `derivative_role_name` in the upload bundle reader, + /// which composes a derivative's exact path from this **and** the manifest's format, so two + /// formats of one role stay distinguishable. + pub const fn role_name(self) -> &'static str { + match self { + Self::Thumbnail => "thumbnail", + Self::Preview => "preview", + } + } + + /// Long-edge cap in pixels, or `None` to keep the source resolution. The 1080p cap in the + /// tier table governs the *video* preview transcode (slice `S-B5`), not the still preview. + pub const fn max_long_edge(self) -> Option { + match self { + Self::Thumbnail => Some(256), + Self::Preview => None, + } + } + + /// Lossy encoder quality for this tier, on the 0..=100 scale every backend here uses. + pub const fn quality(self) -> f32 { + match self { + Self::Thumbnail => 50.0, + Self::Preview => 70.0, + } + } +} + +impl fmt::Display for DerivativeTier { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.role_name()) + } +} + +/// What a derivative's signed manifest has to commit to about its **ciphertext**. +/// +/// Derivative bytes cross the network encrypted, exactly like the original +/// ([Encryption](https://docs/design/cryptography/encryption/) — "every asset — original bytes, +/// derivative bytes, metadata blob — is encrypted client-side"), so the content address a +/// manifest signs is the address of the *ciphertext*, and the nonce prefix that produced it is +/// signed alongside because it also selects the key. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SealedDerivative { + /// SHA-256 over the derivative ciphertext. + pub ciphertext_hash: Hash32, + /// The STREAM nonce prefix the ciphertext was produced under. + pub nonce_prefix: [u8; NONCE_PREFIX_LEN], +} + +/// The encryption seam between `media` and `lifecycle`. +/// +/// `media` owns pixels and knows nothing about album keys; `lifecycle` owns the AMK and must +/// not own codecs. A derivative's manifest cannot be signed until its ciphertext exists — the +/// hash is *of* the ciphertext — so the encryption has to happen inside generation, and this is +/// the narrowest thing that lets it: one method, no key material in any `media` signature. +/// +/// Each call must draw a **fresh** nonce prefix, so two derivatives of one asset get distinct +/// keys and distinct ciphertexts even when their plaintext is identical. +pub trait DerivativeSealer { + /// Encrypt `plaintext` under the source asset's identity and epoch, returning what the + /// manifest commits to. The ciphertext itself is discarded: clients keep the plaintext + /// derivative locally and re-derive the ciphertext at push time from the recorded prefix, + /// exactly as they do for the original. + /// + /// # Errors + /// [`MediaError::Sign`], and only that variant. Two things can go wrong: the underlying + /// encryption refuses, or no unused nonce prefix can be drawn for this `file_id` inside the + /// implementation's retry budget. Both are **workspace** faults rather than pixel ones — a + /// broken signer or a broken CSPRNG, not a photo this build cannot render — so the import + /// path propagates them instead of degrading to a missing thumbnail. An implementation must + /// therefore not report either as [`MediaError::Encode`], which means a codec refused a + /// frame and nothing else. + /// + /// `replaces` plays no part here: a derivative supersedes nothing, so the production + /// implementation passes `None` and enforces non-reuse against the set of prefixes already + /// spent on this `file_id` instead. + fn seal(&self, plaintext: &[u8]) -> Result; +} + +/// Everything the manifest signer needs that the pixels do not carry: the asset identity, the +/// epoch/authorisation context, and the two signing keys that produce the manifest's two hybrid +/// signatures. +pub struct DerivativeContext<'a> { + /// The asset the derivatives are generated from. + pub source_asset_id: Uuid, + /// Primitive bundle in force. + pub crypto_suite_id: u16, + /// Date-based wire protocol version (matches the album pin). + pub protocol_version: String, + /// The AMK epoch whose write-tier key signs the manifests. + pub amk_version: AmkVersion, + /// Device that generated the derivatives. + pub generated_by_device: Uuid, + /// Generating client version string. + pub generated_by_client: String, + /// RFC 3339 generation time (audit-only). + pub generated_at: String, + /// The device DSK (provenance signature); may be hardware-backed. + pub device_signer: &'a dyn Signer, + /// The per-epoch write-tier key (authorisation signature). + pub write_tier_signer: &'a dyn Signer, + /// Encrypts each generated derivative so its manifest can commit to the ciphertext. + pub sealer: &'a dyn DerivativeSealer, + /// The current head of each role's derivative chain, when this asset already has one. + /// + /// A `HashMap` rather than a `BTreeMap` because [`DerivativeRole`] derives `Hash + Eq` and + /// not `Ord`, and adding `Ord` to a signed wire type to key a lookup would be the tail + /// wagging the dog. Iteration order never escapes this map — it is only ever queried by + /// key — so the non-determinism a `HashMap` would otherwise introduce cannot reach the + /// signed bytes. + /// + /// A role's manifests are append-only **across time**, not merely within one call: a + /// backfill that adds the AVIF variant to an asset that already has a JXL thumbnail extends + /// that role's chain rather than starting a second one. Empty on a create, which is why the + /// import path passes an empty map; a regeneration reads it off the persisted bundle. + pub prior_heads: &'a HashMap, + /// What the **original**'s own manifest committed to. The `original` sentinel generates no + /// bytes and encrypts nothing: it is a reference to the original blob, so it signs the + /// original's ciphertext address and the original's nonce prefix, and a receiver resolves it + /// against the blob it already has. + pub original: SealedDerivative, +} + +/// One generated derivative: the encoded bytes plus its signed manifest. +#[derive(Debug, Clone)] +pub struct GeneratedDerivative { + /// Which tier this is. + pub tier: DerivativeTier, + /// Which committed format, or the `Original` sentinel. + pub format: DerivativeFormat, + /// The derivative bytes — the encoder output, and **empty** for + /// [`DerivativeFormat::Original`], whose manifest is a reference to the original rather than + /// a copy of it. + pub bytes: Vec, + /// The signed manifest binding `hash(bytes)`, the role and the format. + pub manifest: DerivativeManifest, +} + +/// The outcome of one asset's still-derivative generation. +/// +/// `deferred` is the per-`(tier, format)` gap S-B13 asks for: a pair with no encoder in this +/// build is *recorded*, not collapsed into the asset-level status. The asset is still +/// `DerivativeStatus::Decoded` — the decode succeeded and a renderable derivative exists — +/// which is why the two live in different places. +#[derive(Debug, Clone, Default)] +pub struct StillDerivatives { + /// The derivatives that were produced, in generation order. + pub generated: Vec, + /// The `(tier, format)` pairs the tier table commits to and this build cannot encode. + pub deferred: Vec<(DerivativeTier, DerivativeFormat)>, +} + +/// Generate the committed still derivatives for `decoded` across `tiers`, signing a +/// [`DerivativeManifest`] for each. +/// +/// Per tier: +/// - if the tier caps the long edge and the source is **not larger** than the cap, a single +/// `format = "original"` manifest is signed as a *reference* to the original blob (no bytes, +/// nothing encrypted, committing to [`DerivativeContext::original`]) — the +/// redundant-derivative sentinel from the contract, never a re-encode; +/// - otherwise the frame is downscaled to the tier, encoded to each encodable format of +/// [`DerivativeFormat::STILL_DELIVERY_ORDER`], and **encrypted** through +/// [`DerivativeContext::sealer`] before its manifest is signed over the ciphertext's address; +/// the formats with no encoder here are recorded as deferrals. +/// +/// Manifests of the same role are hash-chained in generation order, so a role's derivative +/// provenance is append-only exactly like the asset's. +/// +/// # Errors +/// - [`MediaError::Encode`] — a codec refused the frame. That is the **only** thing this variant +/// means here. +/// - [`MediaError::ZeroDimension`] — the source frame has a zero dimension. +/// - [`MediaError::Sign`] — the device signer or the epoch write-tier signer refused, or +/// [`DerivativeContext::sealer`] failed (an encryption refusal, or an exhausted nonce-prefix +/// draw). +/// +/// The split is the contract rather than bookkeeping. The import path degrades the first two to +/// "this asset has no thumbnail" and commits the original anyway; it **propagates** `Sign`, +/// because a workspace that cannot author a signed record is broken in a way a missing +/// derivative is not — the same fault would stop the asset's own manifest. +#[tracing::instrument( + level = "debug", + skip_all, + fields(asset_id = %ctx.source_asset_id, tiers = tiers.len()) +)] +pub fn generate_still_derivatives( + decoded: &DecodedImage, + tiers: &[DerivativeTier], + ctx: &DerivativeContext<'_>, +) -> Result { + if decoded.width() == 0 || decoded.height() == 0 { + return Err(MediaError::ZeroDimension { + width: decoded.width(), + height: decoded.height(), + }); + } + + let mut out = StillDerivatives::default(); + let source_long_edge = decoded.width().max(decoded.height()); + + for &tier in tiers { + // Each tier records a distinct role, so its manifests form their own chain — continued + // from wherever that role left off, not restarted. + let mut prior: Option = ctx.prior_heads.get(&tier.role()).copied(); + + if let Some(cap) = tier.max_long_edge() + && source_long_edge <= cap + { + tracing::debug!( + asset_id = %ctx.source_asset_id, + %tier, + source_long_edge, + cap, + "media: source is within the tier cap; signing the `original` sentinel" + ); + // A reference to the original blob: no bytes of its own, and **nothing encrypted** + // — it commits to the original's own ciphertext address and nonce prefix, and the + // receiver resolves it against the blob it already holds. See + // `DerivativeFormat::Original`. + out.generated.push(sign_derivative( + ctx, + tier, + DerivativeFormat::Original, + Vec::new(), + ctx.original, + &mut prior, + )?); + continue; + } + + let work = match tier.max_long_edge() { + Some(cap) => downscale_rgba8(&decoded.image, cap), + None => decoded.image.clone(), + }; + for format in DerivativeFormat::STILL_DELIVERY_ORDER { + if !format.is_encodable() { + tracing::warn!( + asset_id = %ctx.source_asset_id, + %tier, + %format, + "media: no encoder for this (tier, format) in this build; the tier still \ + ships its encodable variants and this pair is backfillable (S-B1 remainder)" + ); + out.deferred.push((tier, format)); + continue; + } + // Guarded: `libwebp`'s successor here is `zune-jpegxl`, also pre-1.0, and a panic + // inside it must not abort an import that may hold the only copy of a file. The + // stage name makes a caught unwind attributable to the encoder rather than to the + // decoder that ran before it. + let bytes = super::decode::guarded("encode", || encode(&work, format, tier))?; + // Encrypt before signing: the manifest's content address is the ciphertext's, so + // the ciphertext has to exist first. A fresh prefix per derivative, so two + // derivatives of one asset never share a key even for identical plaintext. + let sealed = ctx.sealer.seal(&bytes)?; + out.generated.push(sign_derivative( + ctx, tier, format, bytes, sealed, &mut prior, + )?); + } + } + + tracing::debug!( + asset_id = %ctx.source_asset_id, + generated = out.generated.len(), + deferred = out.deferred.len(), + "media: still derivatives generated" + ); + Ok(out) +} + +/// Encode a tier-sized RGBA8 frame to `format`. +/// +/// **Every encode passes [`MetadataEmbedOptions::none`] and an empty [`ImageMetadata`]**, and +/// both are load-bearing rather than tidy: the crate's own default is `all()`, so a +/// default-configured encode copies the source's EXIF — GPS fix included — into the derivative +/// bytes, and the JXL backend has a working `append_to_jxl` that would do exactly that. A +/// thumbnail is the derivative most likely to be served widest, so leaking a home address into +/// it would be the worst possible place for that default to win. Passing empty metadata means +/// the source's block is never even read. A test asserts the absence rather than trusting this +/// comment. +fn encode( + frame: &RgbaImage, + format: DerivativeFormat, + tier: DerivativeTier, +) -> Result, MediaError> { + let options = match format { + DerivativeFormat::Jxl => EncodeOptions::JxlZune(ZuneJxlEncodeConfig { + common: CommonEncodeOptions { + metadata: MetadataEmbedOptions::none(), + bit_depth: BitDepth::Eight, + }, + // The tier table's number, passed through as declared even though the backend + // ignores it: `JxlSimpleEncoder` is lossless, so today this is advisory. Keeping the + // contract's value here rather than hard-coding `0.0` (the crate's explicit + // "lossless" request) makes the eventual libjxl swap a backend change and not a + // quality decision taken again from scratch. + quality: tier.quality(), + // Encoder effort, 1..=9; the simple encoder may ignore this too. 7 is the crate's + // own default and there is no reason to differ. + effort: 7, + }), + // Unreachable: `is_encodable` gates the call, and `Original` never routes here at all + // (it carries no bytes). Kept as a typed refusal rather than a panic so a future + // widening that forgets an arm degrades to a reported failure. Reported as + // `Encode { format }` and not `UnsupportedFormat`, because the latter names a + // `StillFormat` and there is no still format at fault here — the caller asked for a + // *derivative* format this build cannot write, and the message has to say which. + DerivativeFormat::WebP | DerivativeFormat::Avif | DerivativeFormat::Original => { + return Err(MediaError::Encode { + format, + detail: "this build links no encoder for this derivative format".to_string(), + }); + } + }; + + let rgb = to_rgb_u16(frame); + encode_rgb_image_to_vec(&rgb, &ImageMetadata::default(), &options).map_err(|e| { + MediaError::Encode { + format, + detail: e.to_string(), + } + }) +} + +/// Widen packed RGBA8 to the interleaved RGB `u16` the encoders take, dropping alpha. +/// +/// `v * 257` is the crate's own widening, and the encoders narrow it back with `v >> 8`, so an +/// 8-bit frame reaches the codec bit-for-bit. Alpha is dropped because the decode path already +/// flattened it — every frame here is opaque. +fn to_rgb_u16(frame: &RgbaImage) -> RgbImage { + let mut data = Vec::with_capacity(frame.rgba.len() / 4 * 3); + for px in frame.rgba.chunks_exact(4) { + data.push(u16::from(px[0]) * 257); + data.push(u16::from(px[1]) * 257); + data.push(u16::from(px[2]) * 257); + } + RgbImage::with_color_space(frame.width, frame.height, data, ColorSpace::Srgb) +} + +/// Build, sign and chain one derivative manifest. +/// +/// `bytes` is the **plaintext** the client keeps on disk; `sealed` is what the manifest actually +/// commits to — the ciphertext's content address and the nonce prefix that produced it. The two +/// are separate arguments because they are separate artefacts: only one of them ever crosses the +/// network, and only the other is ever painted locally. +/// +/// `pub(super)` so the module's tests can exercise the chaining directly. That is not test +/// convenience for its own sake: today exactly one still format is encodable, so a single call +/// to [`generate_still_derivatives`] produces one manifest per role and the multi-link case — +/// the part of the chain that can actually be wrong — is unreachable through the public entry +/// point until a second encoder lands. +pub(super) fn sign_derivative( + ctx: &DerivativeContext<'_>, + tier: DerivativeTier, + format: DerivativeFormat, + bytes: Vec, + sealed: SealedDerivative, + prior: &mut Option, +) -> Result { + let core = DerivativeCore { + version: DERIVATIVE_MANIFEST_VERSION.into(), + crypto_suite_id: ctx.crypto_suite_id, + protocol_version: Some(ctx.protocol_version.clone()), + amk_version: Some(ctx.amk_version), + source_asset_id: ctx.source_asset_id, + role: tier.role(), + format: format.mime().into(), + // The address of the **ciphertext**, not of `bytes`: `bytes` is the plaintext kept + // locally, and what a receiver content-addresses is what crossed the network. + ciphertext_hash: sealed.ciphertext_hash, + nonce_prefix: sealed.nonce_prefix, + generated_by_device: ctx.generated_by_device, + generated_by_client: ctx.generated_by_client.clone(), + model_id: None, + model_version: None, + generated_at: ctx.generated_at.clone(), + prior_provenance_hash: *prior, + }; + let manifest = core + .sign(ctx.device_signer, ctx.write_tier_signer) + .map_err(|e: CryptoError| MediaError::Sign { + detail: format!("signing the derivative manifest: {e}"), + })?; + // The next manifest of this role chains to this one: SHA-256 over its canonical CBOR, + // signatures included — the same content-hash link the asset provenance chain uses. + *prior = Some(hash::hash_bytes( + &cbor::to_canonical_vec(&manifest).map_err(|e| MediaError::Sign { + detail: format!("serialising the derivative manifest: {e}"), + })?, + )); + Ok(GeneratedDerivative { + tier, + format, + bytes, + manifest, + }) +} diff --git a/capsule-core/src/media/detect.rs b/capsule-core/src/media/detect.rs new file mode 100644 index 00000000..e2a001c2 --- /dev/null +++ b/capsule-core/src/media/detect.rs @@ -0,0 +1,280 @@ +//! Capsule's closed still-format set, its magic-byte table, and the codec-coverage predicate. +//! +//! # Why Capsule sniffs rather than delegating +//! +//! `rawshift-image` ships `detect_standard_format`, and Capsule deliberately does not use it as +//! the primary table. Its `heic | heis | hevc | hevx` arm is `#[cfg(feature = "heic-decode")]`, +//! so a build without the HEIC codec cannot *recognise* an Apple HEIC either — its major brand +//! is `heic`. That would make the typed refusal for exactly the formats this build cannot decode +//! depend on whether it can decode them: a HEIC would arrive as "not a still image" instead of +//! "a still image with no codec here", which is the difference between a reportable, +//! backfillable gap and an apparent non-image. Capsule's reference library is HEIC end to end, +//! so that distinction is the whole point of slice `S-B13`. +//! +//! Two smaller reasons ride along, both about the brand table rather than a feature: the crate +//! reads the generic HEIF brand `mif1` as **AVIF**, and it does not recognise `heix` or `msf1` +//! under any configuration. +//! +//! The two tables are held together by a test rather than by hope: +//! `capsule-core`'s `still_format_agrees_with_rawshift_detection` asserts that for every format +//! both sides define unconditionally, Capsule's sniff and `detect_standard_format` name the same +//! thing. +//! +//! # Bytes first, extension second +//! +//! Detection is by header, so a `.jpg` that is really a HEIC is classified as HEIC. The +//! extension is consulted in exactly two places, both of them cases a header genuinely cannot +//! settle: +//! +//! 1. **RAW refinement.** ARW, CR2, DNG and NEF are TIFF containers and CR3 is ISO-BMFF; their +//! headers are their container's, so only the extension distinguishes a Sony ARW from a +//! scanner's TIFF. A misrefinement costs a deferral, never wrong pixels, because no RAW +//! family decodes in this build. +//! 2. **Fallback.** When the header sniffs to nothing at all. + +use std::fmt; + +/// The decode budget in pixels, refused **before** the decoder allocates. +/// +/// **128 Mpx**, which admits every real camera — a 102 Mpx medium-format frame has ~25% +/// headroom — while bounding what one still can ask the process for. +/// +/// The bound is worth stating honestly, because the naive figure is a large understatement. A +/// frame at this ceiling costs, in sequence and with overlap: 0.77 GB for `zune-png`'s `u16` +/// samples, another 0.77 GB when `decode_png` reallocates to drop the alpha channel, 0.51 GB for +/// Capsule's RGBA8 copy, and 0.77 GB again when the encode path widens a tier-sized frame back +/// to RGB `u16`. Peak is on the order of **2.5 GB**, not the ~1 GB a single buffer suggests. +/// Since `media` is implied by `native`, that peak happens on a phone as well as a workstation, +/// where it is an OOM kill rather than an error — which is the reason this is not simply set as +/// high as an allocation bomb would require. +pub const MAX_DECODE_PIXELS: u64 = 128_000_000; + +/// The closed set of still-image formats Capsule models. +/// +/// Closed on purpose: a still Capsule cannot name is not a still it silently ignores, it is a +/// [`MediaError::NotAStillImage`](super::MediaError::NotAStillImage). Membership here says +/// "Capsule knows this is a photo"; [`is_decodable`](Self::is_decodable) says whether *this +/// build* can read its pixels. The two are deliberately separate — that gap is what +/// `DerivativeStatus::DeferredNoCodec` reports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum StillFormat { + /// JPEG / JFIF. Decoded by `zune-jpeg`. + Jpeg, + /// PNG. Decoded by `zune-png`; alpha is flattened at the decode boundary. + Png, + /// WebP. **Recognised, not decoded**, and unusually the codec exists — it just does not + /// compile. `rawshift-image`'s WebP module passes `*const i8` where `libwebp-sys` 0.14.4 + /// declares `*const c_char`, and `c_char` is `u8` on aarch64, so the crate's `webp` feature + /// is an E0308 on every 64-bit ARM target — which is every mobile target Capsule ships. The + /// module is compiled by decode *or* encode, so there is no decode-only escape. Enabling it + /// would mean thumbnails on desktop and none on a phone; recognising and deferring it is the + /// honest outcome until upstream fixes the cast. + WebP, + /// JPEG XL. Decoded by `jxl-oxide`. No lossy encoder without C libjxl. + Jxl, + /// TIFF. Decoded by the `tiff` crate. Also the container of most RAW families. + Tiff, + /// GIF. Decoded by `gif`; the first frame only. + Gif, + /// Netpbm (P5/P6/P7/PFM). Recognised, not decoded: the `ppm-decode` backend is deliberately + /// not enabled. Netpbm is an intermediate and test-fixture format rather than something a + /// photo library holds, so recognising it and deferring is the honest outcome — and it costs + /// one fewer dependency than a codec nobody's library needs. + Ppm, + /// AVIF. Recognised, not decoded — the backend needs system libdav1d. + Avif, + /// HEIC / HEIF. Recognised, not decoded — the backend needs system libheif. + Heic, + /// Sony ARW. + Arw, + /// Canon CR2. + Cr2, + /// Canon CR3. + Cr3, + /// Canon CRW. + Crw, + /// Adobe DNG (including Apple ProRAW). + Dng, + /// Nikon NEF. + Nef, + /// Fujifilm RAF. + Raf, +} + +/// Every format this build can read pixels out of — the codec-coverage table, in one place. +/// +/// Logged verbatim on a deferral so the gap is legible in the field rather than only in a doc. +pub const SUPPORTED_STILL_FORMATS: &[StillFormat] = &[ + StillFormat::Jpeg, + StillFormat::Png, + StillFormat::Jxl, + StillFormat::Tiff, + StillFormat::Gif, +]; + +/// The RAW families, all of them recognised and none of them decodable here. +const RAW_FORMATS: &[StillFormat] = &[ + StillFormat::Arw, + StillFormat::Cr2, + StillFormat::Cr3, + StillFormat::Crw, + StillFormat::Dng, + StillFormat::Nef, + StillFormat::Raf, +]; + +impl StillFormat { + /// Whether *this build* can read pixels out of the format — the single codec-coverage + /// predicate, and the gate the decode path checks before touching a decoder. + pub fn is_decodable(self) -> bool { + SUPPORTED_STILL_FORMATS.contains(&self) + } + + /// Whether the format is one of the RAW families. RAW is recognised, never decoded here; + /// its container is TIFF or ISO-BMFF, so the extension is what names the family. + pub fn is_raw(self) -> bool { + RAW_FORMATS.contains(&self) + } + + /// The canonical media type — what the sidecar's `content_type` carries when detection + /// succeeded. + pub const fn mime(self) -> &'static str { + match self { + Self::Jpeg => "image/jpeg", + Self::Png => "image/png", + Self::WebP => "image/webp", + Self::Jxl => "image/jxl", + Self::Tiff => "image/tiff", + Self::Gif => "image/gif", + Self::Ppm => "image/x-portable-anymap", + Self::Avif => "image/avif", + Self::Heic => "image/heic", + Self::Arw => "image/x-sony-arw", + Self::Cr2 => "image/x-canon-cr2", + Self::Cr3 => "image/x-canon-cr3", + Self::Crw => "image/x-canon-crw", + Self::Dng => "image/x-adobe-dng", + Self::Nef => "image/x-nikon-nef", + Self::Raf => "image/x-fuji-raf", + } + } + + /// The lowercase extension table — the *fallback*, used only where a header cannot settle + /// the question (see the module docs). Extensions arrive lowercased. + pub fn from_extension(ext: &str) -> Option { + Some(match ext { + "jpg" | "jpeg" | "jpe" | "jfif" => Self::Jpeg, + "png" => Self::Png, + "webp" => Self::WebP, + "jxl" => Self::Jxl, + "tif" | "tiff" => Self::Tiff, + "gif" => Self::Gif, + "ppm" | "pgm" | "pnm" | "pfm" => Self::Ppm, + "avif" | "avifs" => Self::Avif, + "heic" | "heif" | "hif" => Self::Heic, + "arw" => Self::Arw, + "cr2" => Self::Cr2, + "cr3" => Self::Cr3, + "crw" => Self::Crw, + "dng" => Self::Dng, + "nef" | "nrw" => Self::Nef, + "raf" => Self::Raf, + _ => return None, + }) + } + + /// Sniff the format from the file header alone. + /// + /// `None` means "no still image Capsule models starts like this" — a video, an SVG, an XMP + /// sidecar, or noise. A RAW file sniffs to its *container* here ([`Tiff`](Self::Tiff), or + /// `None` for CR3's unrecognised `ftyp` brand); [`detect`](Self::detect) is what refines it. + /// + /// Each signature carries **its own** length requirement rather than sharing one floor: a + /// Netpbm header is eleven bytes and a JPEG's SOI three, so a blanket minimum would report + /// a perfectly well-formed short file as "not an image". + pub fn from_bytes(bytes: &[u8]) -> Option { + let at = |start: usize, needle: &[u8]| -> bool { + bytes + .get(start..start + needle.len()) + .is_some_and(|window| window == needle) + }; + + if at(0, b"GIF87a") || at(0, b"GIF89a") { + return Some(Self::Gif); + } + if at(0, b"\xFF\xD8\xFF") { + return Some(Self::Jpeg); + } + if at(0, b"\x89PNG\r\n\x1a\n") { + return Some(Self::Png); + } + if at(0, b"RIFF") && at(8, b"WEBP") { + return Some(Self::WebP); + } + // Bare JXL codestream, then the ISO-BMFF-wrapped form. + if at(0, b"\xFF\x0A") { + return Some(Self::Jxl); + } + if at(4, b"JXL ") { + return Some(Self::Jxl); + } + if at(0, b"II\x2A\x00") || at(0, b"MM\x00\x2A") { + return Some(Self::Tiff); + } + if at(4, b"ftyp") + && let Some(brand) = bytes.get(8..12) + { + return Self::from_isobmff_brand(brand); + } + // Netpbm: 'P' + a binary version + whitespace. P1-P4 are excluded because the + // `zune-ppm` backend does not decode them, matching `rawshift-image`. + if let Some(&[b'P', version, space]) = bytes.get(..3) + && matches!(version, b'5' | b'6' | b'7' | b'F' | b'f') + && space.is_ascii_whitespace() + { + return Some(Self::Ppm); + } + None + } + + /// The ISO-BMFF `ftyp` brand table. + /// + /// `mif1` is the generic HEIF brand and is claimed in practice by both HEIC and AVIF + /// writers. Capsule reads it as HEIC because the reference library is HEIC end to end; + /// `rawshift-image` reads it as AVIF. The divergence is cosmetic while neither decodes — + /// both produce the same + /// [`UnsupportedFormat`](super::MediaError::UnsupportedFormat) refusal and the same + /// `DeferredNoCodec` status — and it is a log-label difference, never a pixel difference. + fn from_isobmff_brand(brand: &[u8]) -> Option { + Some(match brand { + b"avif" | b"avis" => Self::Avif, + b"heic" | b"heix" | b"heis" | b"hevc" | b"hevx" | b"msf1" | b"mif1" => Self::Heic, + b"crx " => Self::Cr3, + _ => return None, + }) + } + + /// Identify a still: header first, extension only where a header cannot settle it. + /// + /// `ext` is the source file's lowercase extension without the dot (`""` when it has none). + /// The two extension-consulting cases are documented on the module. + pub fn detect(bytes: &[u8], ext: &str) -> Option { + match Self::from_bytes(bytes) { + // A TIFF header is also every TIFF-based RAW's header. Refine on the extension, and + // only ever *into* a RAW family — a `.tif` stays TIFF. + Some(Self::Tiff) => match Self::from_extension(ext) { + Some(raw) if raw.is_raw() => Some(raw), + _ => Some(Self::Tiff), + }, + Some(format) => Some(format), + None => Self::from_extension(ext), + } + } +} + +impl fmt::Display for StillFormat { + /// The format's media type — what a log line and an error message both want. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.mime()) + } +} diff --git a/capsule-core/src/media/error.rs b/capsule-core/src/media/error.rs new file mode 100644 index 00000000..b0e65426 --- /dev/null +++ b/capsule-core/src/media/error.rs @@ -0,0 +1,137 @@ +//! The typed failure set of the still pipeline (slice `S-B13`). +//! +//! Every one of these is a *report*, never a rejection: Capsule is a backup tool, so an original +//! whose pixels cannot be read is still imported as a signed, encrypted, `verify_asset`-accepting +//! asset. What varies is only whether a placeholder and a thumbnail could be produced beside it. +//! The point of the enum is that the reasons stay apart — a missing codec is a known, deferred +//! gap, while a *supported* format that fails to decode is a defect somebody should look at. + +use thiserror::Error; + +use super::detect::StillFormat; +use crate::derivative_format::DerivativeFormat; + +/// Which direction of a codec a format was needed for. A build can decode a format it cannot +/// encode (every format here except WebP) and the message has to say which half is missing. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FormatOp { + /// Reading pixels out of the format. + Decode, + /// Writing pixels into the format. + Encode, +} + +impl FormatOp { + /// The lowercase word used in log fields and messages. + pub const fn as_str(self) -> &'static str { + match self { + Self::Decode => "decode", + Self::Encode => "encode", + } + } +} + +/// Why the still pipeline could not produce what was asked of it. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum MediaError { + /// The format was identified and this build links no codec for it in that direction — + /// HEIC, AVIF and the RAW families for decode; everything but WebP for encode. The + /// **expected** gap: the format table is honest about it and + /// [`StillFormat::is_decodable`](super::StillFormat::is_decodable) is the same predicate the + /// pipeline gates on, so this is reached only when a caller bypasses that gate. + #[error("this build links no {} codec for {format}", op.as_str())] + UnsupportedFormat { + /// The format that was identified. + format: StillFormat, + /// Which half of the codec was missing. + op: FormatOp, + }, + /// The bytes are not any still image Capsule models — a video, an XMP sidecar, an SVG, or + /// noise. Distinct from [`UnsupportedFormat`](Self::UnsupportedFormat): there is nothing to + /// defer, because there is no still here to decode later either. + #[error("not a still image Capsule models")] + NotAStillImage, + /// A format this build *does* decode did not decode these particular bytes — truncation, + /// corruption, or a decoder bug. `detail` is the underlying decoder's message, flattened to + /// a `String` (rather than held as a `#[source]`) so this type stays `Clone + PartialEq` and + /// so no pre-1.0 dependency type appears in Capsule's public error shape. + #[error("decoding {format} failed: {detail}")] + Decode { + /// The format that was being decoded. + format: StillFormat, + /// The decoder's own message. + detail: String, + }, + /// A derivative encode failed. Same shape and same reasoning as + /// [`Decode`](Self::Decode). + #[error("encoding {format} failed: {detail}")] + Encode { + /// The derivative format that was being written. + format: DerivativeFormat, + /// The encoder's own message. + detail: String, + }, + /// The decoded frame has a zero dimension. Guarded rather than trusted because + /// [`crate::lqip::Lqip::encode`] and the downscale both need a non-empty frame, and a + /// hand-crafted header can claim one. + #[error("decoded frame has a zero dimension ({width}x{height})")] + ZeroDimension { + /// The width the decoder reported. + width: u32, + /// The height the decoder reported. + height: u32, + }, + /// The header claims more pixels than [`MAX_DECODE_PIXELS`](super::MAX_DECODE_PIXELS) + /// allows, and the decode was refused **before** allocating. + /// + /// This is the decode-bomb guard, and it has to be a pre-decode check rather than a + /// post-decode sanity assert: `rawshift-image` decodes to interleaved RGB `u16`, i.e. six + /// bytes per pixel, so a 60000x60000 PNG asks for ~21 GB inside the decoder before Capsule + /// ever sees a buffer. + #[error("{pixels} pixels exceeds the {limit}-pixel decode budget")] + PixelBudgetExceeded { + /// The pixel count the header claims. + pixels: u64, + /// The budget in force. + limit: u64, + }, + /// The decoder's buffer length does not match the dimensions it reported. A `RgbImage` is + /// constructible from mismatched parts (`RgbImage::new` validates nothing), so this is + /// checked at the boundary rather than assumed. + #[error( + "{format} decoder returned {actual} samples for {width}x{height} (expected {expected})" + )] + BufferLengthMismatch { + /// The format that was decoded. + format: StillFormat, + /// The width the decoder reported. + width: u32, + /// The height the decoder reported. + height: u32, + /// The sample count the dimensions imply. + expected: u128, + /// The sample count the decoder actually returned. + actual: u128, + }, + /// Signing or sealing a derivative manifest failed — a hardware device signer refused, or + /// the album's write-tier key for this epoch is missing. + /// + /// **The one derivative failure that is not about pixels**, and the reason it has its own + /// variant rather than being folded into [`Encode`](Self::Encode): every other error here + /// says "this asset has no thumbnail", which an import survives, while this one says the + /// *workspace* cannot author a signed record — the same fault that would stop the asset's + /// own manifest. The import path propagates this and degrades on everything else, and it can + /// only tell them apart if the type does. + #[error("signing the derivative manifest failed: {detail}")] + Sign { + /// The underlying crypto error's message. + detail: String, + }, + /// A third-party decoder panicked and the unwind was caught at the pipeline boundary. + /// + /// A pre-1.0 decoder fed untrusted bytes is exactly the place a panic is plausible, and an + /// import must never abort over a thumbnail. Reported as a decode failure rather than + /// swallowed, because a panic is a defect worth seeing. + #[error("the decoder panicked")] + DecoderPanic, +} diff --git a/capsule-core/src/media/mod.rs b/capsule-core/src/media/mod.rs new file mode 100644 index 00000000..71e728df --- /dev/null +++ b/capsule-core/src/media/mod.rs @@ -0,0 +1,78 @@ +//! Still-image decode, orientation, metadata normalisation and derivative generation — the +//! Capsule-side owner of the media pipeline (slices `S-B1`, `S-B13`). +//! +//! SSoT: [Thumbnails and Previews](https://docs/design/thumbnails/). +//! +//! # Boundaries +//! +//! Rawshift owns codecs; this module owns everything Capsule decides. Concretely, +//! [`rawshift-image`] performs format sniffing, pixel decode, and the byte encode, while this +//! module owns: +//! +//! - **the closed format sets** — [`StillFormat`](crate::media::StillFormat) (what Capsule +//! models as a still) and [`DerivativeFormat`](crate::media::DerivativeFormat) (what a signed +//! `DerivativeManifest.format` may say); +//! - **the pixel budget and the panic guard**, because a third-party pre-1.0 decoder is fed +//! untrusted bytes on the import path; +//! - **tier sizing and the downscale**, because `rawshift-image` has no resize and because a +//! derivative's bytes are signed, so the resample must be deterministic; +//! - **the metadata strip**, because the crate's own default embeds EXIF (GPS included) into +//! every encode. +//! +//! Every path above is reached through the re-exports below, and the doc links name them by +//! their full `crate::media::…` path deliberately. A bare ``[`StillFormat`]`` here does **not** +//! resolve under the `doc-check-rust` gate (`cargo doc --no-deps`, `-D warnings`) even though +//! the type is re-exported a few lines down — while it *does* resolve when the same command is +//! given `--document-private-items`, which is why the failure only appeared in CI. Rather than +//! guess at which of rustdoc's resolution rules produces that asymmetry, these links use the +//! path that resolves under both. +//! +//! LQIP is *not* here: it lives in the unconditional [`crate::lqip`] module so the import +//! pipeline, the uniffi FFI and `capsule-wasm` share one implementation (slice `S-B14`). This +//! module produces the pixels [`crate::lqip::Lqip::encode`] consumes. +//! +//! # What this build can and cannot do +//! +//! Every gap is a typed +//! [`UnsupportedFormat`](crate::media::MediaError::UnsupportedFormat) or a recorded per-format +//! deferral — never a silent absence, and never a panic (slice `S-B13`). +//! +//! **Decode** covers JPEG, PNG, JXL, TIFF and GIF. **Encode** covers JXL alone, and losslessly: +//! `image/jxl` is the tier table's committed master format, but the pure-Rust backend is +//! `zune-jpegxl`'s `JxlSimpleEncoder`, so the tier's declared `q=50` is advisory today. +//! +//! HEIC, AVIF, WebP and the RAW families sniff correctly and refuse to decode. HEIC and AVIF +//! need system libraries (libheif, libdav1d), AVIF encode needs an assembler (nasm) the cross +//! and cargo-ndk builds do not have, and **WebP is a compile failure rather than a missing +//! toolchain**: `rawshift-image`'s WebP module passes `*const i8` where `libwebp-sys` declares +//! `*const c_char`, which is `u8` on aarch64 — every mobile target — and the module is compiled +//! by decode *or* encode, so there is no decode-only escape. +//! +//! [`rawshift-image`]: https://docs.rs/rawshift-image + +mod decode; +mod derivative; +mod detect; +mod error; +mod resize; + +// `native`-gated because `lifecycle` is its only caller and `lifecycle` is `native`-gated: a +// `--features media` build without `native` (which the aarch64 cross-check uses, to isolate the +// codecs from SQLite's C build) would otherwise carry an unused re-export. +#[cfg(feature = "native")] +pub(crate) use self::decode::guarded; +pub use self::decode::{DecodedImage, Decoder, MediaMetadata, RawshiftDecoder, decode_guarded}; +pub use self::derivative::{ + DerivativeContext, DerivativeSealer, DerivativeTier, GeneratedDerivative, SealedDerivative, + StillDerivatives, generate_still_derivatives, +}; +pub use self::detect::{MAX_DECODE_PIXELS, SUPPORTED_STILL_FORMATS, StillFormat}; +pub use self::error::{FormatOp, MediaError}; +pub use self::resize::{capped_dimensions, downscale_rgba8}; +// Re-exported so `media::DerivativeFormat` keeps resolving, but *owned* by the unconditional +// module: the closed set has to be linkable by the crates that receive a manifest, and they +// build without this feature. See [`crate::derivative_format`]. +pub use crate::derivative_format::{DerivativeFormat, verify_still_format}; + +#[cfg(test)] +mod tests; diff --git a/capsule-core/src/media/resize.rs b/capsule-core/src/media/resize.rs new file mode 100644 index 00000000..7c992481 --- /dev/null +++ b/capsule-core/src/media/resize.rs @@ -0,0 +1,142 @@ +//! Capsule-owned downscale for the derivative tiers. +//! +//! `rawshift-image` has no resize — it offers crop, flips, rotations, blur and lens correction, +//! and nothing that changes the sample grid — so tier sizing is Capsule's, not the codec's. +//! +//! # Why integer area-averaging, specifically +//! +//! A derivative's bytes are content-addressed by a **signed** `DerivativeManifest`, so two runs +//! over the same source must produce the same bytes. That rules out floating-point accumulation +//! whose order or width could differ between builds and targets, and it rules out any resampler +//! with platform-tuned SIMD paths that are not required to be bit-identical. What is left is a +//! box filter accumulated in integers and divided by an exact sample count: deterministic on +//! every target, and the right filter for a large downscale anyway (a box average over the full +//! source rect is alias-free, where a bilinear tap would ignore most of the source pixels). +//! +//! Every product here is computed in `u64`, never in `usize`, because two of the CI-gated +//! targets (`armv7-linux-androideabi`, `i686-linux-android`) are 32-bit and both the +//! destination-to-source boundary and the channel accumulator reach past `u32` for shapes this +//! function accepts. +//! +//! Determinism here is **necessary, not sufficient**, and the distinction matters: the bytes a +//! manifest actually signs come out of `zune-jpegxl`, and an encoder version bump can change +//! them for the same input. That is fine — each generation signs the bytes it produced, and +//! manifests of a role chain in order — but it means "the resample is deterministic" buys +//! reproducibility of *this* step, not a stable content address across toolchains. +//! +//! Upscaling is not a thing this performs: a tier only ever caps a long edge, and a source +//! already inside the cap takes the `format = "original"` sentinel path instead +//! ([`DerivativeTier`](super::DerivativeTier)). + +use crate::lqip::RgbaImage; + +/// The dimensions a `width` x `height` frame takes when its long edge is capped at +/// `max_long_edge`, preserving aspect ratio. +/// +/// Returns the input unchanged when it already fits, so a caller can compare and skip — and +/// unchanged for an empty frame, which is the one case where "never zero" cannot hold: there is +/// no non-degenerate size for a frame with no pixels, and inventing one would have +/// [`downscale_rgba8`] read a buffer that has nothing in it. +pub fn capped_dimensions(width: u32, height: u32, max_long_edge: u32) -> (u32, u32) { + if width == 0 || height == 0 { + return (width, height); + } + let cap = max_long_edge.max(1); + let long_edge = width.max(height); + if long_edge <= cap { + return (width, height); + } + // Rounded rather than truncated so a 3:2 frame keeps its ratio as closely as an integer + // grid allows; `.max(1)` because a very lopsided frame (e.g. 8000x3) would otherwise round + // its short edge to zero and produce an empty buffer. + let scale = |edge: u32| -> u32 { + let numerator = u64::from(edge) * u64::from(cap); + let denominator = u64::from(long_edge); + (((numerator + denominator / 2) / denominator) as u32).max(1) + }; + (scale(width), scale(height)) +} + +/// Downscale packed RGBA8 so its long edge is at most `max_long_edge`. +/// +/// A frame already within the cap — or an empty one — is returned unchanged (cloned), which is +/// what makes this safe to call unconditionally. Deterministic: identical input yields +/// byte-identical output on every target. +pub fn downscale_rgba8(source: &RgbaImage, max_long_edge: u32) -> RgbaImage { + let (dst_w, dst_h) = capped_dimensions(source.width, source.height, max_long_edge); + if (dst_w, dst_h) == (source.width, source.height) { + return source.clone(); + } + + let (src_w, src_h) = (source.width as usize, source.height as usize); + // `u64`, not `usize`: on a 32-bit target `w * h * 4` overflows above ~1 Gpx, and the whole + // point of this branch is to be reached rather than to panic on its own arithmetic. + if source.rgba.len() as u64 != u64::from(source.width) * u64::from(source.height) * 4 { + // Defensive: this is a `pub` entry point and the very next thing it does is index the + // buffer by those dimensions. Every in-tree caller passes a `DecodedImage`, whose + // invariant this is, so a mismatch is a bug in a *new* caller — reported and returned + // unchanged rather than turned into a panic inside an import. + tracing::error!( + width = source.width, + height = source.height, + len = source.rgba.len(), + "media: downscale refused a buffer that does not match its dimensions" + ); + return source.clone(); + } + let (dw, dh) = (dst_w as usize, dst_h as usize); + let mut out = Vec::with_capacity(dw * dh * 4); + + // Boundary products and the channel accumulator are `u64`, not `usize`/`u32`, and both + // widths are load-bearing rather than defensive habit: + // + // - `(y + 1) * src_h` reaches `dst_edge * src_edge`. For a 1 x 256M frame reduced to a + // 256 px long edge that is 6.5e10, which overflows a 32-bit `usize` — and two of the + // CI-gated targets (`armv7-linux-androideabi`, `i686-linux-android`) are 32-bit. + // - the per-channel sum reaches `count * 255`, and `count` is the whole frame when this is + // called with a cap of 1 (a `pub` entry point, so that is reachable), i.e. 6.5e10 again — + // past `u32::MAX`. + // + // Neither is hypothetical-only: a debug build panics on the overflow and a release build + // wraps into wrong pixels or an out-of-bounds index. + for y in 0..dh { + // The source rows this destination row averages. Floor boundaries, so the destination + // grid is an exact partition of the source grid — every source pixel contributes to + // exactly one output pixel. Widened to at least one row because a lopsided cap can put + // two destination rows inside one source row, and an empty rect would divide by zero. + let y0 = (y as u64 * src_h as u64 / dh as u64) as usize; + let y1 = (((y as u64 + 1) * src_h as u64 / dh as u64) as usize) + .max(y0 + 1) + .min(src_h); + for x in 0..dw { + let x0 = (x as u64 * src_w as u64 / dw as u64) as usize; + let x1 = (((x as u64 + 1) * src_w as u64 / dw as u64) as usize) + .max(x0 + 1) + .min(src_w); + + let mut acc = [0u64; 4]; + let count = ((y1 - y0) * (x1 - x0)) as u64; + for sy in y0..y1 { + let row = sy * src_w * 4; + for sx in x0..x1 { + let i = row + sx * 4; + acc[0] += u64::from(source.rgba[i]); + acc[1] += u64::from(source.rgba[i + 1]); + acc[2] += u64::from(source.rgba[i + 2]); + acc[3] += u64::from(source.rgba[i + 3]); + } + } + // Round-half-up on the mean, so a uniform region reproduces its own value exactly + // rather than drifting down by up to one level per reduction. + for channel in acc { + out.push(((channel + count / 2) / count) as u8); + } + } + } + + RgbaImage { + width: dst_w, + height: dst_h, + rgba: out, + } +} diff --git a/capsule-core/src/media/tests.rs b/capsule-core/src/media/tests.rs new file mode 100644 index 00000000..67e6756d --- /dev/null +++ b/capsule-core/src/media/tests.rs @@ -0,0 +1,1695 @@ +//! Unit coverage for the still pipeline (slices `S-B1`, `S-B13`). +//! +//! # Fixtures are built, never committed +//! +//! The repository carries no binary image fixtures and this suite adds none. Three kinds of +//! input appear below, and the mix is deliberate: +//! +//! 1. **Procedural frames** ([`quadrants`], [`gradient`]) encoded in-test by +//! `rawshift-image`'s own JPEG/PNG/WebP encoders. Self-consistent by construction, which is +//! exactly what makes them right for the *orientation* and *EXIF* cases: the crate writes the +//! EXIF block and Capsule reads it back, so the fixture cannot drift from the parser. +//! 2. **A hand-built PNG** ([`hand_written_png`]) — IHDR/IDAT/IEND assembled byte by byte with +//! a local CRC-32 and an uncompressed-deflate zlib stream, so at least one decode case is +//! fed an input no part of the code under test produced. Without it the suite could pass +//! against an encoder and decoder that agree with each other and with nothing else. +//! 3. **Bare magic-byte headers** for the formats this build cannot decode. There is nothing to +//! decode there and nothing to fake: the assertion is that they are *recognised* and refused +//! with a typed error. + +use std::collections::HashMap; + +use rawshift_image::core::metadata::{ImageInfo, ImageMetadata, URational}; +use rawshift_image::core::{BitDepth, MetadataEmbedOptions}; +use rawshift_image::formats::encode_rgb_image_to_vec; +use rawshift_image::formats::export::{ + CommonEncodeOptions, EncodeOptions, JpegEncEncodeConfig, ZuneJxlEncodeConfig, + ZunePngEncodeConfig, +}; +use uuid::Uuid; + +use super::decode::{Decoder, RawshiftDecoder, decode_guarded}; +use super::derivative::{ + DerivativeContext, DerivativeSealer, DerivativeTier, SealedDerivative, StillDerivatives, + generate_still_derivatives, +}; +use super::detect::{MAX_DECODE_PIXELS, SUPPORTED_STILL_FORMATS, StillFormat}; +use super::error::{FormatOp, MediaError}; +use super::resize::{capped_dimensions, downscale_rgba8}; +use crate::crypto::encryption::encrypt_asset_rekey; +use crate::crypto::hash::Hash32; +use crate::crypto::keys::{Amk, AmkVersion, HybridSigningKey}; +use crate::crypto::primitives::{CRYPTO_SUITE_ID, PROTOCOL_VERSION}; +use crate::crypto::provenance::manifest::{DERIVATIVE_MANIFEST_VERSION, DerivativeCore}; +use crate::crypto::provenance::{DerivativeManifest, DerivativeRole}; +use crate::derivative_format::{DerivativeFormat, verify_still_format}; +use crate::lqip::{Gamut, Lqip, RgbaImage}; + +// ── Procedural fixtures ────────────────────────────────────────────────────── + +/// A frame of four flat quadrants at distinct luminances — TL 0, TR 85, BL 170, BR 255. +/// +/// Flat regions rather than a gradient because these fixtures go through a *lossy* JPEG: +/// sampling the middle of a flat quadrant is stable to within a couple of levels at q=90, while +/// a gradient's corner is not. Four distinct values make all eight EXIF orientations +/// distinguishable from one another. +fn quadrants(width: u32, height: u32) -> RgbaImage { + let (w, h) = (width as usize, height as usize); + let mut rgba = Vec::with_capacity(w * h * 4); + for y in 0..h { + for x in 0..w { + let v = match (x < w / 2, y < h / 2) { + (true, true) => 0, + (false, true) => 85, + (true, false) => 170, + (false, false) => 255, + }; + rgba.extend_from_slice(&[v, v, v, 255]); + } + } + RgbaImage { + width, + height, + rgba, + } +} + +/// A deterministic RGB gradient — the general-purpose frame for size and encode cases. +fn gradient(width: u32, height: u32) -> RgbaImage { + let (w, h) = (width as usize, height as usize); + let mut rgba = Vec::with_capacity(w * h * 4); + for y in 0..h { + for x in 0..w { + rgba.extend_from_slice(&[ + (x * 255 / w.max(1)) as u8, + (y * 255 / h.max(1)) as u8, + ((x + y) * 255 / (w + h).max(1)) as u8, + 255, + ]); + } + } + RgbaImage { + width, + height, + rgba, + } +} + +/// Mean of each quadrant's inner half, as `(TL, TR, BL, BR)`. +/// +/// The inner half avoids the quadrant boundaries, where JPEG's 8x8 blocks and chroma +/// subsampling smear one region into the next. +fn quadrant_means(image: &RgbaImage) -> (u32, u32, u32, u32) { + let (w, h) = (image.width as usize, image.height as usize); + let mean = |xs: std::ops::Range, ys: std::ops::Range| -> u32 { + let mut sum = 0u64; + let mut n = 0u64; + for y in ys.clone() { + for x in xs.clone() { + sum += u64::from(image.rgba[(y * w + x) * 4]); + n += 1; + } + } + (sum / n.max(1)) as u32 + }; + let (qw, qh) = (w / 2, h / 2); + let (ix, iy) = (qw / 4, qh / 4); + ( + mean(ix..qw - ix, iy..qh - iy), + mean(qw + ix..w - ix, iy..qh - iy), + mean(ix..qw - ix, qh + iy..h - iy), + mean(qw + ix..w - ix, qh + iy..h - iy), + ) +} + +/// Widen packed RGBA8 into the interleaved RGB `u16` the encoders take. +fn to_rgb_u16(frame: &RgbaImage) -> rawshift_image::core::image::RgbImage { + let mut data = Vec::with_capacity(frame.rgba.len() / 4 * 3); + for px in frame.rgba.chunks_exact(4) { + data.push(u16::from(px[0]) * 257); + data.push(u16::from(px[1]) * 257); + data.push(u16::from(px[2]) * 257); + } + rawshift_image::core::image::RgbImage::with_color_space( + frame.width, + frame.height, + data, + rawshift_image::core::ColorSpace::Srgb, + ) +} + +/// A `CommonEncodeOptions` embedding whatever `metadata` asks for, at 8 bits. +fn common(metadata: MetadataEmbedOptions) -> CommonEncodeOptions { + CommonEncodeOptions { + metadata, + bit_depth: BitDepth::Eight, + } +} + +/// Encode a frame as a baseline JPEG, optionally embedding `metadata`. +/// +/// Quality 90 rather than the tier's 50: this is a *source* fixture, and a decode test should +/// not have to absorb the tier's own quality budget as well. +fn jpeg_bytes(frame: &RgbaImage, metadata: Option<&ImageMetadata>) -> Vec { + let embed = if metadata.is_some() { + MetadataEmbedOptions { + embed_exif: true, + embed_icc: false, + embed_xmp: false, + } + } else { + MetadataEmbedOptions::none() + }; + let options = EncodeOptions::JpegJpegEnc(JpegEncEncodeConfig { + common: common(embed), + quality: 90, + }); + let empty = ImageMetadata::default(); + encode_rgb_image_to_vec(&to_rgb_u16(frame), metadata.unwrap_or(&empty), &options) + .expect("the fixture JPEG encodes") +} + +/// Encode a frame as a PNG with no metadata at all. +fn png_bytes(frame: &RgbaImage) -> Vec { + let options = EncodeOptions::PngZune(ZunePngEncodeConfig { + common: common(MetadataEmbedOptions::none()), + ..ZunePngEncodeConfig::default() + }); + encode_rgb_image_to_vec(&to_rgb_u16(frame), &ImageMetadata::default(), &options) + .expect("the fixture PNG encodes") +} + +/// A bare 12-byte RIFF/WEBP header. +/// +/// A *header*, not an encode, because this build has no WebP codec at all: `rawshift-image`'s +/// WebP module does not compile for aarch64 (`*const i8` against `libwebp-sys`'s `*const +/// c_char`), so the crate's `webp` feature is off in both directions. Twelve bytes is all a +/// detection case needs, and there is nothing to decode. +fn webp_header() -> Vec { + let mut bytes = b"RIFF".to_vec(); + bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(b"WEBP"); + bytes +} + +/// Encode a frame as JXL through the same backend the derivative path uses, optionally +/// embedding `metadata`. +fn jxl_bytes(frame: &RgbaImage, metadata: Option<&ImageMetadata>) -> Vec { + let embed = if metadata.is_some() { + MetadataEmbedOptions::all() + } else { + MetadataEmbedOptions::none() + }; + let options = EncodeOptions::JxlZune(ZuneJxlEncodeConfig { + common: common(embed), + quality: 50.0, + effort: 7, + }); + let empty = ImageMetadata::default(); + encode_rgb_image_to_vec(&to_rgb_u16(frame), metadata.unwrap_or(&empty), &options) + .expect("the fixture JXL encodes") +} + +/// An `ImageMetadata` carrying only an EXIF orientation tag. +fn oriented(orientation: u16) -> ImageMetadata { + ImageMetadata { + image: ImageInfo { + orientation: Some(orientation), + ..ImageInfo::default() + }, + ..ImageMetadata::default() + } +} + +/// The GPS degrees/minutes/seconds triple used by the privacy cases — a distinctive fix whose +/// rationals are searchable as raw bytes. +const GPS_LAT_DMS: [u32; 3] = [51, 30, 26]; +const GPS_LON_DMS: [u32; 3] = [0, 7, 39]; + +/// An `ImageMetadata` carrying a GPS fix — the metadata a thumbnail must never inherit. +fn located() -> ImageMetadata { + let dms = |v: [u32; 3]| { + v.map(|n| URational { + numerator: n, + denominator: 1, + }) + }; + let mut metadata = ImageMetadata::default(); + metadata.gps.latitude = Some(dms(GPS_LAT_DMS)); + metadata.gps.latitude_ref = Some('N'); + metadata.gps.longitude = Some(dms(GPS_LON_DMS)); + metadata.gps.longitude_ref = Some('W'); + metadata +} + +// ── The independently-constructed PNG ──────────────────────────────────────── + +/// CRC-32 (IEEE 802.3), computed here so the PNG fixture owes nothing to a dependency. +fn crc32(bytes: &[u8]) -> u32 { + let mut crc = 0xFFFF_FFFFu32; + for &byte in bytes { + crc ^= u32::from(byte); + for _ in 0..8 { + crc = if crc & 1 == 1 { + (crc >> 1) ^ 0xEDB8_8320 + } else { + crc >> 1 + }; + } + } + !crc +} + +/// Adler-32, the zlib stream checksum. +fn adler32(bytes: &[u8]) -> u32 { + let mut a = 1u32; + let mut b = 0u32; + for &byte in bytes { + a = (a + u32::from(byte)) % 65_521; + b = (b + a) % 65_521; + } + (b << 16) | a +} + +/// One PNG chunk: length, type, payload, CRC over type+payload. +fn png_chunk(kind: &[u8; 4], payload: &[u8]) -> Vec { + let mut chunk = Vec::with_capacity(payload.len() + 12); + chunk.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + chunk.extend_from_slice(kind); + chunk.extend_from_slice(payload); + let mut crc_input = kind.to_vec(); + crc_input.extend_from_slice(payload); + chunk.extend_from_slice(&crc32(&crc_input).to_be_bytes()); + chunk +} + +/// A 2x2 8-bit RGB PNG built byte by byte: signature, IHDR, a stored-deflate IDAT, IEND. +/// +/// `deflate` here is a single final *stored* block (BTYPE 00), so no compressor is involved — +/// the point of this fixture is that nothing under test, and no encoder it shares code with, +/// produced it. +fn hand_written_png() -> Vec { + // Four pixels: red, green, blue, white — each row prefixed by filter type 0 (None). + let raw: Vec = vec![ + 0, 255, 0, 0, 0, 255, 0, // row 0: filter, red, green + 0, 0, 0, 255, 255, 255, 255, // row 1: filter, blue, white + ]; + + let mut zlib = vec![0x78, 0x01]; // CM=8, CINFO=7, FLEVEL=0, FCHECK making it a multiple of 31 + zlib.push(0x01); // final stored block + zlib.extend_from_slice(&(raw.len() as u16).to_le_bytes()); + zlib.extend_from_slice(&(!(raw.len() as u16)).to_le_bytes()); + zlib.extend_from_slice(&raw); + zlib.extend_from_slice(&adler32(&raw).to_be_bytes()); + + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&2u32.to_be_bytes()); // width + ihdr.extend_from_slice(&2u32.to_be_bytes()); // height + ihdr.extend_from_slice(&[8, 2, 0, 0, 0]); // 8-bit, colour type 2 (RGB), deflate, no filter/interlace + + let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); + png.extend(png_chunk(b"IHDR", &ihdr)); + png.extend(png_chunk(b"IDAT", &zlib)); + png.extend(png_chunk(b"IEND", &[])); + png +} + +// ── Detection ──────────────────────────────────────────────────────────────── + +/// A 12-byte header for each format Capsule recognises but cannot decode. Twelve bytes is +/// exactly what [`StillFormat::from_bytes`] requires, so these also pin the minimum length. +fn isobmff(brand: &[u8; 4]) -> Vec { + let mut bytes = vec![0, 0, 0, 0x20]; + bytes.extend_from_slice(b"ftyp"); + bytes.extend_from_slice(brand); + bytes +} + +/// Real encodes on one side, bare headers on the other: every variant of the closed set is +/// reachable from bytes, and the sniff names the right one. +#[test] +fn every_still_format_is_reachable_from_its_header() { + let frame = gradient(8, 8); + let cases: &[(Vec, &str, StillFormat)] = &[ + (jpeg_bytes(&frame, None), "jpg", StillFormat::Jpeg), + (png_bytes(&frame), "png", StillFormat::Png), + (webp_header(), "webp", StillFormat::WebP), + (hand_written_png(), "png", StillFormat::Png), + ( + b"GIF89a\x08\x00\x08\x00\x00\x00".to_vec(), + "gif", + StillFormat::Gif, + ), + ( + b"\xFF\x0A\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00".to_vec(), + "jxl", + StillFormat::Jxl, + ), + ( + b"II\x2A\x00\x08\x00\x00\x00\x00\x00\x00\x00".to_vec(), + "tif", + StillFormat::Tiff, + ), + ( + b"MM\x00\x2A\x00\x00\x00\x08\x00\x00\x00\x00".to_vec(), + "tiff", + StillFormat::Tiff, + ), + (b"P6\n8 8\n255\n".to_vec(), "ppm", StillFormat::Ppm), + (isobmff(b"avif"), "avif", StillFormat::Avif), + (webp_header(), "webp", StillFormat::WebP), + (isobmff(b"heic"), "heic", StillFormat::Heic), + (isobmff(b"mif1"), "heic", StillFormat::Heic), + (isobmff(b"crx "), "cr3", StillFormat::Cr3), + // TIFF-container RAW: the header says TIFF and only the extension names the family. + ( + b"II\x2A\x00\x08\x00\x00\x00\x00\x00\x00\x00".to_vec(), + "arw", + StillFormat::Arw, + ), + ( + b"II\x2A\x00\x08\x00\x00\x00\x00\x00\x00\x00".to_vec(), + "cr2", + StillFormat::Cr2, + ), + ( + b"II\x2A\x00\x08\x00\x00\x00\x00\x00\x00\x00".to_vec(), + "dng", + StillFormat::Dng, + ), + ( + b"MM\x00\x2A\x00\x00\x00\x08\x00\x00\x00\x00".to_vec(), + "nef", + StillFormat::Nef, + ), + ]; + for (bytes, ext, expected) in cases { + assert_eq!( + StillFormat::detect(bytes, ext), + Some(*expected), + "detecting a .{ext} fixture" + ); + } + + // CRW and RAF have no in-tree header fixture; they are extension-only entries, and the + // point of asserting them is that the fallback table covers the whole RAW set. + assert_eq!(StillFormat::detect(b"", "crw"), Some(StillFormat::Crw)); + assert_eq!(StillFormat::detect(b"", "raf"), Some(StillFormat::Raf)); +} + +/// Bytes win over the extension. A HEIC named `.jpg` must not be handed to the JPEG decoder — +/// that is the difference between a typed "no codec for HEIC" deferral and a decode failure +/// blamed on JPEG. +#[test] +fn the_header_beats_a_lying_extension() { + assert_eq!( + StillFormat::detect(&isobmff(b"heic"), "jpg"), + Some(StillFormat::Heic) + ); + let png = png_bytes(&gradient(4, 4)); + assert_eq!(StillFormat::detect(&png, "jpeg"), Some(StillFormat::Png)); + + // The one refinement that runs the other way is *into* a RAW family, and only from a TIFF + // header — a real `.tif` stays TIFF. + let tiff_header = b"II\x2A\x00\x08\x00\x00\x00\x00\x00\x00\x00"; + assert_eq!( + StillFormat::detect(tiff_header, "tif"), + Some(StillFormat::Tiff) + ); +} + +/// Nothing recognisable, and never a panic. `detect` is the first thing untrusted bytes touch. +#[test] +fn unrecognisable_bytes_are_not_a_still() { + let cases: &[&[u8]] = &[ + b"", + b"\x00", + b"\xFF\xD8", // a JPEG SOI truncated below the 12-byte floor + b"noise-x!", // 8 bytes, under the floor + b"not an image at all, really", // long enough, no signature + b"\x00\x00\x00\x20ftypqt ", // ISO-BMFF with an unmodelled brand (QuickTime) + b"", + b"\x1AE\xDF\xA3\x01\x00\x00\x00\x00\x00\x00\x23", // Matroska/WebM + ]; + for bytes in cases { + assert_eq!( + StillFormat::detect(bytes, ""), + None, + "these bytes are not a still Capsule models: {:?}", + &bytes[..bytes.len().min(12)] + ); + } +} + +/// The Capsule table and `rawshift-image`'s own `detect_standard_format` must not drift where +/// both define an answer. +/// +/// Scoped to the formats whose signature is unconditional on both sides. The ISO-BMFF brands are +/// excluded on purpose: the crate's HEIC arm is feature-gated (so it cannot recognise HEIC in +/// this build — the reason Capsule sniffs at all), and it reads the generic `mif1` brand as AVIF +/// where Capsule reads it as HEIC. Neither decodes here, so that divergence is a log label, not +/// a pixel. +#[test] +fn still_format_agrees_with_rawshift_detection() { + use rawshift_image::formats::{StandardFormat, detect_standard_format}; + + let frame = gradient(8, 8); + let cases: &[(Vec, StandardFormat, StillFormat)] = &[ + ( + jpeg_bytes(&frame, None), + StandardFormat::Jpeg, + StillFormat::Jpeg, + ), + (png_bytes(&frame), StandardFormat::Png, StillFormat::Png), + (webp_header(), StandardFormat::WebP, StillFormat::WebP), + (hand_written_png(), StandardFormat::Png, StillFormat::Png), + ( + b"GIF89a\x08\x00\x08\x00\x00\x00".to_vec(), + StandardFormat::Gif, + StillFormat::Gif, + ), + ( + b"\xFF\x0A\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00".to_vec(), + StandardFormat::Jxl, + StillFormat::Jxl, + ), + ( + b"II\x2A\x00\x08\x00\x00\x00\x00\x00\x00\x00".to_vec(), + StandardFormat::Tiff, + StillFormat::Tiff, + ), + ( + b"P6\n8 8\n255\n".to_vec(), + StandardFormat::Ppm, + StillFormat::Ppm, + ), + // A JPEG SOI is three bytes and a Netpbm header eleven, so both sides recognise a file + // far shorter than a blanket minimum-length floor would admit. + (isobmff(b"avif"), StandardFormat::Avif, StillFormat::Avif), + ]; + for (bytes, theirs, ours) in cases { + assert_eq!(detect_standard_format(bytes), Some(*theirs)); + assert_eq!(StillFormat::from_bytes(bytes), Some(*ours)); + } +} + +/// The coverage table is the single answer to "can this build read these pixels?", and it has +/// to agree with what the decoder actually does. +#[test] +fn is_decodable_matches_the_supported_table() { + for format in SUPPORTED_STILL_FORMATS { + assert!(format.is_decodable(), "{format} is in the supported table"); + assert!(!format.is_raw(), "no RAW family decodes in this build"); + } + for format in [ + StillFormat::Ppm, + StillFormat::WebP, + StillFormat::Avif, + StillFormat::Heic, + StillFormat::Arw, + StillFormat::Cr2, + StillFormat::Cr3, + StillFormat::Crw, + StillFormat::Dng, + StillFormat::Nef, + StillFormat::Raf, + ] { + assert!(!format.is_decodable(), "{format} has no decoder here"); + } + // Every mime is distinct, so a `content_type` cannot silently collide. + let mut mimes: Vec<&str> = SUPPORTED_STILL_FORMATS.iter().map(|f| f.mime()).collect(); + mimes.sort_unstable(); + let count = mimes.len(); + mimes.dedup(); + assert_eq!(mimes.len(), count, "each format has its own media type"); +} + +// ── Decode ─────────────────────────────────────────────────────────────────── + +/// Every decodable format round-trips to the dimensions it was built with, with a full RGBA8 +/// buffer and uniform opaque alpha. +#[test] +fn decodes_every_supported_container_to_opaque_rgba8() { + let frame = gradient(6, 4); + let cases: &[(Vec, &str, StillFormat, u32, u32)] = &[ + (jpeg_bytes(&frame, None), "jpg", StillFormat::Jpeg, 6, 4), + (png_bytes(&frame), "png", StillFormat::Png, 6, 4), + (jxl_bytes(&frame, None), "jxl", StillFormat::Jxl, 6, 4), + (hand_written_png(), "png", StillFormat::Png, 2, 2), + ]; + for (bytes, ext, format, width, height) in cases { + let decoded = RawshiftDecoder + .decode(bytes, ext) + .unwrap_or_else(|e| panic!("decoding the .{ext} fixture: {e}")); + assert_eq!(decoded.format, *format); + assert_eq!((decoded.width(), decoded.height()), (*width, *height)); + assert_eq!( + decoded.image.rgba.len() as u32, + width * height * 4, + "the buffer is exactly w*h*4" + ); + assert!( + decoded.image.rgba.chunks_exact(4).all(|px| px[3] == 255), + "every decoded frame is opaque" + ); + assert_eq!(decoded.orientation_applied, 1, "no tag, no transform"); + } +} + +/// The hand-built PNG's actual pixels, checked against the bytes that were written into it. +/// +/// This is the one place the suite is not self-consistent: the input owes nothing to the encoder +/// the other cases use, so agreement here is evidence about the decoder rather than about a +/// matched pair. +#[test] +fn the_hand_written_png_decodes_to_the_pixels_it_declares() { + let decoded = RawshiftDecoder + .decode(&hand_written_png(), "png") + .expect("a hand-built 2x2 RGB PNG decodes"); + assert_eq!((decoded.width(), decoded.height()), (2, 2)); + assert_eq!( + decoded.image.rgba, + vec![ + 255, 0, 0, 255, // red + 0, 255, 0, 255, // green + 0, 0, 255, 255, // blue + 255, 255, 255, 255, // white + ] + ); +} + +/// PNG alpha is **flattened**, not preserved — `rawshift-image` decodes to RGB with no alpha +/// channel. Asserted so the loss cannot regress into an "alpha survives" assumption somewhere +/// downstream. +#[test] +fn png_alpha_is_flattened_to_opaque() { + // A 1x1 fully transparent RGBA PNG, hand-built for the same reason as the fixture above. + let raw = vec![0u8, 0, 0, 0, 0]; // filter 0 + RGBA(0,0,0,0) + let mut zlib = vec![0x78, 0x01, 0x01]; + zlib.extend_from_slice(&(raw.len() as u16).to_le_bytes()); + zlib.extend_from_slice(&(!(raw.len() as u16)).to_le_bytes()); + zlib.extend_from_slice(&raw); + zlib.extend_from_slice(&adler32(&raw).to_be_bytes()); + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&1u32.to_be_bytes()); + ihdr.extend_from_slice(&1u32.to_be_bytes()); + ihdr.extend_from_slice(&[8, 6, 0, 0, 0]); // colour type 6 = RGBA + let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); + png.extend(png_chunk(b"IHDR", &ihdr)); + png.extend(png_chunk(b"IDAT", &zlib)); + png.extend(png_chunk(b"IEND", &[])); + + let decoded = RawshiftDecoder + .decode(&png, "png") + .expect("RGBA PNG decodes"); + assert_eq!(decoded.image.rgba, vec![0, 0, 0, 255], "alpha is dropped"); +} + +/// A recognised format with no codec refuses **before** any decoder runs, and says which half +/// of the codec is missing. This is the S-B13 contract at the seam. +#[test] +fn a_format_with_no_codec_refuses_with_a_typed_error() { + let cases: &[(Vec, &str, StillFormat)] = &[ + (isobmff(b"heic"), "heic", StillFormat::Heic), + (isobmff(b"avif"), "avif", StillFormat::Avif), + (isobmff(b"crx "), "cr3", StillFormat::Cr3), + ( + b"II\x2A\x00\x08\x00\x00\x00\x00\x00\x00\x00".to_vec(), + "arw", + StillFormat::Arw, + ), + (b"".to_vec(), "raf", StillFormat::Raf), + (b"P6\n8 8\n255\n".to_vec(), "ppm", StillFormat::Ppm), + ]; + for (bytes, ext, format) in cases { + assert_eq!( + RawshiftDecoder.decode(bytes, ext), + Err(MediaError::UnsupportedFormat { + format: *format, + op: FormatOp::Decode, + }), + "a .{ext} must defer rather than fail" + ); + assert_eq!( + RawshiftDecoder.probe(bytes, ext), + Err(MediaError::UnsupportedFormat { + format: *format, + op: FormatOp::Decode, + }), + ); + } +} + +/// Bytes that are no still at all are a distinct outcome from a still with no codec: there is +/// nothing to backfill later. +#[test] +fn non_still_bytes_are_not_a_deferral() { + assert_eq!( + RawshiftDecoder.decode(b"\x1AE\xDF\xA3 a webm, not a photo", "webm"), + Err(MediaError::NotAStillImage) + ); +} + +/// A supported format whose bytes are broken is a **decode failure**, not a deferral — the +/// distinction the run summary reports and the one worth investigating. +#[test] +fn corrupt_bytes_of_a_supported_format_are_a_decode_failure() { + let mut jpeg = jpeg_bytes(&gradient(16, 16), None); + jpeg.truncate(jpeg.len() / 2); + match RawshiftDecoder.decode(&jpeg, "jpg") { + Err(MediaError::Decode { format, .. }) => assert_eq!(format, StillFormat::Jpeg), + other => panic!("a truncated JPEG must be a decode failure, got {other:?}"), + } + + // Not even a header: the extension is the only evidence, and it says a format we decode. + match RawshiftDecoder.decode(b"this is definitely not a jpeg", "jpeg") { + Err(MediaError::Decode { format, .. }) => assert_eq!(format, StillFormat::Jpeg), + other => panic!("garbage under a .jpeg name must be a decode failure, got {other:?}"), + } +} + +/// The decode-bomb guard: a header claiming more than the budget is refused before the decoder +/// allocates. Built as a PNG header alone, because the *point* is that no pixel data is needed +/// to trigger it — a 33-byte file must not be able to ask for 21 GB. +#[test] +fn an_oversized_header_is_refused_before_decoding() { + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&30_000u32.to_be_bytes()); + ihdr.extend_from_slice(&30_000u32.to_be_bytes()); + ihdr.extend_from_slice(&[8, 2, 0, 0, 0]); + let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); + png.extend(png_chunk(b"IHDR", &ihdr)); + + assert_eq!( + RawshiftDecoder.probe(&png, "png"), + Err(MediaError::PixelBudgetExceeded { + pixels: 900_000_000, + limit: MAX_DECODE_PIXELS, + }), + ); + assert_eq!( + RawshiftDecoder.decode(&png, "png"), + Err(MediaError::PixelBudgetExceeded { + pixels: 900_000_000, + limit: MAX_DECODE_PIXELS, + }), + "decode probes first, so the guard covers it too" + ); + // The budget is a ceiling on the frame, not on the file: a small image is unaffected. + assert!( + RawshiftDecoder + .probe(&png_bytes(&gradient(4, 4)), "png") + .is_ok() + ); +} + +/// A probe reports the header's stored dimensions and the EXIF orientation without decoding, +/// and knows which pairs are transposed on display. +#[test] +fn a_probe_reports_stored_dimensions_and_the_upright_pair() { + let frame = gradient(12, 6); + let upright = RawshiftDecoder + .probe(&jpeg_bytes(&frame, Some(&oriented(1))), "jpg") + .expect("probe"); + assert_eq!(upright.stored_dimensions, (12, 6)); + assert_eq!(upright.upright_dimensions(), (12, 6)); + assert_eq!(upright.orientation, Some(1)); + assert_eq!(upright.gamut, Gamut::Srgb); + + let rotated = RawshiftDecoder + .probe(&jpeg_bytes(&frame, Some(&oriented(6))), "jpg") + .expect("probe"); + assert_eq!(rotated.stored_dimensions, (12, 6)); + assert_eq!( + rotated.upright_dimensions(), + (6, 12), + "a quarter-turn transposes what a viewer shows" + ); +} + +/// All eight EXIF orientations, as a permutation of four marked quadrants plus the dimension +/// pair. This is the table an upright frame depends on, and it is hand-derived from the EXIF +/// definitions rather than from the transform code. +#[test] +fn every_exif_orientation_lands_upright() { + // (orientation, transposed?, (TL, TR, BL, BR) after the transform) + let table: &[(u16, bool, (u32, u32, u32, u32))] = &[ + (1, false, (0, 85, 170, 255)), // identity + (2, false, (85, 0, 255, 170)), // mirror horizontal + (3, false, (255, 170, 85, 0)), // rotate 180 + (4, false, (170, 255, 0, 85)), // mirror vertical + (5, true, (0, 170, 85, 255)), // transpose + (6, true, (170, 0, 255, 85)), // rotate 90 CW + (7, true, (255, 85, 170, 0)), // transverse + (8, true, (85, 255, 0, 170)), // rotate 90 CCW + ]; + let (width, height) = (64u32, 32u32); + let frame = quadrants(width, height); + + for &(orientation, transposed, expected) in table { + let bytes = jpeg_bytes(&frame, Some(&oriented(orientation))); + let decoded = RawshiftDecoder + .decode(&bytes, "jpg") + .unwrap_or_else(|e| panic!("orientation {orientation}: {e}")); + + assert_eq!( + decoded.orientation_applied, orientation, + "the consumed tag is recorded so a renderer does not rotate again" + ); + let expected_dims = if transposed { + (height, width) + } else { + (width, height) + }; + assert_eq!( + (decoded.width(), decoded.height()), + expected_dims, + "orientation {orientation} dimensions" + ); + + let got = quadrant_means(&decoded.image); + let close = |a: u32, b: u32| a.abs_diff(b) <= 20; + assert!( + close(got.0, expected.0) + && close(got.1, expected.1) + && close(got.2, expected.2) + && close(got.3, expected.3), + "orientation {orientation}: quadrants {got:?} are not {expected:?}" + ); + } +} + +/// An orientation value outside 1..=8 is dropped rather than recorded. `apply_orientation` +/// warns and no-ops on one, so honouring it would leave `orientation_applied` claiming a +/// transform that never happened. +#[test] +fn an_out_of_range_orientation_tag_is_ignored() { + let bytes = jpeg_bytes(&gradient(8, 8), Some(&oriented(42))); + let probed = RawshiftDecoder.probe(&bytes, "jpg").expect("probe"); + assert_eq!(probed.orientation, None); + let decoded = RawshiftDecoder.decode(&bytes, "jpg").expect("decode"); + assert_eq!(decoded.orientation_applied, 1); +} + +// ── The panic guard ────────────────────────────────────────────────────────── + +/// A [`Decoder`] that panics, and one that lies about its buffer — the two failures real bytes +/// cannot be relied on to produce. +struct HostileDecoder { + panic: bool, +} + +impl Decoder for HostileDecoder { + fn probe(&self, _bytes: &[u8], _ext: &str) -> Result { + Err(MediaError::NotAStillImage) + } + + fn decode(&self, _bytes: &[u8], _ext: &str) -> Result { + assert!( + !self.panic, + "a third-party decoder panicking on untrusted bytes" + ); + Err(MediaError::NotAStillImage) + } +} + +/// A panicking decoder becomes a reported error, never an aborted import. +#[test] +fn a_panicking_decoder_is_caught_at_the_boundary() { + let previous = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let caught = decode_guarded(&HostileDecoder { panic: true }, b"whatever", "jpg"); + std::panic::set_hook(previous); + assert_eq!(caught, Err(MediaError::DecoderPanic)); + + // The guard is transparent when nothing panics. + assert_eq!( + decode_guarded(&HostileDecoder { panic: false }, b"whatever", "jpg"), + Err(MediaError::NotAStillImage) + ); + assert!(decode_guarded(&RawshiftDecoder, &png_bytes(&gradient(4, 4)), "png").is_ok()); +} + +// ── Resize ─────────────────────────────────────────────────────────────────── + +/// The sizing rule: cap the long edge, keep the aspect ratio, never return a zero edge, and +/// leave a frame that already fits exactly as it was. +#[test] +fn capped_dimensions_preserves_aspect_and_never_returns_zero() { + assert_eq!(capped_dimensions(512, 384, 256), (256, 192)); + assert_eq!(capped_dimensions(384, 512, 256), (192, 256)); + assert_eq!(capped_dimensions(1000, 1000, 256), (256, 256)); + assert_eq!(capped_dimensions(4032, 3024, 256), (256, 192)); + // Already inside the cap: untouched, which is what lets a caller compare and skip. + assert_eq!(capped_dimensions(128, 96, 256), (128, 96)); + assert_eq!(capped_dimensions(256, 256, 256), (256, 256)); + // Extreme ratios round the short edge to 1 rather than to an empty buffer. + assert_eq!(capped_dimensions(8000, 3, 256), (256, 1)); + assert_eq!(capped_dimensions(3, 8000, 256), (1, 256)); + // A zero cap is raised to 1 rather than producing an empty frame. + assert_eq!(capped_dimensions(100, 50, 0), (1, 1)); +} + +/// The downscale is deterministic and its output is exactly `w*h*4`. Determinism is a +/// requirement, not a nicety: the derivative's bytes are content-addressed by a signed manifest. +#[test] +fn the_downscale_is_deterministic_and_correctly_sized() { + let source = gradient(512, 384); + let first = downscale_rgba8(&source, 256); + let second = downscale_rgba8(&source, 256); + assert_eq!((first.width, first.height), (256, 192)); + assert_eq!(first.rgba.len(), 256 * 192 * 4); + assert_eq!(first.rgba, second.rgba, "identical input, identical bytes"); + + // A frame within the cap comes back untouched. + let small = gradient(100, 80); + assert_eq!(downscale_rgba8(&small, 256), small); +} + +/// A box average over a flat region reproduces that region's own value exactly — the property +/// that keeps a downscaled thumbnail from drifting darker with every reduction. +#[test] +fn the_downscale_preserves_flat_regions_and_stays_opaque() { + let uniform = RgbaImage { + width: 64, + height: 64, + rgba: vec![137, 42, 200, 255].repeat(64 * 64), + }; + let out = downscale_rgba8(&uniform, 16); + assert_eq!((out.width, out.height), (16, 16)); + assert!( + out.rgba.chunks_exact(4).all(|px| px == [137, 42, 200, 255]), + "a flat region survives an area average exactly" + ); + + // The quadrant markers survive a 4x reduction, i.e. the filter is not smearing regions into + // one another beyond their own boundary. + let reduced = downscale_rgba8(&quadrants(128, 128), 32); + let means = quadrant_means(&reduced); + assert_eq!(means, (0, 85, 170, 255)); +} + +// ── Derivatives ────────────────────────────────────────────────────────────── + +/// Two signing keys and a fixed context — the epoch/authorisation material a manifest needs +/// that pixels do not carry. +fn signers() -> (HybridSigningKey, HybridSigningKey) { + ( + HybridSigningKey::from_seed_bytes(&[7; 32], &[8; 32]), + HybridSigningKey::from_seed_bytes(&[9; 32], &[10; 32]), + ) +} + +/// A real AMK-backed sealer — the same `encrypt_asset_rekey` construction the import path uses, +/// so these tests exercise the production encryption rather than a stand-in. +struct TestSealer { + amk: Amk, + asset_id: Uuid, +} + +impl DerivativeSealer for TestSealer { + fn seal(&self, plaintext: &[u8]) -> Result { + let (enc, _ciphertext, _key) = + encrypt_asset_rekey(&self.amk, &self.asset_id, plaintext, None).expect("sealing"); + Ok(SealedDerivative { + ciphertext_hash: enc.ciphertext_hash, + nonce_prefix: enc.nonce_prefix, + }) + } +} + +fn sealer(asset_id: Uuid) -> TestSealer { + TestSealer { + amk: Amk::from_bytes([0x5A; 32]), + asset_id, + } +} + +/// What the *original*'s own manifest committed to; the `original` sentinel signs exactly this. +const ORIGINAL_SEAL: SealedDerivative = SealedDerivative { + ciphertext_hash: Hash32([0xC1; 32]), + nonce_prefix: [1, 2, 3, 4, 5, 6, 7], +}; + +/// No prior chain: every test that does not say otherwise generates for a fresh asset. +fn no_prior_heads() -> HashMap { + HashMap::new() +} + +fn context<'a>( + device: &'a HybridSigningKey, + write_tier: &'a HybridSigningKey, + sealer: &'a dyn DerivativeSealer, + prior_heads: &'a HashMap, + asset_id: Uuid, +) -> DerivativeContext<'a> { + DerivativeContext { + source_asset_id: asset_id, + crypto_suite_id: CRYPTO_SUITE_ID, + protocol_version: PROTOCOL_VERSION.into(), + amk_version: AmkVersion(1), + generated_by_device: Uuid::from_u128(0xD1), + generated_by_client: "capsule-core/test".into(), + generated_at: "2026-09-01T00:00:00Z".into(), + device_signer: device, + write_tier_signer: write_tier, + sealer, + prior_heads, + original: ORIGINAL_SEAL, + } +} + +fn generate(frame: &RgbaImage, original: &[u8]) -> StillDerivatives { + let (device, write_tier) = signers(); + let seal = sealer(Uuid::from_u128(0xB1)); + let heads = no_prior_heads(); + let ctx = context(&device, &write_tier, &seal, &heads, Uuid::from_u128(0xB1)); + let decoded = RawshiftDecoder + .decode(original, "png") + .expect("the fixture decodes"); + assert_eq!( + (decoded.width(), decoded.height()), + (frame.width, frame.height) + ); + generate_still_derivatives(&decoded, &DerivativeTier::GENERATED, &ctx) + .expect("generation succeeds") +} + +/// The thumbnail tier over a source larger than the cap: real JXL bytes, a signed manifest +/// binding their hash, and the two formats this build cannot encode recorded as deferrals rather +/// than silently omitted. +#[test] +fn the_thumbnail_tier_encodes_jxl_and_defers_the_rest() { + let frame = gradient(512, 384); + let original = png_bytes(&frame); + let result = generate(&frame, &original); + + assert_eq!(result.generated.len(), 1, "one encodable format today"); + let thumb = &result.generated[0]; + assert_eq!(thumb.tier, DerivativeTier::Thumbnail); + assert_eq!(thumb.format, DerivativeFormat::Jxl); + assert_eq!(thumb.manifest.core.format, "image/jxl"); + assert_eq!(thumb.manifest.core.role, DerivativeRole::Thumbnail); + // The manifest commits to the **ciphertext**, not to the plaintext on disk. Re-derive it + // the way the push path does — from the recorded prefix under the same AMK — and the + // signed address has to come back. + let seal = sealer(Uuid::from_u128(0xB1)); + let file_key = seal + .amk + .derive_file_key(&Uuid::from_u128(0xB1), &thumb.manifest.core.nonce_prefix); + let (_, ciphertext) = crate::crypto::encryption::stream::encrypt_asset_vec_with_prefix( + &file_key, + thumb.manifest.core.nonce_prefix, + &thumb.bytes, + ); + assert_eq!( + thumb.manifest.core.ciphertext_hash, + crate::crypto::hash::hash_bytes(&ciphertext), + "the manifest binds the ciphertext the push path re-derives" + ); + assert_ne!( + thumb.manifest.core.ciphertext_hash, + crate::crypto::hash::hash_bytes(&thumb.bytes), + "and that is not the plaintext's address — a thumbnail is a recognisable copy of a \ + private photo and does not cross the network in the clear" + ); + assert_eq!( + crate::crypto::encryption::stream::decrypt_asset_vec( + &file_key, + &thumb.manifest.core.nonce_prefix, + &ciphertext + ) + .expect("the ciphertext authenticates"), + thumb.bytes, + "and it round-trips back to the bytes on disk" + ); + assert_eq!(thumb.manifest.core.version, DERIVATIVE_MANIFEST_VERSION); + assert!( + thumb.manifest.core.prior_provenance_hash.is_none(), + "first of its role" + ); + + // The bytes are a real JXL of the tier's size. + assert_eq!( + StillFormat::from_bytes(&thumb.bytes), + Some(StillFormat::Jxl) + ); + let back = RawshiftDecoder + .decode(&thumb.bytes, "jxl") + .expect("the thumbnail decodes"); + assert_eq!((back.width(), back.height()), (256, 192)); + + // The gap is per (tier, format), recorded rather than collapsed. WebP is here for a + // different reason from AVIF: not a missing toolchain but a codec that does not compile for + // aarch64, so it is deferred on every target rather than only where nasm is absent. + assert_eq!( + result.deferred, + vec![ + (DerivativeTier::Thumbnail, DerivativeFormat::Avif), + (DerivativeTier::Thumbnail, DerivativeFormat::WebP), + ] + ); + + // Both signatures verify over the canonical core. + let (device, write_tier) = signers(); + let bytes = thumb.manifest.core.signing_bytes(); + assert!( + device + .verifying_key() + .verify(&bytes, &thumb.manifest.device_sig) + ); + assert!( + write_tier + .verifying_key() + .verify(&bytes, &thumb.manifest.write_sig) + ); +} + +/// The tier's declared `q=50` is **advisory today**: `zune-jpegxl`'s `JxlSimpleEncoder` is +/// lossless, so the thumbnail round-trips its downscaled pixels exactly and costs what a +/// lossless encode costs. +/// +/// Asserted rather than left as a comment, because it is the one place this build visibly +/// departs from the tier table and the departure disappears the moment a lossy backend lands. +#[test] +fn the_jxl_thumbnail_is_lossless_today() { + let frame = gradient(512, 384); + let original = png_bytes(&frame); + let result = generate(&frame, &original); + let thumb = &result.generated[0]; + + let back = RawshiftDecoder + .decode(&thumb.bytes, "jxl") + .expect("the thumbnail decodes"); + let expected = downscale_rgba8(&gradient(512, 384), 256); + assert_eq!( + back.image, expected, + "a lossless encode reproduces the downscaled frame exactly" + ); +} + +/// A source no larger than the tier's cap takes the signed `original` sentinel — an explicit +/// marker, distinct from an absent derivative, and never a redundant re-encode. +#[test] +fn a_source_within_the_cap_signs_the_original_sentinel() { + let frame = gradient(128, 96); + let original = png_bytes(&frame); + let result = generate(&frame, &original); + + assert_eq!(result.generated.len(), 1); + let only = &result.generated[0]; + assert_eq!(only.format, DerivativeFormat::Original); + assert_eq!(only.manifest.core.format, "original"); + assert!( + only.bytes.is_empty(), + "the sentinel *references* the original rather than copying it: a copy would put the \ + source's EXIF, GPS included, into a derivative blob" + ); + assert_eq!( + only.manifest.core.ciphertext_hash, ORIGINAL_SEAL.ciphertext_hash, + "the sentinel signs the **original's** ciphertext address — the blob a receiver already \ + holds — and encrypts nothing of its own" + ); + assert_eq!( + only.manifest.core.nonce_prefix, ORIGINAL_SEAL.nonce_prefix, + "and the original's prefix, so the reference selects the same key" + ); + assert!( + result.deferred.is_empty(), + "nothing was deferred: the tier is satisfied by the original, not by a missing encoder" + ); + assert_eq!(DerivativeFormat::Original.extension(), None); +} + +/// Manifests of the same role chain by content hash over the previous one's canonical CBOR, +/// signatures included — the same append-only link the asset provenance chain uses. +/// +/// Exercised through [`sign_derivative`](super::derivative::sign_derivative) rather than +/// through [`generate_still_derivatives`], and deliberately: only JXL is encodable today, so a +/// single call produces one manifest per role and the multi-link case — the half that can +/// actually be wrong — is unreachable from the public entry point until a second encoder lands +/// (the filed `S-B1` remainder, #437). +#[test] +fn manifests_of_one_role_form_an_append_only_chain() { + let (device, write_tier) = signers(); + let seal = sealer(Uuid::from_u128(0xB2)); + let heads = no_prior_heads(); + let ctx = context(&device, &write_tier, &seal, &heads, Uuid::from_u128(0xB2)); + let mut prior = None; + + let first = super::derivative::sign_derivative( + &ctx, + DerivativeTier::Thumbnail, + DerivativeFormat::Jxl, + b"first generation bytes".to_vec(), + seal.seal(b"first generation bytes").expect("sealing"), + &mut prior, + ) + .expect("signing the first manifest"); + assert!( + first.manifest.core.prior_provenance_hash.is_none(), + "the first manifest of a role starts that role's chain" + ); + + let expected_link = crate::crypto::hash::hash_bytes( + &crate::cbor::to_canonical_vec(&first.manifest).expect("canonical CBOR"), + ); + assert_eq!( + prior, + Some(expected_link), + "the cursor advances to this manifest" + ); + + let second = super::derivative::sign_derivative( + &ctx, + DerivativeTier::Thumbnail, + DerivativeFormat::Jxl, + b"second generation bytes".to_vec(), + seal.seal(b"second generation bytes").expect("sealing"), + &mut prior, + ) + .expect("signing the second manifest"); + assert_eq!( + second.manifest.core.prior_provenance_hash, + Some(expected_link), + "the second manifest chains to the first by content hash" + ); + + // Breaking the link is detectable: the hash covers the signatures, so any edit to the first + // manifest moves the value the second one has to carry. + let mut tampered = first.manifest.clone(); + tampered.core.generated_at = "2026-09-02T00:00:00Z".into(); + let tampered_link = crate::crypto::hash::hash_bytes( + &crate::cbor::to_canonical_vec(&tampered).expect("canonical CBOR"), + ); + assert_ne!( + tampered_link, expected_link, + "a rewritten predecessor no longer matches the link its successor signed" + ); +} + +/// Each tier records its own role, so each role is its own chain rather than one interleaved +/// sequence. +#[test] +fn each_tier_starts_its_own_role_chain() { + let frame = gradient(512, 384); + let original = png_bytes(&frame); + let (device, write_tier) = signers(); + let decoded = RawshiftDecoder.decode(&original, "png").expect("decode"); + + let seal = sealer(Uuid::from_u128(0xB5)); + let both = generate_still_derivatives( + &decoded, + &[DerivativeTier::Thumbnail, DerivativeTier::Preview], + &context( + &device, + &write_tier, + &seal, + &no_prior_heads(), + Uuid::from_u128(0xB5), + ), + ) + .expect("both tiers"); + + let roles: Vec = both + .generated + .iter() + .map(|d| d.manifest.core.role) + .collect(); + assert_eq!( + roles, + vec![DerivativeRole::Thumbnail, DerivativeRole::Preview] + ); + for derivative in &both.generated { + assert!( + derivative.manifest.core.prior_provenance_hash.is_none(), + "the first manifest of each role starts that role's chain" + ); + } + + // The preview tier keeps the source resolution; only the thumbnail caps a long edge. + let previewed = both + .generated + .iter() + .find(|d| d.tier == DerivativeTier::Preview) + .expect("a preview was generated"); + let back = RawshiftDecoder + .decode(&previewed.bytes, "jxl") + .expect("the preview decodes"); + assert_eq!((back.width(), back.height()), (512, 384)); +} + +/// Two derivatives of one asset get **distinct** nonce prefixes, and therefore distinct keys and +/// distinct ciphertexts, even when their plaintext is byte-identical. +/// +/// This is the property that makes per-derivative sealing worth doing rather than reusing the +/// original's key: a shared prefix would reuse a keystream across two blobs under one file key, +/// which is the failure the encryption doc's per-file derivation exists to prevent. +#[test] +fn two_derivatives_of_one_asset_never_share_a_nonce_prefix() { + let (device, write_tier) = signers(); + let asset_id = Uuid::from_u128(0xB6); + let seal = sealer(asset_id); + let heads = no_prior_heads(); + let ctx = context(&device, &write_tier, &seal, &heads, asset_id); + let identical = b"byte-identical derivative plaintext".to_vec(); + + let mut prior = None; + let first = super::derivative::sign_derivative( + &ctx, + DerivativeTier::Thumbnail, + DerivativeFormat::Jxl, + identical.clone(), + seal.seal(&identical).expect("sealing"), + &mut prior, + ) + .expect("first"); + let mut prior = None; + let second = super::derivative::sign_derivative( + &ctx, + DerivativeTier::Preview, + DerivativeFormat::Jxl, + identical.clone(), + seal.seal(&identical).expect("sealing"), + &mut prior, + ) + .expect("second"); + + assert_eq!( + first.bytes, second.bytes, + "the plaintext really is identical" + ); + assert_ne!( + first.manifest.core.nonce_prefix, second.manifest.core.nonce_prefix, + "a fresh prefix is drawn per derivative" + ); + assert_ne!( + first.manifest.core.ciphertext_hash, second.manifest.core.ciphertext_hash, + "so identical plaintext does not produce a shared ciphertext or a shared key" + ); +} + +/// **The privacy case.** A thumbnail must not inherit the source's EXIF, and above all not its +/// GPS fix. +/// +/// The control half is what makes this a real test: the crate's *default* is to embed +/// everything, so the same frame encoded the way `MetadataEmbedOptions::default()` would encode +/// it does carry the fix. Capsule's derivative does not. +#[test] +fn a_thumbnail_carries_no_exif_and_no_gps() { + let frame = gradient(512, 384); + let located_metadata = located(); + let source = jpeg_bytes(&frame, Some(&located_metadata)); + + // The source really does carry the fix — otherwise the assertion below proves nothing. + assert!( + contains(&source, b"Exif\0\0"), + "the fixture JPEG must carry an EXIF APP1 segment" + ); + assert!( + gps_rationals_present(&source), + "the fixture JPEG must carry the GPS rationals" + ); + + // The control: the same frame through the same JXL backend, configured the way the crate's + // own `MetadataEmbedOptions::default()` would configure it. It leaks — which is what makes + // the assertion below a test of Capsule's strip rather than of a codec that never embeds. + let leaky = jxl_bytes(&frame, Some(&located_metadata)); + assert!( + gps_rationals_present(&leaky), + "the control must leak, or this test is not testing the strip" + ); + + // Capsule's derivative, over the same GPS-bearing source. + let (device, write_tier) = signers(); + let decoded = RawshiftDecoder.decode(&source, "jpg").expect("decode"); + let seal = sealer(Uuid::from_u128(0xB3)); + let result = generate_still_derivatives( + &decoded, + &DerivativeTier::GENERATED, + &context( + &device, + &write_tier, + &seal, + &no_prior_heads(), + Uuid::from_u128(0xB3), + ), + ) + .expect("generation"); + let thumb = &result.generated[0].bytes; + assert!(!thumb.is_empty(), "the thumbnail has bytes to inspect"); + assert!(!contains(thumb, b"Exif"), "no EXIF box in the thumbnail"); + assert!(!contains(thumb, b"xml "), "no XMP box"); + assert!(!contains(thumb, b"jumb"), "no metadata container box"); + assert!( + !gps_rationals_present(thumb), + "the GPS rationals must not survive into a thumbnail" + ); +} + +/// Whether `needle` appears anywhere in `haystack`. +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) +} + +/// Whether the fixture's GPS degrees/minutes/seconds appear as EXIF rationals — a +/// `numerator/denominator` pair per component, in either byte order. +fn gps_rationals_present(bytes: &[u8]) -> bool { + let rational = |value: u32| -> [Vec; 2] { + let mut be = value.to_be_bytes().to_vec(); + be.extend_from_slice(&1u32.to_be_bytes()); + let mut le = value.to_le_bytes().to_vec(); + le.extend_from_slice(&1u32.to_le_bytes()); + [be, le] + }; + // The seconds component of each coordinate is the most distinctive value; requiring both + // keeps an incidental byte match from reading as a leak. + let has = |value: u32| rational(value).iter().any(|n| contains(bytes, n)); + has(GPS_LAT_DMS[2]) && has(GPS_LON_DMS[2]) +} + +// ── The closed format set ──────────────────────────────────────────────────── + +/// `mime` and `parse` are inverses over the whole closed set, and nothing outside it parses. +#[test] +fn the_closed_format_set_round_trips_and_admits_nothing_else() { + for format in [ + DerivativeFormat::Jxl, + DerivativeFormat::Avif, + DerivativeFormat::WebP, + DerivativeFormat::Original, + ] { + assert_eq!(DerivativeFormat::parse(format.mime()), Some(format)); + assert!(DerivativeFormat::is_recognized(format.mime())); + } + for rejected in [ + "image/future-codec", + "image/jpeg", + "image/png", + "IMAGE/WEBP", + "original ", + "", + "embedding/mobileclip-b", + ] { + assert!( + !DerivativeFormat::is_recognized(rejected), + "{rejected:?} is outside the closed set" + ); + } + // Only JXL — the table's committed *master* format — and the sentinel can be produced here. + // AVIF is blocked on a build-host assembler; WebP is blocked on an upstream defect, its + // codec not compiling for aarch64 at all. + assert!(DerivativeFormat::Jxl.is_encodable()); + assert!(DerivativeFormat::Original.is_encodable()); + assert!(!DerivativeFormat::Avif.is_encodable()); + assert!(!DerivativeFormat::WebP.is_encodable()); +} + +/// A signed still-role manifest whose `format` is outside the closed set is rejected at +/// verification, and an embedding-role manifest — which writes `embedding/{model_id}` into the +/// same field — is not caught in the crossfire. +#[test] +fn verification_rejects_an_unrecognised_still_format() { + let (device, write_tier) = signers(); + let sign = |role: DerivativeRole, format: &str| -> DerivativeManifest { + DerivativeCore { + version: DERIVATIVE_MANIFEST_VERSION.into(), + crypto_suite_id: CRYPTO_SUITE_ID, + protocol_version: Some(PROTOCOL_VERSION.into()), + amk_version: Some(AmkVersion(1)), + source_asset_id: Uuid::from_u128(0xB4), + role, + format: format.into(), + ciphertext_hash: crate::crypto::hash::hash_bytes(b"bytes"), + nonce_prefix: [9, 8, 7, 6, 5, 4, 3], + generated_by_device: Uuid::from_u128(0xD1), + generated_by_client: "capsule-core/test".into(), + model_id: None, + model_version: None, + generated_at: "2026-09-01T00:00:00Z".into(), + prior_provenance_hash: None, + } + .sign(&device, &write_tier) + .expect("signing") + }; + + assert_eq!( + verify_still_format(&sign(DerivativeRole::Thumbnail, "image/webp")), + Ok(Some(DerivativeFormat::WebP)) + ); + assert_eq!( + verify_still_format(&sign(DerivativeRole::Preview, "original")), + Ok(Some(DerivativeFormat::Original)) + ); + assert_eq!( + verify_still_format(&sign(DerivativeRole::Thumbnail, "image/future-codec")), + Err("image/future-codec".to_string()), + "an unrecognised still format is a structural rejection" + ); + assert_eq!( + verify_still_format(&sign(DerivativeRole::Embedding, "embedding/mobileclip-b")), + Ok(None), + "the embedding-role grammar is not this set's business" + ); +} + +// ── The LQIP producer ──────────────────────────────────────────────────────── + +/// A decoded frame is exactly what the unconditional LQIP encoder takes — the reason +/// [`DecodedImage`](super::DecodedImage) carries a [`RgbaImage`] rather than its own buffer +/// type. +#[test] +fn a_decoded_frame_encodes_an_lqip_at_the_committed_width() { + let frame = gradient(200, 150); + let decoded = RawshiftDecoder + .decode(&png_bytes(&frame), "png") + .expect("decode"); + let lqip = Lqip::encode( + decoded.width(), + decoded.height(), + &decoded.image.rgba, + decoded.gamut, + ) + .expect("a decoded frame is a valid LQIP source"); + assert_eq!(lqip.as_bytes().len(), 32, "DEFAULT_TIER is 32 bytes"); + assert_eq!( + lqip.to_sidecar().format_version, + crate::lqip::LQIP_FORMAT_V1 + ); + + // The placeholder is computed from the full-resolution frame, not from the thumbnail: + // chromahash band-limits on the read side, so pre-resizing would cap fidelity. + let thumb = downscale_rgba8(&decoded.image, 256); + let from_thumb = Lqip::encode(thumb.width, thumb.height, &thumb.rgba, decoded.gamut) + .expect("a downscaled frame also encodes"); + assert_eq!( + from_thumb.as_bytes().len(), + 32, + "the tier is fixed regardless of the source size" + ); +} + +/// The per-channel accumulator genuinely crosses `u32::MAX`, and the boundary arithmetic is +/// documented for what it is. +/// +/// **The accumulator overflow is real and reachable.** Reducing a frame to a cap of 1 sums every +/// sample into one destination pixel, so the running total is `pixels * 255`. At 4200x4200 that +/// is 4.5e9 — past `u32::MAX` (4.29e9) — and a `u32` accumulator would panic in debug and wrap +/// into wrong pixels in release. `downscale_rgba8` is a `pub` entry point, so a cap of 1 is +/// reachable even though the tier table only ever passes 256. +/// +/// **The boundary product is defensive, not demonstrated.** `(y + 1) * src_h` reaches +/// `dst_edge * src_edge`, which for the shapes this build actually produces (a 256 px cap) stays +/// far inside 32 bits. It is computed in `u64` anyway because the function is public and its +/// inputs are not bounded by the tier table — but the earlier claim that a 1x300000 frame +/// crossed `u32::MAX` was simply wrong arithmetic (7.7e7, not 7.7e10), and a test asserting a +/// false reason is worse than no test. +#[test] +fn the_downscale_accumulator_survives_crossing_u32_max() { + // 4200 * 4200 * 255 = 4_501_980_000 > u32::MAX. + let edge = 4200u32; + let pixels = u64::from(edge) * u64::from(edge); + assert!( + pixels * 255 > u64::from(u32::MAX), + "the fixture must actually cross the boundary it exists to test" + ); + + let flat = RgbaImage { + width: edge, + height: edge, + rgba: vec![255, 255, 255, 255].repeat((edge * edge) as usize), + }; + let single = downscale_rgba8(&flat, 1); + assert_eq!((single.width, single.height), (1, 1)); + assert_eq!( + single.rgba, + vec![255, 255, 255, 255], + "every sample sums into one pixel, and the mean is still 255" + ); + + // The tall-frame shape, kept because it is the one the tier path can actually meet: a + // lopsided source reduced to a 256 px long edge. + let tall = RgbaImage { + width: 1, + height: 300_000, + rgba: vec![200, 100, 50, 255].repeat(300_000), + }; + let reduced = downscale_rgba8(&tall, 256); + assert_eq!((reduced.width, reduced.height), (1, 256)); + assert!( + reduced + .rgba + .chunks_exact(4) + .all(|px| px == [200, 100, 50, 255]), + "a flat frame survives a 1172x row reduction exactly" + ); +} + +/// A sealer that refuses is a **workspace** fault and must be distinguishable at the type level +/// from a codec that refuses, because the import path propagates one and degrades on the other. +/// +/// This is the `F1` contract: `MediaError::Sign` is its own variant precisely so +/// `prepare_still` can tell "this asset has no thumbnail" from "this workspace cannot author a +/// signed record", instead of string-matching both into one `LifecycleError::Io`. +#[test] +fn a_signing_fault_is_its_own_variant_not_an_encode_failure() { + struct RefusingSealer; + impl DerivativeSealer for RefusingSealer { + fn seal(&self, _plaintext: &[u8]) -> Result { + Err(MediaError::Sign { + detail: "the hardware signer refused".into(), + }) + } + } + + let frame = gradient(512, 384); + let original = png_bytes(&frame); + let (device, write_tier) = signers(); + let decoded = RawshiftDecoder.decode(&original, "png").expect("decode"); + + let error = generate_still_derivatives( + &decoded, + &DerivativeTier::GENERATED, + &context( + &device, + &write_tier, + &RefusingSealer, + &no_prior_heads(), + Uuid::from_u128(0xB8), + ), + ) + .expect_err("a refusing sealer fails generation"); + assert!( + matches!(error, MediaError::Sign { .. }), + "a signing/sealing refusal keeps its own identity all the way out: {error:?}" + ); +} + +/// **A role's chain continues across generations.** A second run over an asset that already has +/// a thumbnail extends that role's chain rather than starting a parallel one. +/// +/// This is what makes derivative provenance append-only *in time* rather than merely within one +/// call, and it is the property a `#437` backfill depends on: adding the AVIF variant to an +/// asset that already has JXL must not fork the record. A forked chain is not something a later +/// run can repair, so it is asserted before the backfill exists. +#[test] +fn a_roles_chain_continues_across_generation_runs() { + let frame = gradient(512, 384); + let original = png_bytes(&frame); + let (device, write_tier) = signers(); + let asset_id = Uuid::from_u128(0xB7); + let seal = sealer(asset_id); + let decoded = RawshiftDecoder.decode(&original, "png").expect("decode"); + + // First generation: the role's chain starts. + let first = generate_still_derivatives( + &decoded, + &DerivativeTier::GENERATED, + &context(&device, &write_tier, &seal, &no_prior_heads(), asset_id), + ) + .expect("first run"); + let head = &first.generated[0].manifest; + assert!( + head.core.prior_provenance_hash.is_none(), + "the first manifest of a role starts that role's chain" + ); + + // What a reader would compute from the persisted bundle. + let link = crate::crypto::hash::hash_bytes( + &crate::cbor::to_canonical_vec(head).expect("canonical CBOR"), + ); + let mut heads = HashMap::new(); + heads.insert(DerivativeRole::Thumbnail, link); + + // Second generation, handed that head. + let second = generate_still_derivatives( + &decoded, + &DerivativeTier::GENERATED, + &context(&device, &write_tier, &seal, &heads, asset_id), + ) + .expect("second run"); + assert_eq!( + second.generated[0].manifest.core.prior_provenance_hash, + Some(link), + "the second run extends the chain instead of forking it" + ); + + // A role with no recorded head still starts cleanly — the map is a lookup, not a gate. + let preview = generate_still_derivatives( + &decoded, + &[DerivativeTier::Preview], + &context(&device, &write_tier, &seal, &heads, asset_id), + ) + .expect("preview run"); + assert!( + preview.generated[0] + .manifest + .core + .prior_provenance_hash + .is_none(), + "a role the map does not mention starts its own chain" + ); +} + +/// A **panicking sealer/encoder** is caught at the same boundary a panicking decoder is, so one +/// bad frame cannot abort a 20,000-photo import part way through. +/// +/// The decode guard was never the whole story: `chromahash` and the JXL encoder are both pre-1.0 +/// too, and they run *after* the decoder on the same untrusted pixels. This mirrors +/// [`HostileDecoder`] on the encode side. +#[test] +fn a_panicking_encoder_is_caught_like_a_panicking_decoder() { + struct PanickingSealer; + impl DerivativeSealer for PanickingSealer { + fn seal(&self, _plaintext: &[u8]) -> Result { + panic!("a pre-1.0 codec panicking on a frame the decoder accepted"); + } + } + + let frame = gradient(512, 384); + let original = png_bytes(&frame); + let (device, write_tier) = signers(); + let decoded = RawshiftDecoder.decode(&original, "png").expect("decode"); + + let previous = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let caught = super::guarded("derivatives", || { + generate_still_derivatives( + &decoded, + &DerivativeTier::GENERATED, + &context( + &device, + &write_tier, + &PanickingSealer, + &no_prior_heads(), + Uuid::from_u128(0xB9), + ), + ) + }); + std::panic::set_hook(previous); + + assert!( + matches!(caught, Err(MediaError::DecoderPanic)), + "an unwind from anywhere in generation becomes a reported error, never an abort" + ); +} diff --git a/capsule-docs/planned-modules.txt b/capsule-docs/planned-modules.txt index d78427f0..f162da0f 100644 --- a/capsule-docs/planned-modules.txt +++ b/capsule-docs/planned-modules.txt @@ -12,7 +12,7 @@ # there. Everything here is a commitment nobody has met yet — read it as the # module-layer answer to "what has been designed and not built?" -capsule-core::media The Capsule-side owner of decode, metadata extraction and derivative generation, which will consume Rawshift once Rawshift stabilizes. Rawshift is a pinned submodule today and is not a workspace dependency, so nothing consumes it and this module has no body to write yet. Lane B in SLICES.md. +capsule-core::media::video The video half of the media module: first-frame still extraction and the H.264 baseline preview transcode. The still half now exists (`capsule-core::media`, slice S-B1 on rawshift-image 0.1.1); this does not, because `rawshift-video` is unpublished and the transcode toolchain touches nothing the still path does. Contract: design/thumbnails.md § Video Previews. Slice S-B5 in SLICES.md. capsule-core::notify Alert classes and their trigger predicates, so every platform evaluates one shared decision function rather than reimplementing the taxonomy. Contract: design/notifications.md. Tier 0 has no server half, so this is client-only work. capsule-core::import::camera The PTP/IP tethered-camera source adapter (S-B9). Post-v1; the contract exists so the adapter seam is fixed before anything implements it. capsule-server::federation Server-to-server federation pull. The whole surface is post-v1 — `capsule-server` has no federation route, no capability-token verifier and no per-peer budget enforcement. diff --git a/capsule-docs/src/content/docs/design/dependencies.md b/capsule-docs/src/content/docs/design/dependencies.md index 548962bc..ed5fd6f1 100644 --- a/capsule-docs/src/content/docs/design/dependencies.md +++ b/capsule-docs/src/content/docs/design/dependencies.md @@ -40,6 +40,7 @@ Mechanically, every Rust version is pinned once in the root `Cargo.toml` `[works | ORM | `sea-orm` (`sqlx-postgres` on the server, `sqlx-sqlite` in the CLI) | The rebuildable index databases only — sidecars stay canonical per [Principles](/design/principles/). | — | | Embedded SQLite | `rusqlite` (`bundled`) | `capsule-core`'s `library.sqlite`. | — | | Vector index | `sqlite-vec` (`vec0`) | The client-local embedding index in `capsule-core`'s `library.sqlite` — per-task `vec0` virtual tables under the [embedding-provenance](/design/ai/#embedding-provenance) invariant. Optional + `native`-gated alongside `rusqlite` (registers as a SQLite auto-extension; not `wasm32`). | Server-side vector-DB idioms (pgvector/HNSW) do not apply — the index is client-local SQLite by design. | +| Still decode / encode | `rawshift-image` **0.1.1** (`default-features = false`, features `jpeg-decode`, `png-decode`, `jxl`, `tiff-decode`, `gif-decode`; the JPEG/PNG **encoders** ride `[dev-dependencies]`, so `jpeg-encoder` and its conjunctive IJG arm stay out of every release binary while remaining in the graph `cargo deny --all-features` inspects — the exception and its NOTICE section stay matched rather than going stale) | `capsule-core::media` behind the `media` feature, which `native` implies (slices `S-B1`, `S-B13`) — format sniffing, pixel decode, EXIF orientation and the derivative byte encode. A **registry** dependency, not the pinned `rawshift/` submodule: that tree is an uninitialised newer v1-in-progress checkout and not a workspace member. Depended on directly rather than through the `rawshift` facade because only the per-crate dependency gives per-format Cargo control, which the crate's own docs recommend and which this row needs — the format set is a licence, build-host and **portability** decision, not a convenience. **Every enabled codec is pure Rust and links no C**: zune for JPEG/PNG decode and encode, `jxl-oxide` for JXL decode, `zune-jpegxl` for the JXL encode that produces the thumbnail tier, plus `tiff` and `gif`. MPL-2.0 (with `rawshift-core`), already allow-listed in `deny.toml`; both are named in the root `NOTICE` MPL list. `jpeg-encoder`'s conjunctive IJG arm was already excepted and is matched again by this row. Tiers, quality and the closed format set are the contract at [Thumbnails](/design/thumbnails/); this row owns the pin. | **`webp` is absent because it does not compile, not because it was not wanted.** It was the first choice — `image/webp` is in the format table and `libwebp` has the exact q=50 knob — but `rawshift-image`'s WebP module passes `*const i8` where `libwebp-sys` 0.14.4 declares `*const c_char`, and `c_char` is `u8` on aarch64, so it is an E0308 on every 64-bit ARM target; the module is compiled by decode *or* encode, so decode-only does not escape it. Every mobile target is aarch64, so enabling it would mean thumbnails on desktop and none on a phone. **Also deliberately absent**, each a toolchain rather than a design gap: `heic` (system libheif), `avif` (`image`'s `avif-native` -> system libdav1d for decode; `ravif` -> `rav1e/asm` -> `nasm` on every x86_64 build host for encode), `svg` (resvg), and the RAW families (`experimental`/`raw-stabilizing`; Canon CR3 pixel decode is unimplemented upstream). And the JXL encode is **lossless** — `zune-jpegxl`'s `JxlSimpleEncoder` — so a thumbnail costs more bytes than the table's q=50 intends; a lossy JXL needs C libjxl (`bindgen` + `pkg-config`). Every one of these is a typed `media::MediaError::UnsupportedFormat` or a recorded per-format deferral, never a silent gap. **Not** on the wasm32 sealing surface: `media` is absent from the `--no-default-features` build, so `cargo tree --target wasm32-unknown-unknown -i rawshift-image` is empty. Rawshift must never wrap Chromahash (`AGENTS.md`); see the LQIP row below. | | LQIP placeholder codec | `chromahash` **0.7.1** | `capsule-core::lqip` (slice `S-B14`) — the only encoder/decoder for the signed sidecar `lqip` field. Imported **directly**, never through Rawshift (`AGENTS.md`), and deliberately outside `capsule-core::media` — the Rawshift-consuming module — so one implementation serves the import pipeline, the uniffi FFI, and `capsule-wasm`. The tier, byte width and versioned fallback are the contract at [Thumbnails — LQIP](/design/thumbnails/#lqip); this row owns only the pin. The `AGENTS.md` gate that read "after its v1 release" is **amended to 0.7.1** — the release the project accepts as ready — and `xtask`'s architecture check stopped forbidding the crate in `2f8beeb`, because a check that forbids an approved dependency has stopped describing a decision and started blocking one. | **`thumbhash` is retired, not excepted.** The Rust crate behind `capsule-core`'s `media` feature and the npm package in `capsule-web` both go; `thumbhash` stays in the architecture check's retired-dependency list so it cannot return. BlurHash was never adopted. | | Free-space probe | `rustix` (Unix, `fs`) + `windows-sys` (Windows, `Win32_Storage_FileSystem`) | `capsule-core::library::available_bytes` — the streaming-import free-space probe (`statvfs` / `GetDiskFreeSpaceEx`). Host-only, behind the `native` feature; the wasm32 sealing build links neither. | — | | Windows TPM (TBS) | `windows-sys` (Windows, `Win32_System_TpmBaseServices`) | `capsule-core::crypto::keys::tbs` — the Windows device-key `HardwareSigner` (slice S-F4). The raw TPM 2.0 command channel (`Tbsi_Context_Create` / `Tbsip_Submit_Command`) the tss-esapi reference (`crypto::keys::tpm`, Linux) wraps; links `tbs.dll` via raw-dylib, so no new crate — an extra feature on the existing `windows-sys` row. `#[cfg(windows)]`-gated; the pure wire codec + mock tests run on any host. | Not tss-esapi on Windows: TBS is native and avoids the `libtss2`/bindgen build. | diff --git a/capsule-docs/src/content/docs/design/module-map.md b/capsule-docs/src/content/docs/design/module-map.md index 0f50199e..d5537039 100644 --- a/capsule-docs/src/content/docs/design/module-map.md +++ b/capsule-docs/src/content/docs/design/module-map.md @@ -159,3 +159,26 @@ covers (`rg "E2E case N"`), and slices in the repo-root `SLICES.md` reference th user's native client decapsulates, rewraps the key under the album AMK, and adopts it in place → the asset appears in the library and `verify_asset`-accepts on a second device. The only case exercising the web/WASM client and the wrapped-key path. + +### Status + +Every landed case is a named test (`rg "E2E case N"`); the server-side cases run in the +`capsule-e2e` crate against the real composition root (`boot::assemble` under the memory +profile, bound to an ephemeral port) with the real SDK and a real library, no container and no +environment gate. "Blocked on" names the issue that holds the rest of the case's wording. + +| Case | Named test | Status | Blocked on | +| --- | --- | --- | --- | +| 1 | `capsule-e2e/tests/case_01_auth_sync_query.rs` | landed (harness supplies the provenance rung; #464) | #464 | +| 2 | `capsule-e2e/tests/case_02_import_upload_finalize.rs` | landed (harness supplies the provenance rung; #464) | #464 | +| 3 | server half `capsule-server/tests/sync.rs`; client half `capsule-e2e/tests/case_03_sync_pickup.rs` | landed (harness supplies the provenance rung; #464) | #464; B's `verify_asset` needs the album keys (cases 6, 12) | +| 4 | — | not started | federation (#406) | +| 5 | `capsule-sdk/src/peering/tests.rs` | in-process shape | live two-host shape, post-v1 | +| 6 | `capsule-e2e/tests/case_06_backup_restore.rs` | landed; the restored asset reads and its chain walks | #467 (open as the recovered account), #468 (verify: no authority in the artifact) | +| 7 | `capsule-e2e/tests/case_07_lifecycle.rs` | landed (harness supplies the provenance rung; #464) | #464 | +| 8 | server leg `capsule-e2e/tests/case_08_upgrade_ceremony.rs`; ceremony `capsule-core/src/crypto/authority/openmls_authority/tests.rs` | server leg landed; ceremony in-process | a library cannot sign an intent (private DSK) | +| 9 | `capsule-e2e/tests/protocol_contract.rs` | landed (the UI leg is out of scope) | — | +| 10 | `capsule-core/tests/model_regen_e2e.rs` (`E2E case 10`) | landed | — | +| 11 | | lands with #447 (an in-memory fault decorator on the index) | the process-restart variant (#447 defers it) | +| 12 | server leg `capsule-e2e/tests/case_12_enrollment.rs` | server leg landed; verified-channel half unrepresented (#471) | #471, #467, #405 (MLS join) | +| 13 | server leg `capsule-e2e/tests/case_13_web_drop_adopt.rs`; seal KAT `capsule-core/tests/drop_adopt_kat.rs` | server leg landed to the durable adopted original | #469 (adopt registers nothing to publish) | diff --git a/capsule-docs/src/content/docs/design/thumbnails.md b/capsule-docs/src/content/docs/design/thumbnails.md index 55c211f2..1f54ea23 100644 --- a/capsule-docs/src/content/docs/design/thumbnails.md +++ b/capsule-docs/src/content/docs/design/thumbnails.md @@ -29,6 +29,35 @@ Two derivative tiers per photo asset and one preview tier for video assets: - **WebP** is the last-resort fallback for the rare client lacking AVIF. We deliberately do not fall back to JPEG — WebP covers everything JPEG would. - **H.264 baseline** for video previews — universally decodable, cheap to decode on every platform. AV1 was considered but mobile encode cost is still high in 2026. +:::note[Implementation status — what ships today] +The table above is the **contract**, not an inventory of what is built. As of `#410`, +`capsule-core::media` (on `rawshift-image` 0.1.1, behind the `media` feature that `native` +implies) generates the **thumbnail tier as JXL** and nothing else. Concretely: + +| Tier | Photo formats generated | Missing, and why | +| --- | --- | --- | +| Thumbnail | **JXL**, 256 px long edge, **lossless**; or the `original` sentinel when the source is already inside the cap | **AVIF** needs `nasm` on every x86_64 build host (`ravif` → `rav1e/asm`). **WebP** does not compile: `rawshift-image`'s codec passes `*const i8` where `libwebp-sys` declares `*const c_char`, which is `u8` on aarch64 — every mobile target. | +| Preview | — | Blocked with a *lossy* master codec: a source-resolution lossless still would rival the original in size. | +| Video (either tier) | — | `rawshift-video` is unpublished; slice `S-B5`. | + +The thumbnail's declared **q=50 is advisory today**: the pure-Rust backend is `zune-jpegxl`'s +`JxlSimpleEncoder`, which is lossless, so a thumbnail costs more bytes than the table intends. A +lossy JXL needs C libjxl. That is the one place the build knowingly departs from this table, and +it is asserted by a test rather than left to be discovered. + +Decode is JPEG, PNG, JXL, TIFF and GIF. **HEIC, AVIF, WebP and the RAW families are recognised +and refused** — HEIC and AVIF need system libheif / libdav1d, and WebP shares the aarch64 defect +above in both directions (the crate compiles that module for decode *or* encode). + +**Generated is not yet uploaded.** The server's upload policy fixes a closed content-type set for the protocol version, and `image/jxl` is not in it, so the T1 session for a JXL thumbnail is refused with `error.upload.unsupported_content_type` — every still larger than the 256 px cap, since only a small one gets the byte-free `original` sentinel. That is [#470](https://github.com/Capsulsaurus/Capsule/issues/470), server-side and being fixed separately; the thumbnails on disk and their signed manifests are unaffected. + +None of this is silent. A format with no codec is a typed +`media::MediaError::UnsupportedFormat { format, op }`, and a `(tier, format)` pair with no encoder +is recorded on `media::StillDerivatives::deferred` and counted by +`ImportExecutionSummary::deferred_format_count()` — so the distance between this table and the +build is a number the import run reports. The remainder is tracked as the `S-B1` follow-up. +::: + ### Video Previews The table above stays the SSoT for the video formats; this section only names the implementation seam. Video derivative generation — the first-frame still and the H.264 baseline preview transcode — is its own implementation slice (`S-B5` in the repo-root `SLICES.md`), split from still-image generation (`S-B1`) because transcode brings a distinct toolchain (demux, video decode, H.264/AAC encode) the still path never touches. Both slices sign their outputs identically through the [`DerivativeManifest`](#derivative-provenance) path. @@ -56,7 +85,7 @@ Four calls carry the whole contract, and the module uses no more than these: ### Where LQIP Lives -`capsule-core::lqip` — a dedicated module, slice `S-B14` in the repo-root `SLICES.md`. It is deliberately **not** in `capsule-core::media`, which retires to `legacy-review/` with the rest of the decode/encode stack: a placeholder scheme every client depends on cannot live inside something scheduled for teardown. It is equally not in Rawshift — `AGENTS.md` is explicit that Rawshift owns media decoding but must not wrap Chromahash, which Capsule imports directly. +`capsule-core::lqip` — a dedicated module, slice `S-B14` in the repo-root `SLICES.md`. It is deliberately **not** in `capsule-core::media`, and the reason outlived the teardown that first prompted it: `media` is `native`-only wherever it exists — it links image codecs and a decode budget sized for a workstation, neither of which a browser has any use for, so a placeholder scheme every client depends on cannot live inside it and still reach the browser. It is equally not in Rawshift — `AGENTS.md` is explicit that Rawshift owns media decoding but must not wrap Chromahash, which Capsule imports directly. `media` is the module that *produces* the pixels this one hashes; it never owns the hash. A small Capsule-owned module outside the retiring stack satisfies both constraints at once, and is reachable from all three places a placeholder is produced or consumed: the import pipeline, the native apps through the uniffi FFI, and the browser through `capsule-wasm`. That is the point of a single home — one implementation for every surface, so a photo's placeholder does not depend on which client happened to import it. @@ -79,6 +108,8 @@ Thumbnails and previews are *ephemeral by recovery posture* (they can always be The full derivative manifest structure and the `derivative-add` / `derivative-replace` action set are owned by [Cryptography — Derivative Provenance](/design/cryptography/provenance/#derivative-provenance) and [Authorization — The Closed Action Set](/design/authorization/#the-closed-action-set); this doc owns only the *format* of the derivative bytes. The two interact at exactly one point: the `DerivativeManifest.format` field names the codec/format from the table above, and the verifying side rejects a manifest whose `format` is not currently recognized (the closed-enum rule from [Threat Model — Schema Rules](/design/threat-model/schema-rules/#schema-evolution-and-field-grammar)). +A receiver never needs a manifest for a tier that was satisfied by the original itself. The signed sidecar carries the asset's pixel `dimensions`, so a receiver can see for itself that an original at or below the thumbnail tier's long edge needs no thumbnail, and must not schedule a regeneration for one; the `format = "original"` sentinel is a **local** record that keeps "this original is small" apart from "this thumbnail is missing" for the client's own rebuild path, and it is not shipped. + A thumbnail whose `DerivativeManifest` fails verification is **regenerated locally from the original** rather than trusted — the [recovery-first principle](/design/principles/) means a derivative is always rebuildable, so refusal-and-regenerate is the safe default. The corrupt copy is discarded (not quarantined — it carries no irreplaceable bytes), and the corresponding regeneration appends a new `derivative-replace` provenance record. ## Validation diff --git a/capsule-e2e/Cargo.toml b/capsule-e2e/Cargo.toml new file mode 100644 index 00000000..5b9d3caa --- /dev/null +++ b/capsule-e2e/Cargo.toml @@ -0,0 +1,42 @@ +# The bounded E2E surface (design/module-map.md, "E2E Test Surface"): the real `capsule-sdk` +# and the real `capsule-core` library driven over TCP against the real `capsule-server` +# composition root. A test crate rather than a `tests/` directory of one of those crates because +# the cases need all of them at once — case 1 alone needs the CLI's sync orchestration, its +# migrations and sea-orm, none of which belong in the server's or the SDK's dev graph. +# +# Every dependency below is already in `Cargo.lock`; this crate adds no external crate. +[package] +name = "capsule-e2e" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "End-to-end cases over the real SDK, library and server composition root" + +[lib] +name = "capsule_e2e" +path = "src/lib.rs" +doctest = false + +[dependencies] +capsule-core = { path = "../capsule-core" } +capsule-sdk = { path = "../capsule-sdk" } +# The lib target only: `boot::assemble` is the composition root `serve --memory` runs. +capsule-server = { path = "../capsule-server" } +# `server` binds the assembled service to an ephemeral port; the workspace pin carries only the +# OpenAPI feature, and the server crate itself has `server` as a dev-dependency feature. +kynos = { workspace = true, features = ["server"] } +base64 = { workspace = true } +jiff = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tempfile = "3" +tokio = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true, features = ["v7"] } + +[dev-dependencies] +# Case 1's client leg is the CLI's own `remote::sync` / `remote::list` over its SQLite store. +capsule-cli = { path = "../capsule-cli" } +capsule-cli-migration = { path = "../capsule-cli/migration" } +sea-orm = { workspace = true, features = ["sqlx-sqlite", "runtime-tokio-rustls"] } diff --git a/capsule-e2e/src/fixtures.rs b/capsule-e2e/src/fixtures.rs new file mode 100644 index 00000000..7c629d24 --- /dev/null +++ b/capsule-e2e/src/fixtures.rs @@ -0,0 +1,179 @@ +//! Byte-built fixtures, so the repository carries no binary test asset. + +/// A real 8×8 grayscale baseline JPEG carrying an EXIF APP1 segment, built byte by byte. +/// +/// The same construction as the CLI's import round trip +/// (`capsule-cli/tests/import_round_trip.rs`), which a test crate cannot import. An 8×8 still +/// sits inside the thumbnail tier's 256-pixel cap, so the media stack signs the byte-free +/// `original` sentinel for it and the upload bundle carries no derivative bytes — the cheap +/// fixture for every case that is not about derivatives; [`large_synthetic_jpeg`] is the one +/// that produces a real thumbnail. +#[must_use] +pub fn synthetic_jpeg() -> Vec { + synthetic_jpeg_sized(8, 8) +} + +/// A 512×512 still of the same construction: past the thumbnail tier's long-edge cap, so the +/// media stack decodes it and encodes a real JXL thumbnail for the upload ladder's T1 — the +/// still E2E case 2 pushes. +#[must_use] +pub fn large_synthetic_jpeg() -> Vec { + synthetic_jpeg_sized(512, 512) +} + +/// A grayscale baseline JPEG of `width`×`height` (each a multiple of 8) carrying an EXIF APP1 +/// segment, built byte by byte. +/// +/// The EXIF block is a big-endian TIFF structure with three IFDs — IFD0 (make/model + +/// pointers), the Exif SubIFD (`DateTimeOriginal`, `OffsetTimeOriginal`, pixel dimensions) and +/// the GPS IFD — and the image is a genuine baseline JPEG a conformant decoder accepts: every +/// 8×8 block is the shortest legal encoding of an all-zero block (DC category 0, then EOB), two +/// bits each, so the picture is a flat mid-grey. +#[must_use] +pub fn synthetic_jpeg_sized(width: u16, height: u16) -> Vec { + assert!( + width.is_multiple_of(8) && height.is_multiple_of(8) && width > 0 && height > 0, + "dimensions are whole 8×8 blocks" + ); + const ASCII: u16 = 2; + const LONG: u16 = 4; + const RATIONAL: u16 = 5; + + const MAKE: &[u8] = b"Capsule\0"; + const MODEL: &[u8] = b"Synth\0"; + const DATE_TIME_ORIGINAL: &[u8] = b"2019:03:04 05:06:07\0"; + const OFFSET_TIME_ORIGINAL: &[u8] = b"+00:00\0"; + + // Each IFD here holds four entries: 2 count bytes + 4×12 entry bytes + 4 next-IFD bytes. + const IFD_LEN: u32 = 2 + 4 * 12 + 4; + const IFD0_AT: u32 = 8; + const EXIF_IFD_AT: u32 = IFD0_AT + IFD_LEN; + const GPS_IFD_AT: u32 = EXIF_IFD_AT + IFD_LEN; + const DATA_AT: u32 = GPS_IFD_AT + IFD_LEN; + const MAKE_AT: u32 = DATA_AT; + const MODEL_AT: u32 = MAKE_AT + MAKE.len() as u32; + const DTO_AT: u32 = MODEL_AT + MODEL.len() as u32; + const OTO_AT: u32 = DTO_AT + DATE_TIME_ORIGINAL.len() as u32; + // Rationals are 4-byte quantities; one pad byte keeps them aligned. + const LAT_AT: u32 = OTO_AT + OFFSET_TIME_ORIGINAL.len() as u32 + 1; + const LON_AT: u32 = LAT_AT + 24; + + /// One 12-byte IFD entry whose value is an offset into the TIFF block. + fn at(tag: u16, kind: u16, count: u32, offset: u32) -> Vec { + let mut e = Vec::with_capacity(12); + e.extend_from_slice(&tag.to_be_bytes()); + e.extend_from_slice(&kind.to_be_bytes()); + e.extend_from_slice(&count.to_be_bytes()); + e.extend_from_slice(&offset.to_be_bytes()); + e + } + + /// One 12-byte IFD entry whose value fits in the 4 inline bytes. + fn inline(tag: u16, kind: u16, count: u32, value: [u8; 4]) -> Vec { + let mut e = Vec::with_capacity(12); + e.extend_from_slice(&tag.to_be_bytes()); + e.extend_from_slice(&kind.to_be_bytes()); + e.extend_from_slice(&count.to_be_bytes()); + e.extend_from_slice(&value); + e + } + + fn rational(numerator: u32, denominator: u32) -> Vec { + let mut r = Vec::with_capacity(8); + r.extend_from_slice(&numerator.to_be_bytes()); + r.extend_from_slice(&denominator.to_be_bytes()); + r + } + + let mut tiff = Vec::new(); + tiff.extend_from_slice(b"MM"); + tiff.extend_from_slice(&42u16.to_be_bytes()); + tiff.extend_from_slice(&IFD0_AT.to_be_bytes()); + + // IFD0: Make, Model, and the pointers to the two sub-IFDs. + tiff.extend_from_slice(&4u16.to_be_bytes()); + tiff.extend(at(0x010F, ASCII, MAKE.len() as u32, MAKE_AT)); + tiff.extend(at(0x0110, ASCII, MODEL.len() as u32, MODEL_AT)); + tiff.extend(at(0x8769, LONG, 1, EXIF_IFD_AT)); + tiff.extend(at(0x8825, LONG, 1, GPS_IFD_AT)); + tiff.extend_from_slice(&0u32.to_be_bytes()); + + // Exif SubIFD: capture time, its UTC offset, and the pixel dimensions. + tiff.extend_from_slice(&4u16.to_be_bytes()); + tiff.extend(at(0x9003, ASCII, DATE_TIME_ORIGINAL.len() as u32, DTO_AT)); + tiff.extend(at(0x9011, ASCII, OFFSET_TIME_ORIGINAL.len() as u32, OTO_AT)); + tiff.extend(inline(0xA002, LONG, 1, u32::from(width).to_be_bytes())); + tiff.extend(inline(0xA003, LONG, 1, u32::from(height).to_be_bytes())); + tiff.extend_from_slice(&0u32.to_be_bytes()); + + // GPS IFD: 48°51'29.6"N, 2°17'40.2"W. + tiff.extend_from_slice(&4u16.to_be_bytes()); + tiff.extend(inline(0x0001, ASCII, 2, *b"N\0\0\0")); + tiff.extend(at(0x0002, RATIONAL, 3, LAT_AT)); + tiff.extend(inline(0x0003, ASCII, 2, *b"W\0\0\0")); + tiff.extend(at(0x0004, RATIONAL, 3, LON_AT)); + tiff.extend_from_slice(&0u32.to_be_bytes()); + + // The out-of-line values, in the order the offsets above declare. + tiff.extend_from_slice(MAKE); + tiff.extend_from_slice(MODEL); + tiff.extend_from_slice(DATE_TIME_ORIGINAL); + tiff.extend_from_slice(OFFSET_TIME_ORIGINAL); + tiff.push(0); + for (numerator, denominator) in [(48, 1), (51, 1), (296, 10), (2, 1), (17, 1), (402, 10)] { + tiff.extend(rational(numerator, denominator)); + } + assert_eq!( + tiff.len() as u32, + LON_AT + 24, + "the TIFF block must be exactly as long as its own offsets claim" + ); + + let mut app1 = b"Exif\0\0".to_vec(); + app1.extend_from_slice(&tiff); + + let mut jpeg = vec![0xFF, 0xD8]; // SOI + jpeg.extend_from_slice(&[0xFF, 0xE1]); // APP1 + jpeg.extend_from_slice(&((app1.len() + 2) as u16).to_be_bytes()); + jpeg.extend_from_slice(&app1); + + // DQT: one flat 8-bit luminance table. + jpeg.extend_from_slice(&[0xFF, 0xDB]); + jpeg.extend_from_slice(&(2u16 + 1 + 64).to_be_bytes()); + jpeg.push(0x00); + jpeg.extend(std::iter::repeat_n(1u8, 64)); + + // SOF0: baseline, 8-bit, one component with no subsampling. + jpeg.extend_from_slice(&[0xFF, 0xC0]); + jpeg.extend_from_slice(&11u16.to_be_bytes()); + jpeg.extend_from_slice(&[0x08]); + jpeg.extend_from_slice(&height.to_be_bytes()); + jpeg.extend_from_slice(&width.to_be_bytes()); + jpeg.extend_from_slice(&[0x01, 0x01, 0x11, 0x00]); + + // DHT: a DC and an AC table each holding a single 1-bit code for symbol 0. + for class_and_id in [0x00u8, 0x10] { + jpeg.extend_from_slice(&[0xFF, 0xC4]); + jpeg.extend_from_slice(&(2u16 + 1 + 16 + 1).to_be_bytes()); + jpeg.push(class_and_id); + jpeg.push(1); + jpeg.extend(std::iter::repeat_n(0u8, 15)); + jpeg.push(0x00); + } + + // SOS, then the entropy-coded data: two zero bits per block (DC category 0, then EOB), + // the last partial byte padded with 1 bits as the standard requires. + jpeg.extend_from_slice(&[0xFF, 0xDA]); + jpeg.extend_from_slice(&8u16.to_be_bytes()); + jpeg.extend_from_slice(&[0x01, 0x01, 0x00, 0x00, 0x3F, 0x00]); + let blocks = usize::from(width / 8) * usize::from(height / 8); + let bits = blocks * 2; + jpeg.extend(std::iter::repeat_n(0u8, bits / 8)); + let dangling = bits % 8; + if dangling != 0 { + jpeg.push((1u8 << (8 - dangling)) - 1); + } + + jpeg.extend_from_slice(&[0xFF, 0xD9]); // EOI + jpeg +} diff --git a/capsule-e2e/src/lib.rs b/capsule-e2e/src/lib.rs new file mode 100644 index 00000000..999d14c7 --- /dev/null +++ b/capsule-e2e/src/lib.rs @@ -0,0 +1,437 @@ +//! The harness behind the bounded E2E cases (design/module-map.md, "E2E Test Surface"). +//! +//! Three real things, wired the way production wires them, and nothing standing in for any of +//! them: +//! +//! - **The server** is the composition root — [`capsule_server::boot::assemble`] under the +//! memory profile, the same function `capsule-server serve --memory` runs — bound to an +//! ephemeral port. Real argon2 accounts, the real provisioned write authority, a real +//! filesystem blob store under a temp root, the system clock. Not the test-only `Fixture` +//! the server's own suites use: its `SwallowingBlobs`, `TestAuthority` and `ManualClock` are +//! doubles, and a case that passed against them would prove the doubles. +//! - **The client** is `capsule-sdk` as shipped: every request leaves through the SDK's one +//! HTTP client and therefore carries the protocol handshake the server gates on. +//! - **The library** is a real [`capsule_core::lifecycle::Workspace`] on a temp root, with a +//! fast Argon2id parameter set for the *library* passphrase only (the wrap records its own +//! parameters, so nothing under test reads a weaker setting than it would in the field). +//! +//! What the harness adds on top of the SDK is exactly the seams the SDK does not have yet, each +//! recorded as a finding in the pull request that landed this crate: +//! +//! - the **provenance rung** ([`push::push_asset`]): the SDK's push ladder ships metadata, +//! derivatives and the original but never the `provenance` blob, and the server publishes an +//! asset to the feed only once it holds both index-tier roles (issue #464); +//! - the **directory publish** ([`Device::publish_directory`]): the server requires the +//! `X-Capsule-Identity-Key` header on every publish and the SDK's `DirectoryClient` does not +//! send it (issue #466), and a directory must name the *server's* account id, which a +//! `Workspace` cannot learn (issue #467). +//! +//! Every test that uses this crate names its case — `E2E case N` — so `rg "E2E case N"` finds +//! it, per the module map's contract. + +pub mod fixtures; +pub mod push; + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use capsule_core::crypto::keys::{DeviceDirectory, DeviceEntry, DirectoryCore, HybridSigningKey}; +use capsule_core::crypto::primitives::Argon2Params; +use capsule_core::crypto::provenance::record::ProvenanceRecord; +use capsule_core::lifecycle::Workspace; +use capsule_sdk::albums::{AlbumClient, AlbumTransport}; +use capsule_sdk::auth::{AuthClient, Session}; +use capsule_sdk::client::AuthenticatedClient; +use capsule_sdk::sync::{FeedEntry, SyncConsumer, SyncState}; +use capsule_sdk::upload::{UploadClient, UploadTransport}; +use capsule_server::blob::address::{ContentAddress, blob_path}; +use capsule_server::boot::{self, Assembled}; +use capsule_server::config::{Config, Demands, Overrides}; +use tempfile::TempDir; +use uuid::Uuid; + +/// The protocol date this build speaks — the same constant the SDK's transport sends. +pub const PROTOCOL_VERSION: &str = capsule_core::crypto::primitives::PROTOCOL_VERSION; + +/// A PKCS#8 v1 Ed25519 key, base64: the retired deployment's `.env.example` value, which signs +/// nothing anywhere (the server's own binary test uses the same bytes for the same reason). +pub const JWT_ED25519_DER: &str = + "MC4CAQAwBQYDK2VwBCIEIN6eTvXEL7xMZWHY8rTk7VbQSGSuRkle5MVfiiYUStLF"; + +/// Every account's password. The server hashes it with its production argon2 parameters. +pub const PASSWORD: &str = "correct horse battery staple"; + +/// Every library's passphrase, wrapped under [`FAST_KDF`]. +pub const PASSPHRASE: &[u8] = b"library passphrase"; + +/// Fast Argon2id for the fixture libraries — the CLI's own precedent +/// (`capsule-cli/tests/import_round_trip.rs`). The wrapped blob records these parameters and +/// `unwrap` reads them back, so no code under test runs a weaker setting than it would in the +/// field; only the fixture's own unlock is cheap. +pub const FAST_KDF: Argon2Params = Argon2Params { + mem_kib: 64, + t_cost: 1, + p_cost: 1, +}; + +/// The feed page size the harness pulls with; small enough that `has_more` paging is exercised +/// by any case that pushes more than a handful of assets. +pub const PAGE_SIZE: u32 = 64; + +/// The composition root, assembled and listening on an ephemeral port. +/// +/// Holds the [`Assembled`] so a case can reach the operator workers +/// (`assembled.maintenance`) over the same stores the router serves, and the blob root so a +/// case can assert bytes at their content address on disk. +pub struct Server { + base_url: String, + /// The assembled application: `app` for the router, `maintenance` for the operator workers. + pub assembled: Assembled, + /// `BLOB_ROOT`: where the filesystem blob store files finalized bytes. + pub blob_root: TempDir, + serve: tokio::task::JoinHandle<()>, +} + +impl Server { + /// Boot with the server's default protocol window. + pub async fn boot() -> Self { + Self::boot_with(None).await + } + + /// Boot with `PROTOCOL_MIN`/`PROTOCOL_MAX` overridden — the knob case 9 turns to put this + /// build outside the window. + pub async fn boot_with_window(min: &str, max: &str) -> Self { + Self::boot_with(Some((min, max))).await + } + + async fn boot_with(window: Option<(&str, &str)>) -> Self { + let blob_root = tempfile::tempdir().expect("a temp blob root"); + // Exactly what `serve --memory` reads: the memory profile needs two variables and + // derives the rest (cursor MAC key, attestation seed) from the signing key. + let mut env: BTreeMap = BTreeMap::new(); + env.insert( + "BLOB_ROOT".to_owned(), + blob_root.path().display().to_string(), + ); + env.insert("JWT_ED25519_DER".to_owned(), JWT_ED25519_DER.to_owned()); + if let Some((min, max)) = window { + env.insert("PROTOCOL_MIN".to_owned(), min.to_owned()); + env.insert("PROTOCOL_MAX".to_owned(), max.to_owned()); + } + let overrides = Overrides { + memory: true, + ..Overrides::default() + }; + let config = Config::load(&env, &overrides, Demands::Serve) + .expect("the memory profile loads from BLOB_ROOT and JWT_ED25519_DER alone"); + let assembled = boot::assemble(&config) + .await + .expect("the composition root assembles under the memory profile"); + let service = assembled.service().expect("the router builds"); + let bound = kynos::server::Server::new(service) + .bind(("127.0.0.1", 0)) + .prepare() + .await + .expect("an ephemeral port binds"); + let address = *bound + .local_addrs() + .first() + .expect("a bound server has an address"); + let serve = tokio::spawn(async move { + let _ = bound.serve().await; + }); + tracing::info!(%address, "e2e server listening"); + Self { + base_url: format!("http://{address}"), + assembled, + blob_root, + serve, + } + } + + /// The API root (`http://127.0.0.1:PORT`): what the generated client, the sync consumer, + /// the recovery client and the upgrade client take. + #[must_use] + pub fn base_url(&self) -> &str { + &self.base_url + } + + /// `{root}/v1`: what the verify transport and the blob source take. + #[must_use] + pub fn v1(&self) -> String { + format!("{}/v1", self.base_url) + } + + /// `{root}/v1/auth`: what `AuthClient` and the directory publish take. + #[must_use] + pub fn auth_base(&self) -> String { + format!("{}/v1/auth", self.base_url) + } + + /// `{root}/v1/upload`: the upload transport's root. + #[must_use] + pub fn upload_base(&self) -> String { + format!("{}/v1/upload", self.base_url) + } + + /// `{root}/v1/albums`: the album transport's root. + #[must_use] + pub fn albums_base(&self) -> String { + format!("{}/v1/albums", self.base_url) + } + + /// Where the filesystem blob store files the blob at content address `hex`. + #[must_use] + pub fn blob_path(&self, hex: &str) -> PathBuf { + let address = ContentAddress::parse(hex).expect("a lowercase SHA-256 hex digest"); + blob_path(self.blob_root.path(), &address) + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.serve.abort(); + } +} + +/// One account on one [`Server`], with its live SDK session and its real library. +/// +/// The account's device directory is published by the harness (see +/// [`Device::publish_directory`]) and names two devices: the library's own, so the manifests it +/// signs satisfy invariant 7, and a standalone *proposer* device whose signing key the harness +/// holds — the `Workspace` keeps its device signing key private, and case 8 needs a key that +/// can sign an `UpgradeIntent`. +pub struct Device { + /// The registered e-mail. + pub email: String, + /// The live SDK session; clone it freely, clones share one token store. + pub session: Session, + /// The **server's** id for this account, read from `GET /v1/auth/profile`. Distinct from + /// `workspace.user_id()`, which the library mints locally — there is no seam to align them. + pub user_id: Uuid, + /// The real library. + pub workspace: Workspace, + /// The library root. + pub root: TempDir, + /// Scratch space for files to import. + pub staging: TempDir, + /// The identity key the published directory is signed with. + pub identity: HybridSigningKey, + /// A device signing key the harness holds, listed in the directory as `proposer_id`. + pub proposer: HybridSigningKey, + /// The proposer device's id. + pub proposer_id: Uuid, + directory_version: u64, +} + +impl Device { + /// Register a fresh account, create its library, publish its directory and provision its + /// default album on the server — everything a first upload needs. + pub async fn register(server: &Server, label: &str) -> Self { + let email = format!("{label}-{}@e2e.capsule.test", Uuid::now_v7().simple()); + let auth = AuthClient::new(&server.auth_base()).expect("the auth base parses"); + let session = auth + .register(&email, PASSWORD) + .await + .expect("a fresh account registers"); + let generated = AuthenticatedClient::new(server.base_url(), session.clone()) + .expect("the API root parses"); + let profile = generated + .get_profile(PROTOCOL_VERSION, None) + .await + .expect("the profile answers") + .into_inner(); + let user_id = Uuid::parse_str(&profile.user_id).expect("the account id is a UUID"); + + let root = tempfile::tempdir().expect("a temp library root"); + let staging = tempfile::tempdir().expect("a temp staging dir"); + let mut workspace = + Workspace::create_with_params(root.path(), PASSPHRASE, FAST_KDF).expect("a library"); + let default_album = workspace.default_album_id(); + workspace + .ensure_album(default_album, "Imports") + .expect("the default album's keys exist"); + + let mut device = Self { + email, + session, + user_id, + workspace, + root, + staging, + identity: HybridSigningKey::generate(), + proposer: HybridSigningKey::generate(), + proposer_id: Uuid::now_v7(), + directory_version: 0, + }; + device.publish_directory(server).await; + let albums = AlbumClient::new(AlbumTransport::with_session( + device.session.clone(), + server.albums_base(), + )); + push::ensure_album(&albums, default_album) + .await + .expect("the default album provisions"); + device + } + + /// A second live session on the same account — device B in the two-device cases. + pub async fn login_again(&self, server: &Server) -> Session { + AuthClient::new(&server.auth_base()) + .expect("the auth base parses") + .login(&self.email, PASSWORD) + .await + .expect("the account signs in again") + .into_session() + .expect("the account has no second factor") + } + + /// The generated REST client over this session. + #[must_use] + pub fn generated(&self, server: &Server) -> AuthenticatedClient { + AuthenticatedClient::new(server.base_url(), self.session.clone()) + .expect("the API root parses") + } + + /// The upload client over this session, pinned to this build's protocol date. + #[must_use] + pub fn upload_client(&self, server: &Server) -> UploadClient { + UploadClient::new(UploadTransport::with_session( + self.session.clone(), + server.upload_base(), + PROTOCOL_VERSION, + )) + } + + /// The library's own entry in its device directory. + #[must_use] + pub fn library_device(&self) -> DeviceEntry { + let id = self.workspace.device_id(); + self.workspace + .device_directory() + .device(&id) + .cloned() + .expect("a library lists its own device") + } + + /// The directory the harness publishes for this account: the server's account id, the + /// library's device and the proposer device, signed by [`Device::identity`]. + #[must_use] + pub fn directory(&self) -> DeviceDirectory { + let library = self.library_device(); + DirectoryCore { + user_id: self.user_id, + directory_version: self.directory_version, + updated_at: jiff::Timestamp::now().to_string(), + devices: vec![ + library.clone(), + DeviceEntry { + device_id: self.proposer_id, + dsk_public: self.proposer.verifying_key(), + dek_public: None, + added_at: library.added_at, + revoked_at: None, + }, + ], + } + .sign(&self.identity) + } + + /// Publish the next version of [`Device::directory`], returning the version stored. + /// + /// Sent through the session's HTTP client rather than `capsule_sdk::directory` because the + /// server requires `X-Capsule-Identity-Key` (invariant 23's second clause) and the SDK's + /// publish does not carry it — issue #466. + pub async fn publish_directory(&mut self, server: &Server) -> u64 { + self.directory_version += 1; + let body = capsule_core::cbor::to_canonical_vec(&self.directory()) + .expect("a directory serializes"); + let identity = BASE64.encode(self.identity.verifying_key().to_bytes()); + let url = format!("{}/devices/directory", server.auth_base()); + let response = self + .session + .execute(|http| { + http.post(&url) + .header("content-type", "application/cbor") + .header("x-capsule-identity-key", &identity) + .body(body.clone()) + }) + .await + .expect("the publish reaches the server"); + assert_eq!( + response.status().as_u16(), + 200, + "the directory publish is accepted: {}", + response.text().await.unwrap_or_default() + ); + let stored: serde_json::Value = response.json().await.expect("a JSON body"); + let version = stored["directory_version"] + .as_u64() + .expect("the stored directory version"); + assert_eq!(version, self.directory_version); + version + } + + /// Write the 8×8 synthetic JPEG to staging under `file_name` and import it into the + /// default album, returning the asset id. + pub fn import_jpeg(&mut self, file_name: &str) -> Uuid { + self.import_file(file_name, &fixtures::synthetic_jpeg()) + } + + /// Write `bytes` to staging under `file_name` and import the file into the default album, + /// returning the asset id. + pub fn import_file(&mut self, file_name: &str, bytes: &[u8]) -> Uuid { + let path = self.staging.path().join(file_name); + std::fs::write(&path, bytes).expect("the fixture writes"); + let album = self.workspace.default_album_id(); + self.workspace + .import_asset(album, &path) + .expect("the file imports") + } + + /// The head of `asset_id`'s provenance chain. + #[must_use] + pub fn head_record(&self, asset_id: &Uuid) -> ProvenanceRecord { + self.workspace + .asset(asset_id) + .expect("the asset is in the library") + .chain + .records() + .last() + .cloned() + .expect("a chain is never empty") + } + + /// Everything the feed holds for this account, from the beginning, through the SDK. + pub async fn feed(&self, server: &Server) -> Vec { + feed_from_start(server, self.session.clone()).await + } +} + +/// Pull the whole feed for `session` from cursor zero through the SDK consumer. +pub async fn feed_from_start(server: &Server, session: Session) -> Vec { + let consumer = + SyncConsumer::with_session(server.base_url(), session).expect("the API root parses"); + let mut state = SyncState::new(PROTOCOL_VERSION); + let mut entries = Vec::new(); + loop { + let page = consumer + .pull_into(&mut state, PAGE_SIZE) + .await + .expect("the feed answers"); + let more = page.has_more; + entries.extend(page.entries); + if !more { + return entries; + } + } +} + +/// The feed entry for `asset_id`, if the server publishes it. +#[must_use] +pub fn entry_for<'a>(entries: &'a [FeedEntry], asset_id: &Uuid) -> Option<&'a FeedEntry> { + let wanted = asset_id.to_string().into_bytes(); + entries.iter().find(|entry| entry.asset_id == wanted) +} diff --git a/capsule-e2e/src/push.rs b/capsule-e2e/src/push.rs new file mode 100644 index 00000000..a3ed3048 --- /dev/null +++ b/capsule-e2e/src/push.rs @@ -0,0 +1,281 @@ +//! Pushing a library asset to the server: the SDK's ladder plus the rung it omits, and the +//! lifecycle-op posting that chains onto what the rung established. +//! +//! **The provenance rung.** The SDK's `push_bundle` ships the sealed metadata blob, every +//! derivative and the original, and stops: it never uploads a `provenance` blob. The server +//! publishes an asset to other devices only once it holds both index-tier roles — provenance +//! *and* metadata (`capsule-server/src/upload/visibility.rs`) — and the head of the server-side +//! chain is the SHA-256 of the provenance blob's bytes. So the harness uploads one more blob per +//! asset: the canonical CBOR of the chain's head `ProvenanceRecord`, whose digest is by +//! definition core's `record_hash()`. That is what lets a later lifecycle op's +//! `prior_provenance_hash` — the client's record hash of the previous record — match the head +//! the server holds. The finding is filed against the SDK as issue #464 (and the decode side, in +//! core's `sync_apply`, as #465); the encoding decision is recorded in the pull request that +//! landed this crate. +//! +//! Every envelope here is projected from the head manifest's [`ManifestCore`] rather than from +//! an `UploadBundle`, because a bundle re-derives the original's ciphertext and an adopted +//! (wrapped-key) asset or a tombstone head has nothing to re-derive; the projection is the same +//! one `capsule_sdk::push::envelope_for` makes, field for field. + +use std::collections::HashSet; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use capsule_core::crypto::hash::hash_bytes; +use capsule_core::crypto::provenance::manifest::ManifestCore; +use capsule_core::lifecycle::UploadBundle; +use capsule_sdk::net::ConnectionClass; +pub use capsule_sdk::push::ensure_album; +use capsule_sdk::push::{AssetPushReport, push_bundle}; +use capsule_sdk::rest; +use capsule_sdk::staged::StagedScheduler; +use capsule_sdk::upload::{ + BlobRole, CreateUploadRequest, ManifestEnvelope, UploadClient, UploadOutcome, +}; +use uuid::Uuid; + +use crate::{Device, PROTOCOL_VERSION, Server}; + +/// The content type every metadata, provenance and backup blob declares. +pub const OPAQUE_CONTENT_TYPE: &str = "application/octet-stream"; + +/// What one push left behind. +pub struct Pushed { + /// The bundle the library produced for the asset's current head. + pub bundle: UploadBundle, + /// The SDK ladder's report — metadata, derivatives, original. + pub report: AssetPushReport, + /// The content address of the provenance blob the harness added: the server's chain head. + pub provenance_hash: String, +} + +/// Serialize a wire enum (`Action`, `KeyMode`) to its bare protocol string. +fn wire_enum(value: &T) -> String { + serde_json::to_value(value) + .ok() + .and_then(|v| v.as_str().map(str::to_owned)) + .expect("a wire enum serializes to a string") +} + +/// The SDK's [`ManifestEnvelope`] for one blob of the asset whose head is `core`, with +/// `ciphertext_hash` naming **this** blob (the server's invariant-15 consistency rule). +#[must_use] +pub fn sdk_envelope(core: &ManifestCore, blob_hash: &str) -> ManifestEnvelope { + ManifestEnvelope { + crypto_suite_id: core.crypto_suite_id, + protocol_version: core.protocol_version.clone(), + album_id: Some(core.album_id.to_string()), + file_id: core.file_id.to_string(), + amk_version: core.amk_version.0, + ciphertext_hash: blob_hash.to_owned(), + plaintext_size: core.plaintext_size, + chunk_size: core.chunk_size, + key_mode: wire_enum(&core.key_mode), + metadata_blob_hash: core.metadata_blob_hash.map(|h| h.to_hex()), + created_by_user: core.created_by_user.to_string(), + created_by_device: core.created_by_device.to_string(), + client_version: core.client_version.clone(), + timestamp: core.timestamp.clone(), + action: wire_enum(&core.action), + prior_provenance_hash: core.prior_provenance_hash.map(|h| h.to_hex()), + retention_until: core.retention_until.clone(), + } +} + +/// The same projection in the generated type the lifecycle-op and adopt operations take. +#[must_use] +pub fn wire_envelope(core: &ManifestCore, blob_hash: &str) -> rest::types::ManifestEnvelope { + let envelope = sdk_envelope(core, blob_hash); + rest::types::ManifestEnvelope { + crypto_suite_id: i64::from(envelope.crypto_suite_id), + protocol_version: envelope.protocol_version, + album_id: envelope.album_id, + file_id: envelope.file_id, + amk_version: i64::from(envelope.amk_version), + ciphertext_hash: envelope.ciphertext_hash, + plaintext_size: envelope.plaintext_size as i64, + chunk_size: i64::from(envelope.chunk_size), + key_mode: envelope.key_mode, + metadata_blob_hash: envelope.metadata_blob_hash, + original_blob_hash: None, + created_by_user: envelope.created_by_user, + created_by_device: envelope.created_by_device, + client_version: envelope.client_version, + timestamp: envelope.timestamp, + action: envelope.action, + prior_provenance_hash: envelope.prior_provenance_hash, + retention_until: envelope.retention_until, + } +} + +/// Upload `bytes` as one `role` blob of the asset whose head is `core`, returning the content +/// address it landed at. +pub async fn upload_blob( + client: &UploadClient, + core: &ManifestCore, + role: BlobRole, + content_type: &str, + bytes: &[u8], +) -> String { + let hash = hash_bytes(bytes).to_hex(); + let request = CreateUploadRequest { + size: bytes.len() as u64, + hash: hash.clone(), + content_type: content_type.to_owned(), + crypto_suite_id: core.crypto_suite_id, + protocol_version: core.protocol_version.clone(), + blob_role: role, + manifest_envelope: sdk_envelope(core, &hash), + album_id: Some(core.album_id.to_string()), + owner_id: None, + intent_id: None, + }; + let outcome = client + .upload(&request, bytes) + .await + .unwrap_or_else(|error| panic!("the {role:?} blob uploads: {error}")); + assert!( + matches!( + outcome, + UploadOutcome::Completed { .. } | UploadOutcome::AlreadyStored { .. } + ), + "the {role:?} blob finalizes" + ); + hash +} + +/// The canonical CBOR of the chain head — the bytes the provenance blob carries. +#[must_use] +pub fn provenance_bytes(device: &Device, asset_id: &Uuid) -> Vec { + let record = device.head_record(asset_id); + let bytes = capsule_core::cbor::to_canonical_vec(&record).expect("a record serializes"); + debug_assert_eq!( + hash_bytes(&bytes), + record.record_hash(), + "record_hash is the digest of the canonical record bytes" + ); + bytes +} + +/// Upload the provenance rung for `asset_id`, returning its content address. +pub async fn push_provenance(client: &UploadClient, device: &Device, asset_id: &Uuid) -> String { + let core = device.head_record(asset_id).manifest.core; + let bytes = provenance_bytes(device, asset_id); + upload_blob( + client, + &core, + BlobRole::Provenance, + OPAQUE_CONTENT_TYPE, + &bytes, + ) + .await +} + +/// Upload the sealed metadata blob the head manifest binds, returning its content address. +pub async fn push_metadata(client: &UploadClient, device: &Device, asset_id: &Uuid) -> String { + let core = device.head_record(asset_id).manifest.core; + let bytes = device + .workspace + .asset(asset_id) + .expect("the asset is in the library") + .metadata_blob + .clone(); + let hash = upload_blob( + client, + &core, + BlobRole::Metadata, + OPAQUE_CONTENT_TYPE, + &bytes, + ) + .await; + assert_eq!( + Some(hash.as_str()), + core.metadata_blob_hash.map(|h| h.to_hex()).as_deref(), + "the sealed metadata blob is the one the head manifest binds" + ); + hash +} + +/// Push `asset_id` in full: the SDK ladder under `UploadPolicy::Full` on an unmetered link, +/// then the provenance rung. +pub async fn push_asset(device: &Device, server: &Server, asset_id: &Uuid) -> Pushed { + let bundle = device + .workspace + .upload_bundle(asset_id) + .expect("the library builds an upload bundle for its own asset"); + let client = device.upload_client(server); + let scheduler = StagedScheduler::new( + capsule_core::import::UploadPolicy::Full, + ConnectionClass::Unmetered, + ); + let report = push_bundle(&client, &scheduler, &bundle, &HashSet::new(), false) + .await + .expect("the SDK ladder pushes the bundle"); + let provenance_hash = push_provenance(&client, device, asset_id).await; + tracing::info!( + asset_id = %asset_id, + pushed = report.pushed.len(), + %provenance_hash, + "e2e push complete" + ); + Pushed { + bundle, + report, + provenance_hash, + } +} + +/// Publish an adopted asset whose original the server already holds: the metadata blob and +/// the provenance rung — the index tier — and nothing else. +pub async fn push_index_tier(device: &Device, server: &Server, asset_id: &Uuid) -> String { + let client = device.upload_client(server); + push_metadata(&client, device, asset_id).await; + push_provenance(&client, device, asset_id).await +} + +/// Post the library's current chain head for `asset_id` as a lifecycle op +/// (`POST /v1/albums/{album}/ops`) through the generated client. +/// +/// The op carries the head record's canonical CBOR as `manifest_cbor` — so the server's new head +/// is that record's hash — and the sealed metadata blob whenever the head manifest binds one +/// (invariant 25: a hash without its bytes, or bytes without a hash, is a `400`). +pub async fn post_lifecycle_head( + device: &Device, + server: &Server, + asset_id: &Uuid, +) -> rest::types::OpResponse { + let asset = device + .workspace + .asset(asset_id) + .expect("the asset is in the library"); + let core = &asset + .chain + .records() + .last() + .expect("a chain is never empty") + .manifest + .core; + let manifest = provenance_bytes(device, asset_id); + let request = rest::types::OpRequest { + manifest_envelope: wire_envelope(core, &core.ciphertext_hash.to_hex()), + manifest_cbor: BASE64.encode(&manifest), + metadata_blob: core + .metadata_blob_hash + .map(|_| BASE64.encode(&asset.metadata_blob)), + }; + let response = device + .generated(server) + .album_lifecycle_op(core.album_id.to_string(), PROTOCOL_VERSION, None, &request) + .await + .unwrap_or_else(|error| panic!("the {:?} op applies: {error}", core.action)) + .into_inner(); + tracing::info!( + asset_id = %asset_id, + action = %response.action, + sync_seq = response.sync_seq, + replayed = response.replayed, + "e2e lifecycle op applied" + ); + response +} diff --git a/capsule-e2e/tests/case_01_auth_sync_query.rs b/capsule-e2e/tests/case_01_auth_sync_query.rs new file mode 100644 index 00000000..fe2e2d0b --- /dev/null +++ b/capsule-e2e/tests/case_01_auth_sync_query.rs @@ -0,0 +1,73 @@ +//! **E2E case 1** — auth → sync → client-side library query. +//! +//! Sign in → access token → the sync feed returns the account's entries → the client applies +//! them → a local SQLite query lists the expected album. The client leg is the CLI's own +//! orchestration (`capsule_cli::remote::{sync, list}`) over its migrated SQLite store, which +//! is what `capsule sync` and `capsule list` run; the SDK session is the one the CLI would have +//! persisted after `capsule auth login`. + +use capsule_cli::remote::{self, RemoteConfig}; +use capsule_cli::session::SessionStore; +use capsule_e2e::push::push_asset; +use capsule_e2e::{Device, PROTOCOL_VERSION, Server}; +use migration::{Migrator, MigratorTrait as _}; + +#[tokio::test] +async fn e2e_case_1_sign_in_sync_and_a_local_query_lists_the_album() { + let server = Server::boot().await; + let mut device = Device::register(&server, "cli-user").await; + let asset = device.import_jpeg("first.jpg"); + push_asset(&device, &server, &asset).await; + + // The CLI's state: a migrated SQLite store and the session `capsule auth login` persists — + // a fresh sign-in on the account, not the registration session. + let signed_in = device.login_again(&server).await; + let home = tempfile::tempdir().expect("a temp CLI home"); + let db_url = format!( + "sqlite://{}?mode=rwc", + home.path().join("library.sqlite").display() + ); + let db = sea_orm::Database::connect(&db_url) + .await + .expect("the CLI store opens"); + Migrator::up(&db, None) + .await + .expect("the CLI migrations run"); + let store = SessionStore::new(home.path().join("session.json")); + let persisted = signed_in.export().await.expect("a live session exports"); + store.save(&persisted).expect("the session persists"); + + let remote = RemoteConfig { + auth_endpoint: server.auth_base(), + sync_endpoint: server.base_url().to_owned(), + upload_endpoint: server.upload_base(), + albums_endpoint: server.albums_base(), + protocol_version: PROTOCOL_VERSION.to_owned(), + }; + let summary = remote::sync(&remote, &store, &db, 256, false, false) + .await + .expect("`capsule sync` completes"); + assert_eq!(summary.applied, 1, "one entry applied: {summary:?}"); + assert_eq!(summary.albums, 1); + assert!(!summary.dry_run); + + // The client-side library query lists the expected album and asset. + let rows = remote::list(&db, false) + .await + .expect("`capsule list` answers"); + assert_eq!(rows.len(), 1); + let row = &rows[0]; + assert_eq!( + row.album_id, + device.workspace.default_album_id().to_string().into_bytes() + ); + assert_eq!(row.asset_id, asset.to_string().into_bytes()); + assert!(row.original_held); + assert!(!row.tombstoned); + + // A second sync is a no-op — the cursor persisted with the page. + let again = remote::sync(&remote, &store, &db, 256, false, false) + .await + .expect("a second sync completes"); + assert_eq!(again.applied, 0, "nothing new: {again:?}"); +} diff --git a/capsule-e2e/tests/case_02_import_upload_finalize.rs b/capsule-e2e/tests/case_02_import_upload_finalize.rs new file mode 100644 index 00000000..aa6e5acc --- /dev/null +++ b/capsule-e2e/tests/case_02_import_upload_finalize.rs @@ -0,0 +1,135 @@ +//! **E2E case 2** — full import + upload + finalize. +//! +//! Local import → the library's upload bundle → the SDK's staged ladder (metadata, the JXL +//! thumbnail, the original) plus the provenance rung → every blob finalized at its content +//! address under the server's blob root, byte for byte → the server's storage-verify answer is +//! durable → the asset is on the feed with its original held, its derivative referenced and +//! its metadata blob named. +//! +//! The still is 512×512 — past the thumbnail tier's 256-pixel cap — so the media stack decodes +//! it and encodes a real thumbnail, and T1 is a real upload rather than the byte-free sentinel +//! an 8×8 still gets. + +use capsule_core::crypto::hash::Hash32; +use capsule_core::import::UploadTier; +use capsule_e2e::fixtures::large_synthetic_jpeg; +use capsule_e2e::push::{provenance_bytes, push_asset}; +use capsule_e2e::{Device, Server, entry_for}; +use capsule_sdk::verify::{AssetQuery, StorageVerifyClient, VerifyTransport}; + +#[tokio::test] +async fn e2e_case_2_import_upload_finalize_lands_every_blob_at_its_content_address() { + let server = Server::boot().await; + let mut device = Device::register(&server, "importer").await; + let asset = device.import_file("photo.jpg", &large_synthetic_jpeg()); + + let pushed = push_asset(&device, &server, &asset).await; + let bundle = &pushed.bundle; + assert_eq!(bundle.asset_id, asset); + assert!( + !bundle.derivatives.is_empty(), + "a still past the thumbnail cap yields derivative bytes" + ); + assert!( + bundle.derivatives.iter().all(|d| d.format == "image/jxl"), + "this build encodes thumbnails as JXL: {:?}", + bundle + .derivatives + .iter() + .map(|d| &d.format) + .collect::>() + ); + + // The ladder ran every tier: T0, one T1 per derivative, then T2. + let mut expected = vec![UploadTier::Index]; + expected.extend(bundle.derivatives.iter().map(|_| UploadTier::Preview)); + expected.push(UploadTier::Original); + assert_eq!(pushed.report.tier_sequence(), expected); + assert_eq!(pushed.report.deferred, 0); + + // Every blob is on disk at its content address under the blob root, byte for byte. + let on_disk = |hex: &str| std::fs::read(server.blob_path(hex)).expect("the blob is filed"); + assert_eq!(on_disk(&bundle.ciphertext_hash.to_hex()), bundle.ciphertext); + let metadata_hash = bundle + .metadata_blob_hash + .expect("a create binds a metadata blob") + .to_hex(); + assert_eq!(on_disk(&metadata_hash), bundle.metadata_blob); + for derivative in &bundle.derivatives { + assert_eq!( + on_disk(&derivative.ciphertext_hash.to_hex()), + derivative.bytes + ); + } + assert_eq!( + on_disk(&pushed.provenance_hash), + provenance_bytes(&device, &asset) + ); + + // The server's own custody answer for the whole set is durable. + let mut hashes = vec![ + bundle.ciphertext_hash, + Hash32::from_hex(&metadata_hash).expect("a digest"), + Hash32::from_hex(&pushed.provenance_hash).expect("a digest"), + ]; + hashes.extend(bundle.derivatives.iter().map(|d| d.ciphertext_hash)); + let verify = StorageVerifyClient::new(VerifyTransport::with_session( + device.session.clone(), + server.v1(), + )); + let verdicts = verify + .verify( + &[AssetQuery { + asset_id: asset, + blob_hashes: hashes.clone(), + }], + false, + ) + .await + .expect("the verify surface answers"); + assert_eq!(verdicts.len(), 1); + let verdict = &verdicts[0]; + assert_eq!(verdict.asset_id, asset); + assert!( + verdict.durable, + "every named blob is stored and indexed: {verdict:?}" + ); + assert_eq!(verdict.blobs.len(), hashes.len()); + + // The feed publishes the asset: original held, derivative referenced, metadata named. + let feed = device.feed(&server).await; + let entry = entry_for(&feed, &asset).expect("the asset is on the feed"); + assert!(entry.original_held, "the original finalized"); + assert_eq!( + entry + .blobs + .original + .as_ref() + .map(|b| b.ciphertext_hash.as_str()), + Some(bundle.ciphertext_hash.to_hex().as_str()) + ); + let derivative_hashes: Vec<&str> = entry + .blobs + .derivatives + .iter() + .filter(|b| b.role == "derivative") + .map(|b| b.ciphertext_hash.as_str()) + .collect(); + for derivative in &bundle.derivatives { + assert!( + derivative_hashes.contains(&derivative.ciphertext_hash.to_hex().as_str()), + "the {} derivative rides the feed: {derivative_hashes:?}", + derivative.format + ); + } + assert_eq!( + String::from_utf8(entry.metadata_blob.clone()).expect("a hex content address"), + metadata_hash, + "the feed names the metadata blob by its content address" + ); + assert_eq!( + entry.manifest_cbor, + provenance_bytes(&device, &asset), + "the feed serves the provenance blob's bytes unchanged" + ); +} diff --git a/capsule-e2e/tests/case_03_sync_pickup.rs b/capsule-e2e/tests/case_03_sync_pickup.rs new file mode 100644 index 00000000..4b0be6fa --- /dev/null +++ b/capsule-e2e/tests/case_03_sync_pickup.rs @@ -0,0 +1,76 @@ +//! **E2E case 3** — sync feed pickup, the client half. +//! +//! The server half is named in `capsule-server/tests/sync.rs`. Here device A uploads through +//! the real SDK and library; device B — a second session on the same account, with a fresh +//! `SyncState` at cursor zero — pulls the feed, sees the entry, and fetches the metadata blob +//! and the original by content address, byte-equal to what A's bundle held. +//! +//! B's `verify_asset` is not asserted: verification needs the album keys, which reach a second +//! device through enrollment or backup (cases 6 and 12), not through the feed. + +use capsule_e2e::push::push_asset; +use capsule_e2e::{Device, PAGE_SIZE, PROTOCOL_VERSION, Server, entry_for}; +use capsule_sdk::fetch::{BlobSource as _, HttpBlobSource, RangeOutcome, fetch_blob}; +use capsule_sdk::sync::{SyncConsumer, SyncState}; + +#[tokio::test] +async fn e2e_case_3_a_second_device_sees_the_entry_and_fetches_the_bytes() { + let server = Server::boot().await; + let mut a = Device::register(&server, "device-a").await; + let asset = a.import_jpeg("shared.jpg"); + let pushed = push_asset(&a, &server, &asset).await; + + // Device B: same account, its own session, nothing synced yet. + let session_b = a.login_again(&server).await; + let consumer = + SyncConsumer::with_session(server.base_url(), session_b.clone()).expect("a consumer"); + let mut state = SyncState::new(PROTOCOL_VERSION); + assert!(state.cursor().is_start()); + let page = consumer + .pull_into(&mut state, PAGE_SIZE) + .await + .expect("B's first pull"); + assert!(!page.has_more); + assert!(!state.cursor().is_start(), "the cursor advanced"); + let album = a.workspace.default_album_id().to_string().into_bytes(); + assert!( + state.high_water(&album).is_some(), + "the album's high-water mark is set" + ); + + let entry = entry_for(&page.entries, &asset).expect("A's upload is on B's feed"); + assert_eq!(entry.album_id, album); + assert!(entry.original_held); + + // The metadata blob, by the content address the entry carries. The feed states no size for + // it, so B asks for the whole object rather than a range it cannot know the length of. + let source = HttpBlobSource::new(session_b, server.v1()); + let metadata_address = + String::from_utf8(entry.metadata_blob.clone()).expect("a hex content address"); + assert_eq!( + metadata_address, + pushed + .bundle + .metadata_blob_hash + .expect("a create binds a metadata blob") + .to_hex() + ); + let RangeOutcome::Complete { + bytes: metadata_bytes, + } = source.get_range(&metadata_address, 0, None).await + else { + panic!("the metadata blob serves whole"); + }; + assert_eq!(metadata_bytes, pushed.bundle.metadata_blob); + + // The original, by content address and declared size, byte for byte. + let original = entry + .blobs + .original + .as_ref() + .expect("the original is referenced"); + let original_bytes = fetch_blob(&source, &original.ciphertext_hash, original.size) + .await + .expect("the original fetches"); + assert_eq!(original_bytes, pushed.bundle.ciphertext); +} diff --git a/capsule-e2e/tests/case_06_backup_restore.rs b/capsule-e2e/tests/case_06_backup_restore.rs new file mode 100644 index 00000000..ba708554 --- /dev/null +++ b/capsule-e2e/tests/case_06_backup_restore.rs @@ -0,0 +1,114 @@ +//! **E2E case 6** — backup → restore on a fresh device. +//! +//! Export a full backup → bootstrap a new device via passphrase and escrow → import the backup +//! → assert every asset present and verifiable. +//! +//! Two tests, because until a `Workspace` can open *as* a recovered account (issue #467) the +//! escrow and the restore are independent halves: nothing the escrow recovers feeds the +//! restore, and nothing the restore needs comes from the escrow. +//! +//! - **`E2E case 6 (escrow)`**: A escrows its master key at the low-RAM tier through the SDK's +//! `RecoveryClient` over the real route; a second session fetches it byte for byte and +//! `recover_master_key` yields A's key, proved by re-deriving A's default album id. Two +//! Argon2id passes at `DeviceTier::LowRam` (the wrap and the recovery) — the one memory-hard +//! test in the crate. +//! - **`E2E case 6 (restore)`**: A exports a backup; a fresh library on a new root imports it +//! under the exporter's verifying key, reads the asset byte for byte and walks its chain. +//! `verify` is asserted to refuse: the artifact carries no album authority (issue #468). + +use capsule_core::crypto::keys::MasterKey; +use capsule_core::crypto::primitives::DeviceTier; +use capsule_core::crypto::provenance::record::ProvenanceChain; +use capsule_core::lifecycle::{LifecycleError, Workspace}; +use capsule_e2e::fixtures::synthetic_jpeg; +use capsule_e2e::{Device, FAST_KDF, PASSPHRASE, Server}; +use capsule_sdk::recovery::RecoveryClient; + +const RECOVERY_SECRET: &[u8] = b"seven words the user wrote down somewhere safe"; +const BACKUP_PASSPHRASE: &[u8] = b"backup passphrase"; + +/// **E2E case 6 (escrow)**: the master key round-trips through the real escrow route and +/// recovers on a second device. +#[tokio::test] +async fn e2e_case_6_the_escrow_round_trips_and_recovers_the_master_key() { + let server = Server::boot().await; + let a = Device::register(&server, "device-a").await; + + let escrow = a + .workspace + .escrow_master_key(RECOVERY_SECRET, DeviceTier::LowRam) + .expect("the master key wraps under the recovery secret"); + RecoveryClient::new(a.session.clone(), server.base_url()) + .expect("the API root parses") + .store_escrow(&escrow) + .await + .expect("the escrow stores"); + + // The fresh device: a new session on the account, nothing else. + let session_b = a.login_again(&server).await; + let fetched = RecoveryClient::new(session_b, server.base_url()) + .expect("the API root parses") + .fetch_escrow() + .await + .expect("the escrow fetches"); + let wire = fetched.blob().clone(); + assert_eq!(wire, escrow, "the escrow is ciphertext served verbatim"); + + let master = capsule_core::backup::recover_master_key(&wire, RECOVERY_SECRET) + .expect("the recovery secret opens the escrow"); + assert_eq!( + MasterKey::from_bytes(master).derive_default_album_id(), + a.workspace.default_album_id(), + "the recovered master key is A's: it derives A's default album id" + ); +} + +/// **E2E case 6 (restore)**: a fresh library imports the backup, reads every asset and walks +/// its chain; `verify` refuses for want of the album authority the artifact does not carry. +#[tokio::test] +async fn e2e_case_6_a_fresh_library_restores_the_backup() { + let server = Server::boot().await; + let mut a = Device::register(&server, "device-a").await; + let asset = a.import_jpeg("keepsake.jpg"); + let archive = a.staging.path().join("backup.tar"); + a.workspace + .export_backup(&archive, BACKUP_PASSPHRASE) + .expect("the backup exports"); + let exporter = a.workspace.exporter_verifying_key(); + + let root_b = tempfile::tempdir().expect("a fresh library root"); + let mut b = + Workspace::create_with_params(root_b.path(), PASSPHRASE, FAST_KDF).expect("a library"); + assert!(b.asset_ids().is_empty(), "no prior state"); + let restored = b + .import_backup(&archive, BACKUP_PASSPHRASE, &exporter) + .expect("the backup imports under the exporter's key"); + assert_eq!(restored, 1); + assert_eq!(b.asset_ids(), vec![asset]); + assert_eq!( + b.read_plaintext(&asset).expect("the asset decrypts"), + synthetic_jpeg() + ); + assert!( + b.has_album(&a.workspace.default_album_id()), + "the restore folded A's album keys into the fresh library" + ); + + // The restored chain is structurally intact, and the plaintext above is the manifest's: + // the ciphertext decrypted under the recovered album key to the bytes A imported. + let chain = &b.asset(&asset).expect("the restored asset").chain; + assert_eq!(chain.records().len(), 1); + ProvenanceChain::verify_walk(chain.records()).expect("the restored chain walks"); + + // What the fresh device cannot yet do is run `verify_asset`: the backup artifact carries + // the album's content keys and none of its authority (the admin-signed epoch ledger a + // manifest's write signature is checked against), so the library has nothing to verify + // the signature under. Asserted as the current truth; issue #468. + match b.verify(&asset) { + Err(LifecycleError::NotFound(what)) => assert!( + what.contains("authority"), + "the refusal names the missing authority: {what}" + ), + other => panic!("a restored album has no authority to verify under, got {other:?}"), + } +} diff --git a/capsule-e2e/tests/case_07_lifecycle.rs b/capsule-e2e/tests/case_07_lifecycle.rs new file mode 100644 index 00000000..68d29c23 --- /dev/null +++ b/capsule-e2e/tests/case_07_lifecycle.rs @@ -0,0 +1,149 @@ +//! **E2E case 7** — full lifecycle. +//! +//! Create → metadata update → trash → restore → re-delete → hard purge after retention. The +//! provenance chain advances through every transition, and the server refuses purge before +//! `retention_until`. +//! +//! Every transition is authored by the real library, posted as a lifecycle op through the +//! generated client, and observed by an incremental feed reader. The retention floor is the +//! one the library *signed*: a 30-day tombstone is retained by the collector, a zero-day +//! tombstone is purged, on the operator worker over the same stores the router serves. (The +//! server never purges an unsigned floor — absent is "never", not "now".) + +use capsule_e2e::push::{post_lifecycle_head, push_asset}; +use capsule_e2e::{Device, PAGE_SIZE, PROTOCOL_VERSION, Server, entry_for}; +use capsule_sdk::fetch::{FetchError, HttpBlobSource, fetch_blob}; +use capsule_sdk::sync::{ChangeKind, FeedEntry, SyncConsumer, SyncState}; +use capsule_server::gc::{Mode, purge_expired}; +use uuid::Uuid; + +/// The next page of the incremental reader: what changed since the last pull. +async fn next(consumer: &SyncConsumer, state: &mut SyncState) -> Vec { + consumer + .pull_into(state, PAGE_SIZE) + .await + .expect("the feed answers") + .entries +} + +fn kind_of(entries: &[FeedEntry], asset: &Uuid) -> ChangeKind { + entry_for(entries, asset) + .unwrap_or_else(|| panic!("{asset} changed since the last pull")) + .kind +} + +#[tokio::test] +async fn e2e_case_7_the_chain_advances_through_every_transition_and_purge_honours_retention() { + let server = Server::boot().await; + let mut a = Device::register(&server, "owner").await; + let kept = a.import_jpeg("kept.jpg"); + let purged = a.import_jpeg("purged.jpg"); + push_asset(&a, &server, &kept).await; + let purged_bundle = push_asset(&a, &server, &purged).await.bundle; + + let consumer = + SyncConsumer::with_session(server.base_url(), a.session.clone()).expect("a consumer"); + let mut state = SyncState::new(PROTOCOL_VERSION); + let created = next(&consumer, &mut state).await; + assert_eq!(kind_of(&created, &kept), ChangeKind::Created); + assert_eq!(kind_of(&created, &purged), ChangeKind::Created); + + // Metadata update: the caption changes the sealed metadata blob and the chain head. + a.workspace + .set_caption(&kept, "the one we keep") + .expect("the caption sets"); + let op = post_lifecycle_head(&a, &server, &kept).await; + assert_eq!(op.action, "metadata-update"); + assert!(!op.replayed); + assert_eq!( + kind_of(&next(&consumer, &mut state).await, &kept), + ChangeKind::Updated + ); + + // Trash with a 30-day signed floor. + a.workspace + .soft_delete(&kept, 30) + .expect("the asset trashes"); + let op = post_lifecycle_head(&a, &server, &kept).await; + assert_eq!(op.action, "delete"); + assert_eq!( + kind_of(&next(&consumer, &mut state).await, &kept), + ChangeKind::Deleted + ); + + // Restore. + a.workspace.restore(&kept).expect("the asset restores"); + let op = post_lifecycle_head(&a, &server, &kept).await; + assert_eq!(op.action, "trash-restore"); + assert_ne!( + kind_of(&next(&consumer, &mut state).await, &kept), + ChangeKind::Deleted + ); + + // Re-delete, again with the 30-day floor; the other asset with a floor that is already due. + a.workspace + .soft_delete(&kept, 30) + .expect("the asset trashes again"); + assert_eq!( + post_lifecycle_head(&a, &server, &kept).await.action, + "delete" + ); + a.workspace + .soft_delete(&purged, 0) + .expect("the asset trashes"); + assert_eq!( + post_lifecycle_head(&a, &server, &purged).await.action, + "delete" + ); + let deleted = next(&consumer, &mut state).await; + assert_eq!(kind_of(&deleted, &kept), ChangeKind::Deleted); + assert_eq!(kind_of(&deleted, &purged), ChangeKind::Deleted); + + // The chain the library holds is the one the server applied: five records for `kept`. + let chain = &a.workspace.asset(&kept).expect("the asset").chain; + assert_eq!(chain.records().len(), 5); + + // A replay of the same head is idempotent, not a stale-chain refusal. + assert!(post_lifecycle_head(&a, &server, &kept).await.replayed); + + // Purge on the operator worker: the 30-day floor is honoured, the due one is purged. + let report = purge_expired(&server.assembled.maintenance.collection, Mode::Apply, 10) + .await + .expect("the purge runs"); + let names = |ids: &[capsule_server::store::AssetId]| -> Vec { + ids.iter().map(ToString::to_string).collect() + }; + assert_eq!( + names(&report.retained), + vec![kept.to_string()], + "{report:?}" + ); + assert_eq!( + names(&report.purged), + vec![purged.to_string()], + "{report:?}" + ); + + // The purged original is gone to a reader; the retained tombstone's bytes still stand. + let source = HttpBlobSource::new(a.session.clone(), server.v1()); + let gone = fetch_blob( + &source, + &purged_bundle.ciphertext_hash.to_hex(), + purged_bundle.ciphertext.len() as u64, + ) + .await + .expect_err("a purged original no longer serves"); + assert!(matches!(gone, FetchError::Gone), "got {gone:?}"); + assert!( + server + .blob_path( + &a.workspace + .upload_bundle(&kept) + .expect("a bundle") + .ciphertext_hash + .to_hex() + ) + .exists(), + "the retained tombstone's bytes are untouched" + ); +} diff --git a/capsule-e2e/tests/case_08_upgrade_ceremony.rs b/capsule-e2e/tests/case_08_upgrade_ceremony.rs new file mode 100644 index 00000000..4d601b9b --- /dev/null +++ b/capsule-e2e/tests/case_08_upgrade_ceremony.rs @@ -0,0 +1,116 @@ +//! **E2E case 8** — the album upgrade ceremony, the server leg through the SDK. +//! +//! An admin initiates the upgrade → quiesce: a signed `UpgradeIntent` from a device in the +//! published directory is proposed through `capsule_sdk::upgrade`, the phase reads back +//! in flight, a write that does not name the ceremony is refused with the ceremony's code, +//! the abort clears it, and the same write then lands. +//! +//! The client ceremony (drain → tombstone → fork → replay, and resume-from-crash) stays in +//! `capsule-core`'s in-process suite: a `Workspace` keeps its device signing key private and +//! cannot sign an intent, so the proposer here is the harness-held device the directory names. + +use capsule_core::crypto::primitives::CRYPTO_SUITE_ID; +use capsule_core::crypto::upgrade::{SignedUpgradeIntent, UpgradeIntent}; +use capsule_e2e::push::push_asset; +use capsule_e2e::{Device, PROTOCOL_VERSION, Server, entry_for}; +use capsule_sdk::push::{bundle_blobs, create_request}; +use capsule_sdk::upgrade::UpgradeClient; +use capsule_sdk::upload::UploadError; +use uuid::Uuid; + +#[tokio::test] +async fn e2e_case_8_a_proposed_upgrade_quiesces_the_album_until_it_is_aborted() { + let server = Server::boot().await; + let mut a = Device::register(&server, "admin").await; + let album = a.workspace.default_album_id(); + let generated = a.generated(&server); + + let intent = UpgradeIntent { + intent_id: Uuid::now_v7(), + from_protocol_version: PROTOCOL_VERSION.to_owned(), + to_protocol_version: "2030-01-01".to_owned(), + from_suite_id: CRYPTO_SUITE_ID, + to_suite_id: CRYPTO_SUITE_ID, + proposer_user: a.user_id, + proposer_device: a.proposer_id, + deadline_secs: 300, + }; + let intent_id = intent.intent_id; + let proposer_sig = a + .proposer + .sign(&intent.signing_bytes().expect("an intent encodes")); + let signed = capsule_core::cbor::to_canonical_vec(&SignedUpgradeIntent { + intent, + proposer_sig, + }) + .expect("a signed intent serializes"); + + let phase = UpgradeClient::new(a.session.clone(), server.base_url()) + .begin(album, &signed) + .await + .expect("a signed proposal from a directory device is accepted"); + assert_eq!(phase.album_id, album); + assert_eq!(phase.intent_id, Some(intent_id)); + assert_eq!( + phase.in_flight, 0, + "nothing was mid-flight when the album quiesced" + ); + assert!(phase.expires_at.is_some()); + + let read = generated + .album_upgrade_phase(album.to_string(), PROTOCOL_VERSION, None) + .await + .expect("the phase reads") + .into_inner(); + assert_eq!( + read.intent_id.as_deref(), + Some(intent_id.to_string().as_str()) + ); + assert_eq!(read.to_protocol_version.as_deref(), Some("2030-01-01")); + + // A write that does not name the ceremony is refused with its code and the live intent. + let asset = a.import_jpeg("during-quiesce.jpg"); + let bundle = a.workspace.upload_bundle(&asset).expect("a bundle"); + let blobs = bundle_blobs(&bundle); + let (blob, hash) = blobs.first().expect("a T0 blob"); + let request = create_request(&bundle, blob, hash); + assert!(request.intent_id.is_none()); + let refused = a + .upload_client(&server) + .create_session(&request) + .await + .expect_err("a quiescing album refuses a write that names no ceremony"); + match &refused { + UploadError::Rejected { status, code, .. } => { + assert_eq!(*status, 409); + assert_eq!(code.as_deref(), Some("error.upload.album_quiescing")); + } + other => panic!("expected the ceremony's refusal, got {other:?}"), + } + + // Abort: the phase clears and the same write lands. + let aborted = generated + .abort_album_upgrade( + album.to_string(), + intent_id.to_string(), + PROTOCOL_VERSION, + None, + ) + .await + .expect("the proposer aborts") + .into_inner(); + assert_eq!(aborted.intent_id, None); + let cleared = generated + .album_upgrade_phase(album.to_string(), PROTOCOL_VERSION, None) + .await + .expect("the phase reads") + .into_inner(); + assert_eq!(cleared.intent_id, None); + + push_asset(&a, &server, &asset).await; + let feed = a.feed(&server).await; + assert!( + entry_for(&feed, &asset).is_some(), + "the write resumed after the abort" + ); +} diff --git a/capsule-e2e/tests/case_12_enrollment.rs b/capsule-e2e/tests/case_12_enrollment.rs new file mode 100644 index 00000000..8875c5b9 --- /dev/null +++ b/capsule-e2e/tests/case_12_enrollment.rs @@ -0,0 +1,193 @@ +//! **E2E case 12** — cross-device enrollment, the server leg through the SDK. +//! +//! Device A authorizes new device B over a verified channel: fresh local auth → an enrollment +//! code → B redeems it into a relay channel → payloads cross in both directions, each +//! delivered once and never to the wrong mailbox → the initiator closes the channel. Includes +//! one MITM-on-relay abort: the initiator sees a payload that is not the key material it +//! expected and closes, and the enrollee's next drain finds no channel. +//! +//! The client ceremony — B's hardware keys, the safety-code check, A cross-signing B into the +//! directory — is blocked on seams the tree does not have (issues #471 and #467); B's MLS +//! joins and `libraries match` wait on server-side membership (#405). + +use capsule_e2e::{Device, PASSWORD, PROTOCOL_VERSION, Server}; +use capsule_sdk::rest; +use capsule_sdk::rest::types::{ReauthenticateRequest, RedeemRequest, RelayRequest}; + +const TO_ENROLLEE: &str = "to_enrollee"; +const TO_INITIATOR: &str = "to_initiator"; + +/// The enrollee's client: no account yet, but it is a Capsule build and speaks the handshake. +fn enrollee(server: &Server) -> rest::Client { + rest::Client::with_client( + capsule_sdk::net::http_client().expect("the SDK client builds"), + server.base_url(), + ) + .expect("the API root parses") +} + +async fn open_channel(server: &Server, a: &Device) -> String { + let issued = a + .generated(server) + .issue_enrollment_code(PROTOCOL_VERSION, None) + .await + .expect("a freshly authenticated initiator issues a code") + .into_inner(); + assert!(!issued.code.is_empty()); + assert!(!issued.text_fallback.is_empty()); + enrollee(server) + .redeem_enrollment_code(PROTOCOL_VERSION, None, &RedeemRequest { code: issued.code }) + .await + .expect("the enrollee redeems the code") + .into_inner() + .channel_id +} + +fn relay(direction: &str, payload: &str) -> RelayRequest { + RelayRequest { + direction: direction.to_owned(), + payload: payload.to_owned(), + } +} + +#[tokio::test] +async fn e2e_case_12_a_code_opens_a_relay_channel_that_delivers_each_payload_once() { + let server = Server::boot().await; + let a = Device::register(&server, "initiator").await; + let initiator = a.generated(&server); + let b = enrollee(&server); + + // Fresh local auth on the initiator: the password, on the already-authenticated session. + let fresh = initiator + .reauthenticate( + PROTOCOL_VERSION, + None, + &ReauthenticateRequest { + password: PASSWORD.to_owned(), + }, + ) + .await + .expect("the password re-authenticates the session") + .into_inner(); + assert!(!fresh.authenticated_at.is_empty()); + + let channel = open_channel(&server, &a).await; + + // A → B, then B → A; each mailbox holds only its own direction. + initiator + .relay_enrollment_payload( + &channel, + PROTOCOL_VERSION, + None, + &relay(TO_ENROLLEE, "wrapped-album-keys"), + ) + .await + .expect("the initiator relays"); + b.relay_enrollment_payload( + &channel, + PROTOCOL_VERSION, + None, + &relay(TO_INITIATOR, "device-b-public-keys"), + ) + .await + .expect("the enrollee relays"); + let to_b = b + .drain_enrollment_channel(&channel, TO_ENROLLEE, PROTOCOL_VERSION, None) + .await + .expect("the enrollee drains") + .into_inner(); + assert_eq!(to_b.payloads, vec!["wrapped-album-keys"]); + let to_a = initiator + .drain_enrollment_channel(&channel, TO_INITIATOR, PROTOCOL_VERSION, None) + .await + .expect("the initiator drains") + .into_inner(); + assert_eq!(to_a.payloads, vec!["device-b-public-keys"]); + + // Delivered once: both mailboxes are now empty. + for direction in [TO_ENROLLEE, TO_INITIATOR] { + let again = b + .drain_enrollment_channel(&channel, direction, PROTOCOL_VERSION, None) + .await + .expect("a drained mailbox still answers") + .into_inner(); + assert!(again.payloads.is_empty(), "{direction} delivered twice"); + } + + // The initiator closes; the channel is gone for the enrollee. + initiator + .close_enrollment_channel(&channel, PROTOCOL_VERSION, None) + .await + .expect("the initiator closes its channel"); + let closed = b + .drain_enrollment_channel(&channel, TO_ENROLLEE, PROTOCOL_VERSION, None) + .await + .expect_err("a closed channel is not found"); + match closed { + rest::Error::Api(response) => match response.into_inner() { + rest::DrainEnrollmentChannelError::Status404(problem) => { + assert_eq!(problem.code, "error.enrollment.channel_not_found"); + } + other => panic!("expected 404, got {other:?}"), + }, + other => panic!("expected an API refusal, got {other:?}"), + } +} + +#[tokio::test] +async fn e2e_case_12_the_initiator_aborts_on_a_payload_it_did_not_expect() { + let server = Server::boot().await; + let a = Device::register(&server, "initiator").await; + let initiator = a.generated(&server); + let b = enrollee(&server); + // Registration is fresh local auth: the code issues without a separate reauthentication. + let channel = open_channel(&server, &a).await; + + // What B advertised out of band (the safety code the users compare) versus what arrives. + const ADVERTISED: &str = "device-b-public-keys"; + b.relay_enrollment_payload( + &channel, + PROTOCOL_VERSION, + None, + &relay(TO_INITIATOR, "device-m-public-keys"), + ) + .await + .expect("the relay accepts what it is given"); + let arrived = initiator + .drain_enrollment_channel(&channel, TO_INITIATOR, PROTOCOL_VERSION, None) + .await + .expect("the initiator drains") + .into_inner(); + assert_ne!( + arrived.payloads, + vec![ADVERTISED], + "the relay was tampered with" + ); + + // Abort: close, and never send the wrapped keys. + initiator + .close_enrollment_channel(&channel, PROTOCOL_VERSION, None) + .await + .expect("the initiator aborts by closing"); + let aborted = b + .drain_enrollment_channel(&channel, TO_ENROLLEE, PROTOCOL_VERSION, None) + .await + .expect_err("nothing reaches the enrollee after the abort"); + assert!( + matches!( + aborted, + rest::Error::Api(ref response) + if matches!(response.inner(), rest::DrainEnrollmentChannelError::Status404(_)) + ), + "got {aborted:?}" + ); + let relayed_late = b + .relay_enrollment_payload( + &channel, + PROTOCOL_VERSION, + None, + &relay(TO_INITIATOR, ADVERTISED), + ) + .await; + assert!(relayed_late.is_err(), "a closed channel accepts nothing"); +} diff --git a/capsule-e2e/tests/case_13_web_drop_adopt.rs b/capsule-e2e/tests/case_13_web_drop_adopt.rs new file mode 100644 index 00000000..c8ee13c5 --- /dev/null +++ b/capsule-e2e/tests/case_13_web_drop_adopt.rs @@ -0,0 +1,204 @@ +//! **E2E case 13** — web drop → adopt, the server leg. +//! +//! The provisioning user's library issues an upload link; the link is provisioned on the +//! server; a guest with no account and **no protocol handshake** seals a drop to the link's +//! Drop Key and deposits it through the two exempt guest operations; the owner's inbox shows +//! it; the owner's library decapsulates, rewraps the key under the album AMK and adopts it in +//! place; the server adopts the same manifest and holds the drop's bytes as the asset's +//! durable original. The feed leg waits on a library seam recorded at the end of the test. +//! +//! The browser half of the seal is the cross-language KAT (`capsule-core/tests/drop_adopt_kat.rs` +//! and `capsule-web`'s `drop-seal.test.ts`); here the seal runs natively. Verification on a +//! second device waits on key transfer (cases 6 and 12). + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use capsule_core::crypto::hash::hash_bytes; +use capsule_core::crypto::primitives::CRYPTO_SUITE_ID; +use capsule_core::crypto::provenance::manifest::KeyMode; +use capsule_core::drop::{DropAdopter as _, LinkCaps, UploadLinkIssuer as _, seal_drop}; +use capsule_e2e::fixtures::synthetic_jpeg; +use capsule_e2e::push::wire_envelope; +use capsule_e2e::{Device, PROTOCOL_VERSION, Server, entry_for}; +use capsule_sdk::rest; +use capsule_sdk::rest::types::{AdoptRequest, CreateDropRequest, ProvisionLinkRequest}; +use capsule_sdk::verify::{AssetQuery, StorageVerifyClient, VerifyTransport}; + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + bytes.iter().fold(String::new(), |mut out, b| { + let _ = write!(out, "{b:02x}"); + out + }) +} + +#[tokio::test] +async fn e2e_case_13_a_guest_drop_is_deposited_without_a_handshake_and_adopted_in_place() { + let server = Server::boot().await; + let mut owner = Device::register(&server, "owner").await; + let album = owner.workspace.default_album_id(); + let generated = owner.generated(&server); + + // The owner's library issues the link; the server learns its opaque id and Drop Key. + let link = owner + .workspace + .create_link(LinkCaps::default(), None) + .expect("the library issues an upload link"); + let opaque_id = hex(&link.opaque_id); + let provisioned = generated + .provision_link( + PROTOCOL_VERSION, + None, + &ProvisionLinkRequest { + opaque_id: opaque_id.clone(), + drop_pubkey: BASE64.encode(&link.drop_pubkey), + crypto_suite_id: i64::from(CRYPTO_SUITE_ID), + expires_at: None, + max_total_bytes: None, + max_file_count: None, + max_file_size: None, + single_use: Some(false), + passphrase_verifier: None, + }, + ) + .await + .expect("the link provisions") + .into_inner(); + assert_eq!(provisioned.opaque_id, opaque_id); + + // The guest: a bare generated client — no session, no default headers, no handshake. + let guest = rest::Client::new(server.base_url()).expect("the API root parses"); + let plaintext = synthetic_jpeg(); + let sealed = seal_drop(&plaintext, &link.drop_pubkey, "image/jpeg").expect("the drop seals"); + let ciphertext_hash = sealed.descriptor.ciphertext_hash.to_hex(); + let created = guest + .create_drop( + &opaque_id, + &CreateDropRequest { + content_type: "image/jpeg".to_owned(), + size: sealed.ciphertext.len() as i64, + ciphertext_hash: ciphertext_hash.clone(), + kem_ct: BASE64.encode(&sealed.descriptor.kem_ct), + passphrase_proof: None, + suggested_filename: Some("drop.jpg".to_owned()), + }, + ) + .await + .expect("the exempt guest operation admits a client with no handshake") + .into_inner(); + guest + .append_drop_chunk( + &opaque_id, + &created.upload_id, + rest::AppendDropChunkParams { + x_capsule_offset: Some("0".to_owned()), + x_capsule_checksum: Some(hash_bytes(&sealed.ciphertext).to_hex()), + }, + &rest::types::RequestBody4e14fb73::from(sealed.ciphertext.clone()), + ) + .await + .expect("the exempt chunk operation admits the bytes"); + + // The owner's inbox shows the drop, and the bytes are filed at their content address. + let inbox = generated + .list_inbox(PROTOCOL_VERSION, None) + .await + .expect("the inbox answers") + .into_inner(); + assert_eq!(inbox.drops.len(), 1); + let pending = &inbox.drops[0]; + assert_eq!(pending.opaque_id, opaque_id); + assert_eq!(pending.ciphertext_hash, ciphertext_hash); + assert_eq!(pending.size, sealed.ciphertext.len() as i64); + assert!(!pending.adopting); + assert_eq!( + std::fs::read(server.blob_path(&ciphertext_hash)).expect("the drop is filed"), + sealed.ciphertext + ); + + // The owner's library decapsulates and adopts in place: a wrapped-key create manifest. + let drop_id = owner + .workspace + .receive_drop(link.link_id, sealed.clone()) + .expect("the library receives the drop"); + let manifest = owner + .workspace + .adopt(drop_id, album) + .expect("the library adopts the drop"); + let core = &manifest.core; + assert_eq!(core.ciphertext_hash.to_hex(), ciphertext_hash); + assert_eq!(core.key_mode, KeyMode::Wrapped); + assert!( + core.wrapped_file_key.is_some(), + "the guest's key is rewrapped under the AMK" + ); + let asset = core.file_id; + assert_eq!(core.created_by_device, owner.workspace.device_id()); + assert_eq!(core.plaintext_size, plaintext.len() as u64); + + // The server adopts the same manifest: the staged bytes become the asset's original. + let adopted = generated + .adopt_drop( + &pending.drop_id, + PROTOCOL_VERSION, + None, + &AdoptRequest { + album_id: album.to_string(), + asset_id: asset.to_string(), + size: sealed.ciphertext.len() as i64, + hash: ciphertext_hash.clone(), + content_type: "image/jpeg".to_owned(), + crypto_suite_id: i64::from(CRYPTO_SUITE_ID), + protocol_version: core.protocol_version.clone(), + key_mode: "wrapped".to_owned(), + manifest_envelope: wire_envelope(core, &ciphertext_hash), + }, + ) + .await + .expect("the server adopts the drop") + .into_inner(); + assert_eq!(adopted.asset_id, asset.to_string()); + assert!( + generated + .list_inbox(PROTOCOL_VERSION, None) + .await + .expect("the inbox answers") + .into_inner() + .drops + .is_empty(), + "the adopted drop left the inbox" + ); + + // The server holds the drop's bytes as the asset's original: stored, indexed, retrievable. + let verify = StorageVerifyClient::new(VerifyTransport::with_session( + owner.session.clone(), + server.v1(), + )); + let verdicts = verify + .verify( + &[AssetQuery { + asset_id: asset, + blob_hashes: vec![core.ciphertext_hash], + }], + false, + ) + .await + .expect("the verify surface answers"); + assert_eq!(verdicts.len(), 1); + assert!( + verdicts[0].durable, + "the adopted original is durable: {:?}", + verdicts[0] + ); + + // Where the server leg stops: the feed publishes an asset only once it holds the index + // tier — the sealed metadata blob and the provenance record — and the library's adopt + // returns the signed manifest without registering the asset or handing back the metadata + // blob it sealed, so the owner has nothing to publish — issue #469, which the feed leg + // and the second-device verify wait on. + let feed = owner.feed(&server).await; + assert!( + entry_for(&feed, &asset).is_none(), + "an adopted asset with no index tier is not yet published" + ); +} diff --git a/capsule-e2e/tests/protocol_contract.rs b/capsule-e2e/tests/protocol_contract.rs new file mode 100644 index 00000000..245f5038 --- /dev/null +++ b/capsule-e2e/tests/protocol_contract.rs @@ -0,0 +1,206 @@ +//! **E2E case 9** — the cross-version protocol gate, end to end — and the body-less `413`. +//! +//! Case 9's wording: a client whose `protocol_version` falls outside the server's range +//! attempts an upload, receives `426`, and the UI surfaces an actionable error. The UI leg is +//! out of scope here; what the SDK hands the UI is the typed error with the server's window, +//! asserted from three angles: +//! +//! 1. a per-transport pin outside the default window is refused at `POST /v1/upload` with the +//! window the server advertises (`UploadError::UpgradeRequired { min, max }`); +//! 2. a server booted with `PROTOCOL_MIN = PROTOCOL_MAX = 2000-01-01` — the whole +//! `Config` → `boot` → `Negotiation` path — refuses this build's *writes* with `426` and +//! `error.protocol.version_unsupported`, and stamps `X-Capsule-Protocol-Min/Max` on every +//! response, the exempt ones included; +//! 3. a *read* at an out-of-window date succeeds and carries the window (issue #404's +//! decision: reads of any grammatical protocol date are admitted), while a malformed +//! handshake is `400 error.request.malformed`. +//! +//! The `413` contract: Kynos's body-size backstop answers with no problem body, so the SDK +//! reports it as `code: None` rather than minting a code the server never sent. + +use capsule_core::crypto::pwkdf::WrappedSecret; +use capsule_e2e::push::ensure_album; +use capsule_e2e::{Device, PASSWORD, Server}; +use capsule_sdk::auth::{AuthClient, AuthError}; +use capsule_sdk::push::{bundle_blobs, create_request}; +use capsule_sdk::recovery::{RecoveryClient, RecoveryError}; +use capsule_sdk::rest; +use capsule_sdk::upload::{UploadClient, UploadError, UploadTransport}; + +const DEFAULT_MIN: &str = "2026-01-01"; +const DEFAULT_MAX: &str = "2026-12-31"; +const STALE: &str = "1999-01-01"; +const VERSION_UNSUPPORTED: &str = "error.protocol.version_unsupported"; + +/// **E2E case 9**, leg 1: a stale transport pin against the default window. +#[tokio::test] +async fn e2e_case_9_a_stale_pin_is_refused_with_the_servers_window() { + let server = Server::boot().await; + let mut device = Device::register(&server, "stale").await; + let asset = device.import_jpeg("stale.jpg"); + let bundle = device + .workspace + .upload_bundle(&asset) + .expect("a bundle for the asset"); + let blobs = bundle_blobs(&bundle); + let (blob, hash) = blobs.first().expect("a bundle has a T0 blob"); + let request = create_request(&bundle, blob, hash); + + // The pin wins over the transport's default header (`net.rs`): this is the one + // hand-written place the SDK lets a caller speak an older protocol. + let stale = UploadClient::new(UploadTransport::with_session( + device.session.clone(), + server.upload_base(), + STALE, + )); + let refused = stale + .create_session(&request) + .await + .expect_err("a protocol date before the window is refused"); + match refused { + UploadError::UpgradeRequired { min, max, .. } => { + assert_eq!(min.as_deref(), Some(DEFAULT_MIN)); + assert_eq!(max.as_deref(), Some(DEFAULT_MAX)); + } + other => panic!("expected UpgradeRequired, got {other:?}"), + } + + // The same request at this build's date succeeds — the pin was the only difference. + device + .upload_client(&server) + .create_session(&request) + .await + .expect("this build's protocol date is inside the window"); +} + +/// **E2E case 9**, leg 2: a server whose window excludes this build refuses its writes and +/// advertises the window on every response. +#[tokio::test] +async fn e2e_case_9_a_server_outside_this_builds_window_refuses_writes_with_426() { + let server = Server::boot_with_window("2000-01-01", "2000-01-01").await; + + // The first write any client makes — registration — through the SDK's own auth client. + let Err(refused) = AuthClient::new(&server.auth_base()) + .expect("the auth base parses") + .register("nobody@e2e.capsule.test", PASSWORD) + .await + else { + panic!("a write from outside the window is refused"); + }; + match &refused { + AuthError::Unexpected { status, code, .. } => { + assert_eq!(*status, 426); + assert_eq!(code.as_deref(), Some(VERSION_UNSUPPORTED)); + } + other => panic!("expected a 426 with the gate's code, got {other:?}"), + } + assert_eq!(refused.error_code(), Some(VERSION_UNSUPPORTED)); + + // The window rides every response, an exempt operation's included (`GET /v1/version` is + // one of the ten the design exempts, and this client sends no handshake at all). + let exempt = rest::Client::new(server.base_url()).expect("the API root parses"); + let version = exempt.get_version().await.expect("the version is public"); + let header = |name: &str| { + version + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + assert_eq!( + header("x-capsule-protocol-min").as_deref(), + Some("2000-01-01") + ); + assert_eq!( + header("x-capsule-protocol-max").as_deref(), + Some("2000-01-01") + ); +} + +/// **E2E case 9**, leg 3: reads are admitted at any grammatical date and carry the window; +/// a handshake that does not parse is a `400` everywhere the gate stands. +#[tokio::test] +async fn e2e_case_9_reads_at_an_old_protocol_succeed_and_carry_the_window() { + let server = Server::boot().await; + let device = Device::register(&server, "reader").await; + let feed = format!("{}/v1/sync", server.base_url()); + + // An explicit header wins over the transport's default: the request leaves at `1999-01-01`. + let response = device + .session + .execute(|http| http.get(&feed).header("x-capsule-protocol", STALE)) + .await + .expect("the feed answers"); + assert_eq!( + response.status().as_u16(), + 200, + "a read at an old date is admitted" + ); + let header = |name: &str| { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + assert_eq!( + header("x-capsule-protocol-min").as_deref(), + Some(DEFAULT_MIN) + ); + assert_eq!( + header("x-capsule-protocol-max").as_deref(), + Some(DEFAULT_MAX) + ); + + let malformed = device + .session + .execute(|http| http.get(&feed).header("x-capsule-protocol", "yesterday")) + .await + .expect("the gate answers"); + assert_eq!(malformed.status().as_u16(), 400); + let problem: serde_json::Value = malformed.json().await.expect("a problem body"); + assert_eq!(problem["code"], "error.request.malformed"); +} + +/// The body-less `413`: the transport backstop carries no problem body, so the SDK reports +/// `code: None` rather than a code the server never sent. +#[tokio::test] +async fn a_body_past_the_transport_limit_reaches_the_sdk_as_a_codeless_413() { + let server = Server::boot().await; + let device = Device::register(&server, "escrow").await; + let recovery = RecoveryClient::new(device.session.clone(), server.base_url()) + .expect("the API root parses"); + + // 33 MiB: one past the 32 MiB `BodySize` backstop. Fast Argon2id parameters, because + // nothing here derives — the bytes never reach the escrow route's own checks. + let oversized = WrappedSecret { + mem_kib: 64, + t_cost: 1, + p_cost: 1, + salt: [0; 32], + nonce: [0; 12], + ciphertext: vec![0; 33 * 1024 * 1024], + }; + let refused = recovery + .store_escrow(&oversized) + .await + .expect_err("a body past the transport limit is refused"); + match &refused { + RecoveryError::Malformed { code, .. } => assert!( + code.is_none(), + "a body-less 413 carries no code for the SDK to relay, got {code:?}" + ), + other => panic!("expected Malformed, got {other:?}"), + } + assert_eq!(refused.error_code(), None); + + // The account is otherwise healthy: the same client provisions and reads as before. + let albums = + capsule_sdk::albums::AlbumClient::new(capsule_sdk::albums::AlbumTransport::with_session( + device.session.clone(), + server.albums_base(), + )); + ensure_album(&albums, device.workspace.default_album_id()) + .await + .expect("the session survives the refusal"); +} diff --git a/capsule-sdk/build.rs b/capsule-sdk/build.rs index e4432c42..cc527870 100644 --- a/capsule-sdk/build.rs +++ b/capsule-sdk/build.rs @@ -21,7 +21,7 @@ fn main() { build_rest_client(); } -/// Generate the typed REST client from the committed OpenAPI 3.1 schema (slice `S-D8`). +/// Generate the typed REST client from the committed OpenAPI 3.2 schema (slice `S-D8`). fn build_rest_client() { let manifest_dir = PathBuf::from( std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set by cargo"), @@ -73,8 +73,11 @@ fn build_rest_client() { // spec* — and the alternative is worse in a way worth naming: re-labelling them // `application/octet-stream` to satisfy a generator would tell every client that a document // with a schema it knows is opaque bytes, which is the thing the media type exists to deny. - // `capsule_sdk::directory` already hand-writes two of them for the old reason; the other two - // have no client yet. + // All four are hand-written now, and each one says in its own module doc that spargen is + // the only reason: `capsule_sdk::directory` covers the two device-directory operations, + // `capsule_sdk::upgrade` the proposal, and `capsule_sdk::verify`'s + // `StorageVerifyClient::fetch_receipt` the receipt. Teaching spargen the media type retires + // all four narrowings and all four clients — see the tracking issue on the generator. let omitted = [ spargen::OmitRule::operation(spargen::OmitMethod::Post, "/v1/auth/devices/directory"), spargen::OmitRule::operation( diff --git a/capsule-sdk/src/auth.rs b/capsule-sdk/src/auth.rs index 15b588bd..40a10f6b 100644 --- a/capsule-sdk/src/auth.rs +++ b/capsule-sdk/src/auth.rs @@ -675,6 +675,23 @@ impl Session { Ok(()) } + /// Refresh because the server **rejected** `stale`, and return the token that replaced + /// it (single-flight). + /// + /// The reactive counterpart to [`bearer`](Session::bearer)'s pre-flight refresh, and the + /// primitive [`crate::client::AuthenticatedClient`]'s `401` layer drives. Passing the + /// exact token the server refused is the whole point: `ensure_refreshed`'s `Rejected` + /// arm compares it against the store, so a caller whose stale token has *already* been + /// rotated by a concurrent refresh gets the fresh token back with no second network + /// call. [`refresh`](Session::refresh) cannot serve this — it re-reads the *current* + /// token and would therefore refresh again on top of that rotation, spending a + /// single-use refresh token the server has already closed. + #[instrument(skip_all)] + pub(crate) async fn refresh_rejected(&self, stale: &str) -> Result { + self.ensure_refreshed(RefreshTrigger::Rejected(stale.to_owned())) + .await + } + /// Revoke the session server-side and clear the local store. Idempotent: a /// server that no longer honors the token (or an already-empty store) still /// resolves to a cleared, logged-out session. @@ -706,11 +723,11 @@ impl Session { } /// A currently-valid **bearer access token** for injecting into a request the - /// SDK does not build with [`Session::execute`] — notably the sync feed's gRPC - /// call metadata, where the token rides `authorization` metadata rather than a - /// `reqwest` header. Pre-flight-refreshes exactly like [`Session::execute`]; - /// callers that get an `Unauthenticated`/`401` back re-[`refresh`](Session::refresh) - /// and read a fresh token once. + /// SDK does not build with [`Session::execute`] — notably the generated REST client's + /// token-provider seam ([`crate::client::AuthenticatedClient`]), where the token is + /// attached by the client rather than by this module. Pre-flight-refreshes exactly like + /// [`Session::execute`]; the reactive half — a `401` the pre-flight check could not + /// foresee — is [`refresh_rejected`](Session::refresh_rejected)'s. #[instrument(skip_all)] pub async fn bearer(&self) -> Result { self.valid_access_token().await diff --git a/capsule-sdk/src/client.rs b/capsule-sdk/src/client.rs index bf32b4a1..05da060e 100644 --- a/capsule-sdk/src/client.rs +++ b/capsule-sdk/src/client.rs @@ -11,14 +11,33 @@ //! touch a raw token. The refresh/expiry/single-flight logic is reused wholesale from //! [`crate::auth`]; nothing is duplicated here. //! +//! # Both halves of the refresh contract (slice `S-D17`) +//! +//! The token provider is the **proactive** half: it refreshes when the stored token is within +//! its skew of expiry, before the request leaves. That cannot cover a token the server stops +//! honouring early — a revocation mid-flight, or a clock the two ends disagree about — so +//! [`RefreshOn401`] is the **reactive** half: an [`rest::HttpBackend`] wrapping +//! [`rest::ReqwestBackend`] that, on a `401`, refreshes once through the session and replays +//! the request exactly once. The two are complementary and neither duplicates the other; the +//! refresh itself is still `auth`'s single-flight gate. +//! +//! It sits at the transport seam rather than in each caller, so **every** generated operation +//! is covered by one layer that survives regeneration — no generated code is touched, and +//! there is no per-call retry loop to keep in step. +//! //! Scope: this covers the plain request/response REST surfaces the OpenAPI schema declares //! (auth/session, quota, storage-verify, receipts, devices, escrow, …). The stateful upload -//! protocol ([`crate::upload`]) and the gRPC sync feed ([`crate::sync`]) stay hand-written — -//! they are deliberately *not* routed through the generated client. +//! protocol ([`crate::upload`]) stays hand-written. The sync feed ([`crate::sync`]) *is* a +//! generated operation (`GET /v1/sync`), but [`crate::sync::SyncConsumer`] drives it under its +//! own cursor/anti-rewind state machine and builds its own client, because it also serves a +//! static-token mode that has no session to refresh. use std::ops::Deref; use std::sync::Arc; +use reqwest::header::{AUTHORIZATION, HeaderValue}; +use secrecy::ExposeSecret; + use crate::auth::Session; use crate::rest::{self, Client, Credential}; @@ -104,11 +123,113 @@ impl Deref for AuthenticatedClient { } } +/// The bearer prefix the `Authorization` header carries, and the only credential shape +/// [`RefreshOn401`] recognises as a token it can refresh. +const BEARER_PREFIX: &str = "Bearer "; + +/// The reactive half of the refresh contract (slice `S-D17`): an [`rest::HttpBackend`] that, +/// on a `401`, refreshes the session once and replays the request exactly once. +/// +/// # Why the transport seam and not [`rest::Middleware`] +/// +/// A middleware receives a `Next`, and `Next::run` takes `self` by value while `Next` is +/// neither `Clone` nor constructible outside the generated runtime. A middleware therefore +/// *cannot* send a second time, which is the one thing this layer must do. The backend seam +/// has no such constraint, and spargen's own `RetryBackend` is the precedent — including the +/// `Request::try_clone()`-returns-`None` rule for one-shot bodies. +/// +/// # Exactly once, by construction +/// +/// The replay is straight-line code, not a loop with a counter: one send, one refresh, one +/// replay, and whatever the replay answers is returned as-is. A second `401` is surfaced. +struct RefreshOn401 { + inner: Arc, + session: Session, +} + +// `Session` is not `Debug` (it holds token material), and `HttpBackend` requires `Debug` so the +// generated `ClientCore` stays printable. The manual impl names the layer and the backend under +// it, and shows nothing of the session. +impl std::fmt::Debug for RefreshOn401 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RefreshOn401") + .field("inner", &self.inner) + .finish_non_exhaustive() + } +} + +impl rest::HttpBackend for RefreshOn401 { + fn execute(&self, request: reqwest::Request) -> rest::ExecuteFuture<'_> { + // Own clones of the `Arc`/`Session` so the returned future is self-contained, matching + // the seam's expectations (and `RetryBackend`'s shape). + let inner = self.inner.clone(); + let session = self.session.clone(); + Box::pin(async move { + // A one-shot streaming body cannot be resent intact. Execute the original once and + // return: half a body on the wire twice is worse than a `401` the caller can see. + let Some(mut replay) = request.try_clone() else { + tracing::debug!("request body is not replayable; a 401 will not be retried"); + return inner.execute(request).await; + }; + + let response = inner.execute(request).await?; + if response.status() != reqwest::StatusCode::UNAUTHORIZED { + return Ok(response); + } + + // An operation carrying no bearer has nothing to refresh — that `401` is the + // server's answer about the request, not about a stale token. + let Some(stale) = bearer_of(&replay) else { + tracing::debug!("a 401 arrived on a request carrying no bearer; not retrying"); + return Ok(response); + }; + + let fresh = match session.refresh_rejected(&stale).await { + Ok(fresh) => fresh, + // Deliberately the *original* `401`, not a synthetic transport error: the + // generated operation then maps it to its typed `Status401` and the caller + // reads the server's own `error.*` code — which matters because an unreadable + // revocation ledger is also rendered as `401`, and only that code separates an + // outage from an expiry. + Err(error) => { + tracing::warn!(%error, "a 401 could not be recovered; surfacing it"); + return Ok(response); + } + }; + let Ok(header) = + HeaderValue::from_str(&format!("{BEARER_PREFIX}{}", fresh.expose_secret())) + else { + tracing::warn!( + "the refreshed token is not a valid header value; surfacing the 401" + ); + return Ok(response); + }; + replay.headers_mut().insert(AUTHORIZATION, header); + tracing::info!("the typed client's 401 was refreshed; replaying once"); + inner.execute(replay).await + }) + } +} + +/// The bearer token a prepared request carries, if any. `None` for an unauthenticated +/// operation, and for any credential shape this layer cannot refresh. +fn bearer_of(request: &reqwest::Request) -> Option { + request + .headers() + .get(AUTHORIZATION)? + .to_str() + .ok()? + .strip_prefix(BEARER_PREFIX) + .map(str::to_owned) +} + /// Wire a generated client to `base_url` with a bearer credential that pulls a fresh access -/// token from `session` on demand (pre-flight refresh + single-flight live in the session). +/// token from `session` on demand (the proactive half), executing through [`RefreshOn401`] +/// (the reactive half). fn build_client(base_url: &str, session: Session) -> Result { + let provider_session = session.clone(); let provider: rest::TokenProvider = Arc::new(move || { - let session = session.clone(); + let session = provider_session.clone(); // The session yields a currently-valid bearer, refreshing pre-flight if the stored // token is within its refresh skew of expiry; the failure is mapped into spargen's // provider-error type so a dead session is a request-construction error, not a 401. @@ -120,7 +241,14 @@ fn build_client(base_url: &str, session: Session) -> Result }) }); - let client = Client::with_client(reqwest_client(), base_url) + // `with_backend` builds requests on a default `reqwest::Client` and *executes* them + // through the backend, so the executing client — and with it the TLS stack, the redirect + // policy and the timeouts — is still `reqwest_client()`, one layer down. + let backend: Arc = Arc::new(RefreshOn401 { + inner: Arc::new(rest::ReqwestBackend::new(reqwest_client())), + session, + }); + let client = Client::with_backend(backend, base_url) .map_err(|e| ClientError::InvalidBaseUrl { url: base_url.to_string(), reason: e.to_string(), @@ -132,8 +260,29 @@ fn build_client(base_url: &str, session: Session) -> Result /// The generated client's transport: the SDK's one HTTP client /// ([`crate::net::http_client`]) — rustls only, carrying the protocol handshake on every request /// it sends, the generated operations included. +/// +/// **One per process, shared.** A `reqwest::Client` owns a connection pool, and cloning it +/// shares that pool; building a new one throws the pool away. The FFI's escrow verbs construct +/// a fresh [`AuthenticatedClient`] per call (the API root is a per-call argument), so a +/// per-client transport would mean a fresh TLS handshake for every escrow read on a device +/// that does several during one cadence prompt. Nothing here is configured per instance, so +/// there is nothing to vary: the same client serves them all. +/// +/// **What that does not fix.** `Client::with_backend` still builds its *own* default +/// `reqwest::Client` internally, one per [`AuthenticatedClient`]. That one only assembles +/// requests — every byte is executed through the backend below, and therefore through this +/// shared client — so it opens no connection and costs nothing on the wire; what it costs is +/// one throwaway allocation per construction. Removing even that needs a +/// `with_client_and_backend` constructor spargen does not expose, which is generator work of +/// exactly the same kind as the `application/cbor` gap, and lands where that lands. fn reqwest_client() -> reqwest::Client { - crate::net::http_client().expect("a default rustls reqwest client is always constructible") + static SHARED: std::sync::OnceLock = std::sync::OnceLock::new(); + SHARED + .get_or_init(|| { + crate::net::http_client() + .expect("a default rustls reqwest client is always constructible") + }) + .clone() } #[cfg(test)] @@ -408,6 +557,252 @@ mod tests { ); } + /// An RFC 9457 problem body shaped as the generated `CodedProblem`, so a documented + /// non-success status parses into the operation's typed error rather than a decode + /// failure. + fn problem(status: u16, code: &str) -> String { + serde_json::json!({ + "type": "about:blank", + "title": "Unauthorized", + "status": status, + "detail": "the access token was refused", + "code": code, + }) + .to_string() + } + + /// How many requests the mock saw for `path`. + fn hits(server: &MockServer, path: &str) -> usize { + server + .requests + .lock() + .unwrap() + .iter() + .filter(|r| r.path == path) + .count() + } + + /// The bearer each request for `path` carried, in order. + fn bearers(server: &MockServer, path: &str) -> Vec> { + server + .requests + .lock() + .unwrap() + .iter() + .filter(|r| r.path == path) + .map(|r| r.authorization.clone()) + .collect() + } + + // ── S-D17: the reactive half ──────────────────────────────────────────────────────── + + /// **The slice's Done-when.** A token that is valid as far as the *client* can tell and + /// refused by the server — the race the pre-flight check cannot close — is refreshed once + /// and the call replayed once, and the replay carries the rotated token. + #[tokio::test] + async fn a_401_is_refreshed_once_and_the_call_replayed() { + let seen = Arc::new(AtomicUsize::new(0)); + let quota_calls = seen.clone(); + let handler: Handler = Arc::new(move |path| { + let quota_calls = quota_calls.clone(); + Box::pin(async move { + match path.as_str() { + "/refresh" => MockResponse { + status: 200, + body: token_json("access-2", "refresh-2", far_future()), + }, + // The first attempt is refused; the replay is honoured. A server that + // revoked the session mid-flight looks exactly like this. + "/v1/quota" => { + if quota_calls.fetch_add(1, Ordering::SeqCst) == 0 { + MockResponse { + status: 401, + body: problem( + 401, + capsule_i18n::error_codes::REQUEST_UNAUTHENTICATED, + ), + } + } else { + MockResponse { + status: 200, + body: r#"{"state":"ok","used":5}"#.to_string(), + } + } + } + _ => MockResponse { + status: 404, + body: "{}".to_string(), + }, + } + }) + }); + let server = start_mock(handler).await; + // Far-future expiry: the pre-flight check is satisfied, so the *only* thing that can + // rescue this call is the reactive layer. + let session = session_with(&server.base_url, "access-1", "refresh-1", far_future()); + let client = AuthenticatedClient::new(&server.base_url, session).unwrap(); + + let quota = client + .get_quota(capsule_core::crypto::primitives::PROTOCOL_VERSION, None) + .await + .unwrap() + .into_inner(); + assert_eq!( + quota.used, 5, + "the replayed call is the one the caller sees" + ); + + assert_eq!(hits(&server, "/refresh"), 1, "exactly one refresh"); + assert_eq!( + hits(&server, "/v1/quota"), + 2, + "one attempt, one replay — no loop" + ); + assert_eq!( + bearers(&server, "/v1/quota"), + vec![ + Some("Bearer access-1".to_string()), + Some("Bearer access-2".to_string()), + ], + "the replay must carry the rotated token, not the one the server just refused" + ); + } + + /// A server that refuses every credential is answered with exactly one replay, and the + /// second `401` reaches the caller as the operation's typed error carrying the server's + /// own `error.*` code. `exactly once` is the property: not twice, not a loop. + #[tokio::test] + async fn a_persistent_401_is_surfaced_after_exactly_one_replay() { + let handler: Handler = Arc::new(|path| { + Box::pin(async move { + match path.as_str() { + "/refresh" => MockResponse { + status: 200, + body: token_json("access-2", "refresh-2", far_future()), + }, + "/v1/quota" => MockResponse { + status: 401, + body: problem(401, capsule_i18n::error_codes::REQUEST_UNAUTHENTICATED), + }, + _ => MockResponse { + status: 404, + body: "{}".to_string(), + }, + } + }) + }); + let server = start_mock(handler).await; + let session = session_with(&server.base_url, "access-1", "refresh-1", far_future()); + let client = AuthenticatedClient::new(&server.base_url, session).unwrap(); + + let error = client + .get_quota(capsule_core::crypto::primitives::PROTOCOL_VERSION, None) + .await + .expect_err("a credential the server never honours must fail"); + let rest::Error::Api(response) = &error else { + panic!("expected the operation's typed API error, got {error:?}"); + }; + let rest::GetQuotaError::Status401(problem) = response.inner() else { + panic!("expected a typed 401, got {:?}", response.inner()); + }; + assert_eq!( + problem.code, + capsule_i18n::error_codes::REQUEST_UNAUTHENTICATED + ); + + assert_eq!(hits(&server, "/refresh"), 1); + assert_eq!( + hits(&server, "/v1/quota"), + 2, + "exactly one replay — a retry loop would keep going" + ); + } + + /// When the refresh itself fails, the caller gets the **server's** `401` back rather than + /// a synthesized transport error — so the typed `Status401` mapping still fires and the + /// `error.*` code survives. That code is the only thing separating an expired token from + /// an unreadable revocation ledger, which the server also renders as `401`. + #[tokio::test] + async fn a_401_whose_refresh_fails_keeps_the_servers_own_401() { + let handler: Handler = Arc::new(|path| { + Box::pin(async move { + match path.as_str() { + // The refresh token is gone too: nothing here can be rescued. + "/refresh" => MockResponse { + status: 401, + body: problem(401, capsule_i18n::error_codes::AUTH_SESSION_EXPIRED), + }, + "/v1/quota" => MockResponse { + status: 401, + body: problem(401, capsule_i18n::error_codes::AUTH_UNAVAILABLE), + }, + _ => MockResponse { + status: 404, + body: "{}".to_string(), + }, + } + }) + }); + let server = start_mock(handler).await; + let session = session_with(&server.base_url, "access-1", "refresh-1", far_future()); + let client = AuthenticatedClient::new(&server.base_url, session).unwrap(); + + let error = client + .get_quota(capsule_core::crypto::primitives::PROTOCOL_VERSION, None) + .await + .expect_err("nothing can rescue this"); + let rest::Error::Api(response) = &error else { + panic!("a failed refresh must not mask the 401 as a transport error: {error:?}"); + }; + let rest::GetQuotaError::Status401(problem) = response.inner() else { + panic!("expected a typed 401, got {:?}", response.inner()); + }; + assert_eq!( + problem.code, + capsule_i18n::error_codes::AUTH_UNAVAILABLE, + "the code the caller reads is the one the *operation* answered, not the refresh's" + ); + assert_eq!( + hits(&server, "/v1/quota"), + 1, + "a refresh that failed produces nothing worth replaying" + ); + } + + /// An unauthenticated operation's `401` is the server's answer about the request, not + /// about a stale token: there is no bearer to refresh, so nothing is refreshed and + /// nothing is replayed. + #[tokio::test] + async fn an_unauthenticated_401_is_never_retried() { + let handler: Handler = Arc::new(|path| { + Box::pin(async move { + match path.as_str() { + "/refresh" => MockResponse { + status: 200, + body: token_json("access-2", "refresh-2", far_future()), + }, + _ => MockResponse { + status: 401, + body: problem(401, capsule_i18n::error_codes::REQUEST_UNAUTHENTICATED), + }, + } + }) + }); + let server = start_mock(handler).await; + let session = session_with(&server.base_url, "access-1", "refresh-1", far_future()); + let client = AuthenticatedClient::new(&server.base_url, session).unwrap(); + + // `get_version` declares no security requirement, so the generated client attaches no + // credential at all. + client + .get_version() + .await + .expect_err("the mock refuses everything"); + assert_eq!(hits(&server, "/v1/version"), 1, "one request, no replay"); + assert_eq!(hits(&server, "/refresh"), 0, "and no refresh"); + assert_eq!(bearers(&server, "/v1/version"), vec![None]); + } + /// The session's pre-flight refresh fires *through the revived client*: with a stored /// access token already past expiry, the first typed call refreshes once (via the token /// provider), and the API request carries the rotated token — not the stale one. diff --git a/capsule-sdk/src/ffi.rs b/capsule-sdk/src/ffi.rs index a3cd9a63..cc734170 100644 --- a/capsule-sdk/src/ffi.rs +++ b/capsule-sdk/src/ffi.rs @@ -63,7 +63,7 @@ pub use workspace::{ /// variant carries the stable `error.*` catalog `code` (when one applies — clients /// localize it) and the English detail `message` (stays English), mirroring the /// SDK's `{ error, code }` contract so foreign apps switch on the code, never a -/// bare HTTP/gRPC status. +/// bare HTTP status. #[derive(Debug, thiserror::Error, uniffi::Error)] pub enum FfiError { /// An authentication flow (login/register/refresh/logout) failed. @@ -99,8 +99,12 @@ pub enum FfiError { message: String, }, /// A master-key escrow flow (store/fetch) failed. - #[error("escrow failed: {message}")] + #[error("escrow failed ({code:?}): {message}")] Escrow { + /// Stable `error.*` catalog code, when the server supplied one — `error.escrow.*` + /// separates "you have no recovery backup" (a setup prompt) from "we could not read + /// it" (a retry), which is the whole reason the catalog distinguishes them. + code: Option, /// English detail (developer/log message). message: String, }, @@ -131,12 +135,25 @@ impl From for FfiError { impl From for FfiError { fn from(err: RecoveryError) -> Self { - // An auth failure under an escrow call keeps its auth identity so callers can trigger - // interactive re-authentication, exactly as the upload mapping does. - if let RecoveryError::Auth(auth) = err { - return auth.into(); + // A refused credential under an escrow call keeps its auth identity so callers can + // trigger interactive re-authentication, exactly as the upload mapping does. Before + // the escrow calls moved onto the generated client this arrived as + // `RecoveryError::Auth`; routing `Unauthorized` here is what stops the move from + // silently downgrading "sign in again" into "escrow failed". + if let RecoveryError::Unauthorized { code, detail } = err { + return Self::Auth { + code, + message: detail, + }; + } + // A malformed argument is the caller's, not the escrow surface's. + if let RecoveryError::InvalidBaseUrl { .. } = err { + return Self::InvalidArgument { + message: err.to_string(), + }; } Self::Escrow { + code: err.error_code().map(str::to_owned), message: err.to_string(), } } @@ -758,7 +775,7 @@ impl FfiSession { Ok(page.into()) } - /// Store or replace this account's **master-key escrow blob** (`PUT /backup/escrow`). + /// Store or replace this account's **master-key escrow blob** (`PUT /v1/auth/escrow`). /// `blob` is the opaque canonical CBOR /// [`FfiWorkspace::escrow_blob`](FfiWorkspace::escrow_blob) minted — the master key itself /// never crosses this boundary in either direction. @@ -767,26 +784,35 @@ impl FfiSession { /// secret a rotation retired unwraps nothing. /// /// `api_base_url` is the API root the session authenticates against (the per-call endpoint - /// convention this surface already uses for `sync_pull`). + /// convention this surface already uses for `sync_pull`). A URL operation paths cannot hang + /// off is [`FfiError::InvalidArgument`]; a refused credential is [`FfiError::Auth`], so a + /// caller re-authenticates rather than retrying; everything else is + /// [`FfiError::Escrow`] with the server's `error.escrow.*` code when it sent one. pub async fn escrow_put(&self, api_base_url: String, blob: Vec) -> Result<(), FfiError> { let blob = capsule_core::cbor::from_slice(&blob).map_err(|e| FfiError::InvalidArgument { message: format!("escrow blob is not a canonical WrappedSecret: {e}"), })?; - RecoveryClient::new(self.session.clone(), &api_base_url) + RecoveryClient::new(self.session.clone(), &api_base_url)? .store_escrow(&blob) .await?; Ok(()) } - /// Fetch this account's escrow blob (`GET /backup/escrow`) as opaque canonical CBOR — the + /// Fetch this account's escrow blob (`GET /v1/auth/escrow`) as opaque canonical CBOR — the /// bytes [`FfiWorkspace::verify_escrow_blob`](FfiWorkspace::verify_escrow_blob) checks and - /// a recovery flow unwraps. Fails with an `Escrow` error when no escrow is enrolled yet. + /// a recovery flow unwraps. + /// + /// No escrow enrolled yet is [`FfiError::Escrow`] carrying `error.escrow.not_stored` — the + /// code that separates "set up a recovery key" from "we could not read the one you have". + /// A refused credential is [`FfiError::Auth`]. pub async fn escrow_get(&self, api_base_url: String) -> Result, FfiError> { - let cache = RecoveryClient::new(self.session.clone(), &api_base_url) + let cache = RecoveryClient::new(self.session.clone(), &api_base_url)? .fetch_escrow() .await?; capsule_core::cbor::to_canonical_vec(cache.blob()).map_err(|e| FfiError::Escrow { + // A local encode failure is ours, not the server's: no catalog code applies. + code: None, message: format!("encoding the fetched escrow blob failed: {e}"), }) } diff --git a/capsule-sdk/src/ffi/tests.rs b/capsule-sdk/src/ffi/tests.rs index 4cf8612f..b96a9f9e 100644 --- a/capsule-sdk/src/ffi/tests.rs +++ b/capsule-sdk/src/ffi/tests.rs @@ -17,10 +17,11 @@ //! publish. Every verb reaches `capsule-core` for its crypto; what is under test here //! is the wiring, the shapes, and the verdicts. //! -//! `sync_pull` itself is gRPC and is exercised by the native harness against the real -//! server; its Rust-side shape is compiled here (the surface builds) but not -//! behaviorally driven — the sync-apply test below feeds `apply_sync_entry` the exact -//! three byte strings a feed entry carries, which is the half `S-P1` owns. +//! `sync_pull` itself rides the generated `GET /v1/sync` operation (`S-D28` retired the gRPC +//! feed) and is exercised over a socket in `capsule-server/tests/sdk_client.rs` against the +//! real router; its Rust-side shape is compiled here (the surface builds) but not behaviorally +//! driven — the sync-apply test below feeds `apply_sync_entry` the exact three byte strings a +//! feed entry carries, which is the half `S-P1` owns. use std::sync::Arc; @@ -294,6 +295,12 @@ fn enroll(root: &std::path::Path) -> Arc { /// The escrow endpoints are **stateful** — a `PUT` stores the bytes verbatim and a `GET` /// serves them back, exactly as the single-active-escrow contract says — so `escrow_put` /// and `escrow_get` can be asserted as a real round trip rather than two isolated calls. +/// +/// They are served on `/v1/auth/escrow` under the API root, which is the path the committed +/// document declares and the generated client therefore requests. The `PUT` answers a JSON +/// `StoreEscrowResponse` and the empty-escrow `GET` answers an RFC 9457 problem, because a +/// generated operation decodes both — a bare `204` or a body-less `404` would arrive as a +/// decode failure rather than as the typed outcome this test is asserting. async fn flow_server() -> MockServer { let escrow: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); MockServer::start( @@ -310,17 +317,36 @@ async fn flow_server() -> MockServer { .to_string(), ), ("PATCH", "/upload/sess-1") => MockResponse::new(200, "OK"), - ("PUT", "/api/backup/escrow") => { - if let Ok(mut stored) = escrow.lock() { + ("PUT", "/api/v1/auth/escrow") => { + let replaced = if let Ok(mut stored) = escrow.lock() { + let replaced = !stored.is_empty(); stored.clone_from(&req.body); - } - MockResponse::new(204, "No Content") + replaced + } else { + false + }; + MockResponse::new(200, "OK").json_body( + serde_json::json!({ + "stored_at": "2026-01-01T00:00:00Z", + "replaced": replaced, + }) + .to_string(), + ) } - ("GET", "/api/backup/escrow") => { + ("GET", "/api/v1/auth/escrow") => { let stored = escrow.lock().map(|s| s.clone()).unwrap_or_default(); if stored.is_empty() { // Nothing enrolled yet — the typed `NotEnrolled` path. - MockResponse::new(404, "Not Found") + MockResponse::new(404, "Not Found").json_body( + serde_json::json!({ + "type": "about:blank", + "title": "Not found", + "status": 404, + "detail": "no escrow has been stored for this account", + "code": "error.escrow.not_stored", + }) + .to_string(), + ) } else { let mut response = MockResponse::new(200, "OK") .header("Content-Type", "application/octet-stream"); diff --git a/capsule-sdk/src/lib.rs b/capsule-sdk/src/lib.rs index be738cd2..95417c08 100644 --- a/capsule-sdk/src/lib.rs +++ b/capsule-sdk/src/lib.rs @@ -3,15 +3,24 @@ //! //! # REST client generation (spargen; slice `S-D8` in the repo-root `SLICES.md`) //! -//! The typed REST client ([`rest`]) is generated from the server's **OpenAPI 3.1** schema by +//! The typed REST client ([`rest`]) is generated from the server's **OpenAPI 3.2** schema by //! `spargen`, our in-house generator, at build time (see `build.rs`). The previous progenitor //! pipeline is gone deliberately: progenitor consumes OpenAPI 3.0 only, which forced a lossy -//! 3.1→3.0 schema down-conversion — a standing source of drift and failures. We do not -//! downgrade schemas. [`client::AuthenticatedClient`] wraps the generated `Client`, composing -//! it with [`auth`]'s session/token store so callers issue typed calls and never juggle raw -//! tokens. The hand-written upload/sync surfaces ([`upload`], [`sync`]) stay hand-written — -//! their protocols are too stateful for request/response codegen; the generated client covers -//! the plain request/response surfaces (auth, quota, storage-verify, receipts, devices, …). +//! down-conversion of the document the server actually emits — a standing source of drift and +//! failures. We do not downgrade schemas. [`client::AuthenticatedClient`] wraps the generated +//! `Client`, composing it with [`auth`]'s session/token store so callers issue typed calls and +//! never juggle raw tokens. +//! +//! The generated client covers the plain request/response surfaces (auth, quota, +//! storage-verify, receipts, devices, escrow, the sync feed, …). What stays hand-written is +//! *orchestration*, never a second parser: +//! +//! - [`upload`] — the resumable upload state machine, whose protocol is too stateful for +//! request/response codegen; +//! - [`sync`] — the cursor and anti-rewind state machine **over** the generated `GET /v1/sync` +//! operation (`S-D28` retired the gRPC feed; the wire here is generated like every other); +//! - [`directory`], [`upgrade`], and [`verify`]'s receipt fetch — the four `application/cbor` +//! operations `spargen` 0.4 cannot lower, narrowed out in `build.rs`. pub mod albums; pub mod auth; @@ -25,6 +34,11 @@ pub mod push; pub mod recovery; pub mod staged; pub mod sync; +/// The album-upgrade proposal client (`S-C24`). Hand-written for one reason only: its request +/// body is `application/cbor`, which `spargen` 0.4 cannot lower, so `build.rs` narrows the +/// operation out of the generated client. The `GET` and `DELETE` on the same path are JSON and +/// *are* generated — reach them through [`client::AuthenticatedClient`]. +pub mod upgrade; pub mod upload; pub mod verify; @@ -32,14 +46,14 @@ pub mod verify; #[cfg(test)] mod testmock; -/// The typed REST client generated by `spargen` from the server's committed OpenAPI **3.1** +/// The typed REST client generated by `spargen` from the server's committed OpenAPI **3.2** /// schema (`openapi.json`), emitted into `OUT_DIR` by `build.rs` and included verbatim here /// (slice `S-D8`). One `async` method per operation, typed models, typed errors, and an /// embedded freestanding `reqwest` runtime — `spargen` never enters the runtime dependency -/// tree. Do not edit: regenerate the schema with `mise run openapi`; the client re-generates -/// on every build. The ergonomic, session-composed entry point is -/// [`client::AuthenticatedClient`]; reach for [`rest::Client`] directly only for -/// unauthenticated calls. +/// tree. Do not edit: regenerate the schema with `mise run openapi-kynos` (and +/// `mise run openapi-check-kynos` gates it); the client re-generates on every build. The +/// ergonomic, session-composed entry point is [`client::AuthenticatedClient`]; reach for +/// [`rest::Client`] directly only for unauthenticated calls. pub mod rest { #![allow( clippy::all, diff --git a/capsule-sdk/src/push.rs b/capsule-sdk/src/push.rs index 2d82379d..4b2ad5b4 100644 --- a/capsule-sdk/src/push.rs +++ b/capsule-sdk/src/push.rs @@ -14,6 +14,12 @@ //! returns the authoritative offset; across an asset's blobs, a `duplicate_blob` answer is a //! merge, not an error. Re-running a push against an unchanged library is therefore a no-op. //! +//! **Every blob this module ships is ciphertext.** The original is re-derived from its +//! manifest's nonce prefix, the metadata blob is carried sealed, and **derivative blobs are +//! encrypted too** — `capsule-core` re-derives each one from the plaintext it holds locally +//! using the prefix that derivative's signed manifest recorded. Nothing here decrypts, encrypts, +//! or inspects a blob; it moves opaque bytes. +//! //! **One deviation from "the envelope mirrors the signed manifest", and it is the server's //! rule:** invariant 15 requires `manifest_envelope.ciphertext_hash == hash` (the top-level //! declared content address of *this* blob). A bundle's metadata and derivative blobs are not diff --git a/capsule-sdk/src/recovery/mod.rs b/capsule-sdk/src/recovery/mod.rs index 8ae4c232..7846c98b 100644 --- a/capsule-sdk/src/recovery/mod.rs +++ b/capsule-sdk/src/recovery/mod.rs @@ -3,7 +3,7 @@ //! Re-Wrap]). //! //! This module owns the **client half** of the master-key recovery story that the -//! server escrow surface (slice `S-C12`, `PUT`/`GET /backup/escrow`) and the core +//! server escrow surface (slice `S-C12`, `PUT`/`GET /v1/auth/escrow`) and the core //! crypto ([`capsule_core::backup`]) make possible. It has two cohesive halves: //! //! - **[`cadence`]** — the pure, network-free scheduler and prompt state machine @@ -19,11 +19,27 @@ //! [`WrappedSecret`] — byte-identical to what the server stores verbatim and to what the //! core restore path unwraps. //! +//! # The wire is the generated client, not a path this module builds +//! +//! Both escrow operations are `application/octet-stream` in each direction, which is a media +//! type `spargen` lowers, so both are **generated** and neither is narrowed out in +//! `build.rs`. [`RecoveryClient`] therefore orchestrates +//! [`AuthenticatedClient::fetch_escrow`](crate::rest::Client::fetch_escrow) and +//! [`store_escrow`](crate::rest::Client::store_escrow) and hand-writes no request. That is +//! not a preference: `AGENTS.md` requires that everything which parses or serializes is +//! generated, *including* the byte-serving endpoints, and the reason is this module's own +//! history. It used to build `{api_root}/backup/escrow` from a `const` — the Salvo document's +//! path — and when the contract was re-sourced from Kynos to `/v1/auth/escrow` nothing +//! noticed, because a route in a string constant is checked by no gate and this module's own +//! mock answered whichever path it was handed. +//! //! [Backup — Recovery Verification Cadence]: https://docs/design/backup-recovery/#recovery-verification-cadence //! [§ On Repeated Failure: Guided Re-Wrap]: https://docs/design/backup-recovery/#on-repeated-failure-guided-re-wrap pub mod cadence; +use std::sync::Arc; + pub use cadence::{ BACKOFF_INTERVAL_SECS, CAP_INTERVAL_SECS, INITIAL_INTERVAL_SECS, MAX_CONSECUTIVE_SNOOZES, REWRAP_FAILURE_THRESHOLD, REWRAP_MIN_SESSIONS, RearmTrigger, RecoveryCadence, SnoozeDuration, @@ -33,40 +49,118 @@ use capsule_core::backup::{VerifyOutcome, split_seed_2of3, verify_recovery_secre use capsule_core::crypto::primitives::{Argon2Params, DeviceTier}; use capsule_core::crypto::pwkdf::{self, WrappedSecret}; use capsule_core::crypto::rng; +use capsule_i18n::error_codes; use tracing::instrument; -use crate::auth::{AuthError, Session}; - -/// The escrow endpoint path, appended to the caller's API base. -const ESCROW_PATH: &str = "backup/escrow"; +use crate::auth::Session; +use crate::client::{AuthenticatedClient, ClientError}; +use crate::rest; /// Everything the networked recovery flows can fail with. Callers switch on the typed -/// variant, never a bare HTTP status. +/// variant (or its stable `error.*` code), never a bare HTTP status. #[derive(Debug, thiserror::Error)] pub enum RecoveryError { - /// The authenticated request itself failed (transport, session expiry, refresh). - #[error(transparent)] - Auth(#[from] AuthError), - /// Reading the escrow response body off the wire failed. - #[error("reading escrow response body failed: {0}")] - Body(#[source] reqwest::Error), - /// The caller has no escrow stored yet (server returned `404`). Enroll one first. + // There is deliberately no `Auth(AuthError)` variant any more. The session used to build + // these requests itself, so a dead session surfaced as its own typed `AuthError`; the + // generated client attaches the bearer through a token-provider seam that flattens our + // `AuthError` to a string, so the same event now arrives as `Unauthorized` — which is + // where the FFI's `Auth` mapping reads it from. An unconstructible variant would be a + // promise no code path can keep. + /// The API root is not a URL the generated client can hang operation paths off. + #[error("invalid base URL {url:?}: {reason}")] + InvalidBaseUrl { + /// The offending URL. + url: String, + /// Why the generated client rejected it. + reason: String, + }, + /// The call never reached a server answer: DNS, connection refused, a TLS handshake, a + /// timeout, a reset mid-body, or a refusal this client could not parse. Transient — the + /// cadence's next tick tries again. + /// + /// Note that reqwest classifies *every* failure of the request it executes as a request + /// error, so the generated taxonomy's `RequestConstruction` class carries connection + /// failures as well as genuine pre-flight ones; both land here. + #[error("the escrow endpoint could not be reached: {0}")] + Transport(String), + /// The credential was refused (`401`/`403`) and a refresh did not recover it — the user + /// must re-authenticate. + /// + /// `code` is whatever the server stamped on the problem body. Today Capsule stamps one + /// code (`error.request.unauthenticated`) on every `401` it renders, so this does not yet + /// separate an expired token from an unreadable revocation ledger; the field carries the + /// code so that it will the moment the server distinguishes them. + #[error("the escrow endpoint refused the credential: {detail}")] + Unauthorized { + /// The stable `error.*` catalog code from the problem body, when the failure came + /// with one. `None` when the credential could not be produced at all — there was no + /// server answer to carry a code. + code: Option, + /// English detail from the problem body. + detail: String, + }, + /// The caller has no escrow stored yet (server returned a coded `404`). Enroll one first. #[error("no escrow stored for this account")] NotEnrolled, + /// The server refused the blob as one that cannot be an escrow at any version — empty or + /// past the coarse ceiling (`400`), not the declared media type (`415`), or past the + /// transport's body limit (`413`). Retrying the same bytes changes nothing. + #[error("the server rejected the escrow blob: {detail}")] + Malformed { + /// The stable `error.*` catalog code the refusal carried. + code: Option, + /// English detail from the problem body. + detail: String, + }, + /// The escrow store could not answer (`500`). Transient, and coded + /// `error.escrow.unavailable` — which is why it is not folded into + /// [`Transport`](RecoveryError::Transport): a caller that localizes codes has one to show. + #[error("the escrow store could not answer: {detail}")] + Unavailable { + /// The stable `error.*` catalog code the refusal carried. + code: Option, + /// English detail from the problem body. + detail: String, + }, /// The escrow bytes could not be (de)serialized as the canonical `WrappedSecret`. #[error("escrow blob codec error: {0}")] Codec(String), /// The (re-)wrap of the master key under the fresh secret failed in core. #[error("re-wrapping the master key failed: {0}")] Wrap(String), - /// The server returned an unmodeled status. + /// The server returned a status this client does not model as an escrow outcome — an + /// unmodeled status, the body-size backstop's body-less `413` on a read, or the protocol + /// gate's `426` (a write from outside the server's window, issue #404). #[error("unexpected {status} response from the escrow endpoint")] Unexpected { /// The HTTP status code the server returned. status: u16, + /// The stable `error.*` catalog code the response carried, when it came with a coded + /// problem body — `error.protocol.version_unsupported` on a `426`, the one that means + /// "update the client". `None` when there was no body to read a code from. + code: Option, }, } +impl RecoveryError { + /// The stable `error.*` catalog code a client localizes, when one applies. The English + /// [`Display`](std::fmt::Display) form stays the developer/log detail. + #[must_use] + pub fn error_code(&self) -> Option<&str> { + match self { + Self::Unauthorized { code, .. } + | Self::Malformed { code, .. } + | Self::Unavailable { code, .. } + | Self::Unexpected { code, .. } => code.as_deref(), + // The one code this module states rather than reads. `NotEnrolled` is a *state* + // ("this account has escrowed nothing"), not a message, and the server's own code + // for that state is this constant — see `capsule-server/src/routes/escrow.rs`. + Self::NotEnrolled => Some(error_codes::ESCROW_NOT_STORED), + _ => None, + } + } +} + /// A client-side cached copy of the server escrow blob. /// /// Fetched at enrollment and refreshed opportunistically (SSoT § Local Verification); @@ -208,68 +302,88 @@ pub struct GuidedRewrap { } /// The networked recovery client: escrow cache/refresh, stale-cache-aware local -/// verification, and the guided re-wrap. It borrows an authenticated [`Session`], so -/// every call rides the SDK's bearer/refresh machinery. +/// verification, and the guided re-wrap. +/// +/// It holds one [`AuthenticatedClient`], so every call rides the generated operation paths +/// and the SDK's bearer/refresh machinery, and this module states no route of its own. The +/// client is behind an [`Arc`] only so [`RecoveryClient`] stays [`Clone`] — the cadence hands +/// one client to several prompts. There is deliberately no repoint/session-swap accessor: +/// `AuthenticatedClient`'s own take `&mut self` and are unreachable through the `Arc`, and an +/// escrow client that changed origin mid-cadence would be a way to pull one account's escrow +/// into another's cache. Build a new one instead. #[derive(Clone)] pub struct RecoveryClient { - session: Session, - escrow_url: String, + client: Arc, } impl RecoveryClient { - /// Build a recovery client against the API base URL (the same base the auth session - /// authenticates against, e.g. `https://api.example.com`). - #[must_use] - pub fn new(session: Session, api_base_url: &str) -> Self { - let escrow_url = format!("{}/{ESCROW_PATH}", api_base_url.trim_end_matches('/')); - Self { - session, - escrow_url, - } + /// Build a recovery client against the **API root** — the origin the generated operation + /// paths hang off (e.g. `https://api.example.com`), which is the same base + /// [`AuthenticatedClient`] and [`crate::sync::SyncConsumer`] take. + /// + /// # Errors + /// + /// [`RecoveryError::InvalidBaseUrl`] when `api_base_url` is not a URL operation paths can + /// hang off. Fallible where the old hand-written `format!` was not, because the generated + /// client parses the base once at construction rather than per call. + pub fn new(session: Session, api_base_url: &str) -> Result { + let client = + AuthenticatedClient::new(api_base_url, session).map_err(|error| match error { + ClientError::InvalidBaseUrl { url, reason } => { + RecoveryError::InvalidBaseUrl { url, reason } + } + })?; + Ok(Self { + client: Arc::new(client), + }) } - /// Fetch the current escrow blob from the server (`GET /backup/escrow`) into a fresh + /// Fetch the current escrow blob from the server (`GET /v1/auth/escrow`) into a fresh /// [`EscrowCache`]. `404` maps to [`RecoveryError::NotEnrolled`]. #[instrument(skip_all)] pub async fn fetch_escrow(&self) -> Result { - let response = self.session.execute(|c| c.get(&self.escrow_url)).await?; - let status = response.status(); - match status { - reqwest::StatusCode::OK => { - let bytes = response.bytes().await.map_err(RecoveryError::Body)?; - tracing::debug!(len = bytes.len(), "fetched escrow blob"); - EscrowCache::from_wire(&bytes) - } - reqwest::StatusCode::NOT_FOUND => Err(RecoveryError::NotEnrolled), - other => Err(RecoveryError::Unexpected { - status: other.as_u16(), - }), - } + // The protocol date is a required parameter of every gated operation in the document, + // so the generated signature asks for it; the value is the build's own, the same one the + // transport's default header carries. The suite and sidecar schema ride the transport. + let bytes = self + .client + .fetch_escrow( + capsule_core::crypto::primitives::PROTOCOL_VERSION, + rest::FetchEscrowParams::default(), + ) + .await + .map_err(fetch_escrow_error)? + .into_inner(); + tracing::debug!(len = bytes.len(), "fetched escrow blob"); + EscrowCache::from_wire(&bytes) } - /// Store or replace the caller's escrow blob (`PUT /backup/escrow`). Single active + /// Store or replace the caller's escrow blob (`PUT /v1/auth/escrow`). Single active /// escrow: the server overwrites any prior blob in the same transaction (S-C12). + /// + /// The canonical CBOR goes on the wire verbatim; `replaced` and `stored_at` are logged + /// rather than returned, because no caller has asked for them yet and a return type is + /// harder to widen than a log line. #[instrument(skip_all)] pub async fn store_escrow(&self, blob: &WrappedSecret) -> Result<(), RecoveryError> { let body = capsule_core::cbor::to_canonical_vec(blob) .map_err(|e| RecoveryError::Codec(e.to_string()))?; - let response = self - .session - .execute(|c| { - c.put(&self.escrow_url) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(body.clone()) - }) - .await?; - let status = response.status(); - if status.is_success() { - tracing::info!("escrow blob stored (single active escrow: any prior blob replaced)"); - Ok(()) - } else { - Err(RecoveryError::Unexpected { - status: status.as_u16(), - }) - } + let stored = self + .client + .store_escrow( + capsule_core::crypto::primitives::PROTOCOL_VERSION, + rest::StoreEscrowParams::default(), + &rest::types::RequestBody::from(body), + ) + .await + .map_err(store_escrow_error)? + .into_inner(); + tracing::info!( + stored_at = %stored.stored_at, + replaced = stored.replaced, + "escrow blob stored (single active escrow: any prior blob replaced)" + ); + Ok(()) } /// Local recovery-secret verification with the **stale-cache rule** (SSoT § Local @@ -367,6 +481,168 @@ impl RecoveryClient { } } +/// Map a `GET /v1/auth/escrow` refusal onto its typed variant. +/// +/// One readable status table rather than a match buried in the request path. The *inner* +/// match is exhaustive over the generated enum, so a status the document adds to this +/// operation stops the build here; a new `rest::Error` **class** still falls through to +/// [`wire_error`]'s catch-all. +fn fetch_escrow_error(error: rest::Error) -> RecoveryError { + match error { + rest::Error::Api(response) => match response.into_inner() { + rest::FetchEscrowError::Status404(_) => RecoveryError::NotEnrolled, + // The protocol gate's malformed-handshake answer (issue #404): a read is admitted at + // any grammatical protocol date, so the only `400` this operation renders is a + // request whose handshake headers did not parse. Unreachable from this client — the + // transport always sends the build's own — and carried with its code rather than + // swallowed, so a caller that localizes codes still has the server's. + rest::FetchEscrowError::Status400(problem) => RecoveryError::Malformed { + code: Some(problem.code.clone()), + detail: detail(&problem), + }, + rest::FetchEscrowError::Status401(problem) + | rest::FetchEscrowError::Status403(problem) => refused(&problem), + rest::FetchEscrowError::Status500(problem) => unavailable(&problem), + // Declared by the transport backstop and unreachable on a body-less `GET`; kept + // honest rather than folded into a class it does not belong to. + rest::FetchEscrowError::Status413 => RecoveryError::Unexpected { + status: 413, + code: None, + }, + }, + other => wire_error(&other), + } +} + +/// Map a `PUT /v1/auth/escrow` refusal onto its typed variant. +fn store_escrow_error(error: rest::Error) -> RecoveryError { + match error { + rest::Error::Api(response) => match response.into_inner() { + // `400` and `415` are the same answer to the caller: these bytes are not an + // escrow, and sending them again will not help. + rest::StoreEscrowError::Status400(problem) + | rest::StoreEscrowError::Status415(problem) => RecoveryError::Malformed { + code: Some(problem.code.clone()), + detail: detail(&problem), + }, + // `426` is the protocol gate refusing a write from outside the server's window + // (issue #404). It says nothing about the *blob*, so it is not `Malformed`; it is + // an outcome this module does not model, carried with the server's own code — + // `error.protocol.version_unsupported`, the one that means "update the client" — + // so a caller localizing codes reads the gate's judgement, not this client's. + rest::StoreEscrowError::Status426(problem) => RecoveryError::Unexpected { + status: 426, + code: Some(problem.code.clone()), + }, + rest::StoreEscrowError::Status401(problem) + | rest::StoreEscrowError::Status403(problem) => refused(&problem), + rest::StoreEscrowError::Status500(problem) => unavailable(&problem), + // The body-size backstop carries no problem body at all, so there is no code to + // carry and this client does not invent one. Every other code in this module is + // the server's own, and a code minted here would assert that the server said + // something it did not — a client localizing it would be reading the SDK's guess + // as the server's judgement. The English detail says what happened instead; the + // variant already says "these bytes will not do, do not resend them". + rest::StoreEscrowError::Status413 => RecoveryError::Malformed { + code: None, + detail: "the escrow blob exceeds the server's request-body limit".to_owned(), + }, + }, + other => wire_error(&other), + } +} + +/// A refused credential, carrying the problem body's stable code. `CodedProblem.code` is a +/// required member, so a refusal that parsed always has one. +fn refused(problem: &rest::types::CodedProblem) -> RecoveryError { + RecoveryError::Unauthorized { + code: Some(problem.code.clone()), + detail: detail(problem), + } +} + +/// The store could not answer — transient, and the caller's cadence retries. +fn unavailable(problem: &rest::types::CodedProblem) -> RecoveryError { + RecoveryError::Unavailable { + code: Some(problem.code.clone()), + detail: detail(problem), + } +} + +/// The problem body's English detail, or its code when the server sent no detail. +fn detail(problem: &rest::types::CodedProblem) -> String { + problem + .detail + .clone() + .unwrap_or_else(|| problem.code.clone()) +} + +/// Map the generated client's non-`Api` taxonomy classes: an undocumented status keeps its +/// number, and everything else is a wire failure with its source chain preserved. +fn wire_error(error: &rest::Error) -> RecoveryError +where + E: std::error::Error + 'static, +{ + match error { + rest::Error::UnexpectedStatus { status, .. } => RecoveryError::Unexpected { + status: status.as_u16(), + code: None, + }, + // `RequestConstruction` is **not** a pre-flight-only class. reqwest builds every + // failure of the request it executes with `error::request(..)`, so `is_request()` is + // true for connection-refused, DNS and TLS failures too, and the generated taxonomy + // routes all of them here alongside the genuine pre-flight ones. Splitting them by + // class alone would report an unreachable server as "sign in again", which on an + // offline device is the one remedy that cannot work. + // + // The *source* discriminates them: a bearer provider that could not produce a token is + // boxed as the generated runtime's own `AuthError`, and nothing else on this path is. + // A dead session must reach a caller as an auth failure rather than as a transport + // blip, because the two have opposite remedies. + rest::Error::RequestConstruction(inner) => { + if std::error::Error::source(inner) + .and_then(|source| source.downcast_ref::()) + .is_some() + { + RecoveryError::Unauthorized { + code: None, + detail: describe(error), + } + } else { + RecoveryError::Transport(describe(error)) + } + } + // A *declared* refusal whose body was not the coded problem the document promises. + // Deliberately a wire failure and not [`RecoveryError::NotEnrolled`], even though an + // uncoded `404` is its commonest shape: reading any `404` as "this account has + // escrowed nothing" is exactly what let a wrong route look like an empty escrow for a + // whole slice. An intermediary answering `404 text/html` is a broken path, not an + // enrollment state. + rest::Error::Decode { path, .. } => RecoveryError::Transport(format!( + "the escrow endpoint answered a refusal this client could not parse ({path})" + )), + other => RecoveryError::Transport(describe(other)), + } +} + +/// Render a generated-client failure together with its source chain. The taxonomy's own +/// `Display` names the class and, for the wire classes, little else (`"transport failed"`, +/// `"request construction failed"`) — the source chain is where the reqwest/hyper reason a log +/// reader needs actually lives. +fn describe(error: &rest::Error) -> String +where + E: std::error::Error + 'static, +{ + let mut rendered = error.to_string(); + let mut source = std::error::Error::source(error); + while let Some(cause) = source { + rendered.push_str(": "); + rendered.push_str(&cause.to_string()); + source = cause.source(); + } + rendered +} + #[cfg(test)] mod tests { use std::future::Future; @@ -392,9 +668,19 @@ mod tests { // ── Binary-capable mock escrow server ───────────────────────────────────── // - // The escrow surface is `application/octet-stream` in both directions, so unlike the - // auth mock (JSON strings) this one stores and serves raw bytes. A single shared - // slot models the server's single-active-escrow row. + // The escrow surface is `application/octet-stream` on the way out, so unlike the auth + // mock (JSON strings) this one stores and serves raw bytes. A single shared slot models + // the server's single-active-escrow row. + // + // Two things it must now do that it did not have to when this module built its own URL. + // It **routes on the path**, answering `501` to anything that is not `/v1/auth/escrow`, + // because a mock that replies to whatever it is handed is exactly why a wrong route + // survived here. And its refusals carry a real RFC 9457 problem body: the generated + // client decodes a documented non-success status into `CodedProblem`, so a bare status + // with an empty body would arrive as a decode failure rather than the typed variant. + + /// The one path the escrow operations are served on, per the committed document. + const ESCROW_ROUTE: &str = "/v1/auth/escrow"; #[derive(Clone, Default)] struct EscrowStore { @@ -403,14 +689,54 @@ mod tests { struct MockRequest { method: String, + path: String, body: Vec, } struct MockResponse { status: u16, + content_type: &'static str, body: Vec, } + impl MockResponse { + /// An `application/octet-stream` payload — the escrow itself. + fn bytes(status: u16, body: Vec) -> Self { + Self { + status, + content_type: "application/octet-stream", + body, + } + } + + /// A JSON payload — `StoreEscrowResponse`, which the generated client decodes. + fn json(status: u16, body: serde_json::Value) -> Self { + Self { + status, + content_type: "application/json", + body: body.to_string().into_bytes(), + } + } + + /// An RFC 9457 problem, shaped as `CodedProblem` so the generated client can parse it + /// into the operation's typed error. + fn problem(status: u16, code: &str, detail: &str) -> Self { + Self { + status, + content_type: "application/problem+json", + body: serde_json::json!({ + "type": "about:blank", + "title": "Refused", + "status": status, + "detail": detail, + "code": code, + }) + .to_string() + .into_bytes(), + } + } + } + type BoxFut = Pin + Send>>; type Handler = Arc BoxFut + Send + Sync>; @@ -452,11 +778,9 @@ mod tests { let head = String::from_utf8_lossy(&buf[..header_end]).to_string(); let mut lines = head.split("\r\n"); let request_line = lines.next().unwrap_or_default(); - let method = request_line - .split_whitespace() - .next() - .unwrap_or_default() - .to_string(); + let mut request_parts = request_line.split_whitespace(); + let method = request_parts.next().unwrap_or_default().to_string(); + let path = request_parts.next().unwrap_or_default().to_string(); let mut content_length = 0usize; let mut authorized = false; @@ -484,17 +808,19 @@ mod tests { // Every escrow call is owner-scoped: reject anything without a bearer token. let response = if authorized { - handler(MockRequest { method, body }).await + handler(MockRequest { method, path, body }).await } else { - MockResponse { - status: 401, - body: Vec::new(), - } + MockResponse::problem( + 401, + error_codes::REQUEST_UNAUTHENTICATED, + "no bearer credential", + ) }; let payload = format!( - "HTTP/1.1 {} STATUS\r\ncontent-type: application/octet-stream\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + "HTTP/1.1 {} STATUS\r\ncontent-type: {}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", response.status, + response.content_type, response.body.len() ); let mut out = payload.into_bytes(); @@ -505,33 +831,42 @@ mod tests { } /// A handler backed by the shared single-active-escrow slot: `PUT` overwrites it, - /// `GET` serves it verbatim (or 404). + /// `GET` serves it verbatim (or `404`). + /// + /// Anything off `/v1/auth/escrow` is `501`, which arrives as + /// [`RecoveryError::Unexpected`] rather than as a plausible-looking `NotEnrolled` — so a + /// route regression fails loudly here instead of reading as "no escrow stored". fn escrow_handler(store: EscrowStore) -> Handler { Arc::new(move |req| { let store = store.clone(); Box::pin(async move { + if req.path != ESCROW_ROUTE { + return MockResponse::bytes(501, Vec::new()); + } match req.method.as_str() { "PUT" => { - *store.blob.lock().unwrap() = Some(req.body); - MockResponse { - status: 204, - body: Vec::new(), - } + let replaced = store.blob.lock().unwrap().replace(req.body).is_some(); + MockResponse::json( + 200, + serde_json::json!({ + "stored_at": "2026-01-01T00:00:00Z", + "replaced": replaced, + }), + ) } "GET" => match store.blob.lock().unwrap().clone() { - Some(bytes) => MockResponse { - status: 200, - body: bytes, - }, - None => MockResponse { - status: 404, - body: Vec::new(), - }, - }, - _ => MockResponse { - status: 405, - body: Vec::new(), + Some(bytes) => MockResponse::bytes(200, bytes), + None => MockResponse::problem( + 404, + error_codes::ESCROW_NOT_STORED, + "no escrow has been stored for this account", + ), }, + _ => MockResponse::problem( + 405, + error_codes::REQUEST_METHOD_NOT_ALLOWED, + "the escrow surface serves GET and PUT", + ), } }) }) @@ -550,6 +885,20 @@ mod tests { .unwrap() } + /// A session over the mock whose access token expired an hour ago, so any call + /// pre-flight-refreshes — and the escrow mock serves no `/refresh`, so that refresh fails. + /// The result is a [`Session`] that cannot produce a bearer at all. + fn dead_session_for(base: &str) -> Session { + AuthClient::new(base) + .unwrap() + .resume(PersistedSession { + access_token: "test-access".to_string().into(), + refresh_token: "test-refresh".to_string().into(), + access_expires_at_unix: jiff::Timestamp::now().as_second() - 3_600, + }) + .unwrap() + } + fn wrap(master: &[u8; 32], secret: &[u8]) -> WrappedSecret { pwkdf::wrap_with(master, secret, fast_params()).unwrap() } @@ -560,7 +909,7 @@ mod tests { async fn escrow_store_fetch_round_trip() { let store = EscrowStore::default(); let base = start_mock(escrow_handler(store)).await; - let client = RecoveryClient::new(session_for(&base), &base); + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); let master = [0x11u8; 32]; let blob = wrap(&master, b"correct horse battery staple"); @@ -578,7 +927,7 @@ mod tests { #[tokio::test] async fn fetch_without_escrow_is_not_enrolled() { let base = start_mock(escrow_handler(EscrowStore::default())).await; - let client = RecoveryClient::new(session_for(&base), &base); + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); assert!(matches!( client.fetch_escrow().await, Err(RecoveryError::NotEnrolled) @@ -591,7 +940,7 @@ mod tests { async fn verify_correct_secret_against_cache() { let store = EscrowStore::default(); let base = start_mock(escrow_handler(store)).await; - let client = RecoveryClient::new(session_for(&base), &base); + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); let master = [0x22u8; 32]; let blob = wrap(&master, b"the-right-secret"); @@ -614,7 +963,7 @@ mod tests { async fn verify_refreshes_stale_cache_then_passes() { let store = EscrowStore::default(); let base = start_mock(escrow_handler(store.clone())).await; - let client = RecoveryClient::new(session_for(&base), &base); + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); let master = [0x33u8; 32]; // Enroll and cache the OLD wrap. @@ -646,7 +995,7 @@ mod tests { async fn verify_wrong_secret_fails_after_refresh() { let store = EscrowStore::default(); let base = start_mock(escrow_handler(store)).await; - let client = RecoveryClient::new(session_for(&base), &base); + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); let master = [0x44u8; 32]; let blob = wrap(&master, b"real-secret"); @@ -679,7 +1028,7 @@ mod tests { let store = EscrowStore::default(); let base = start_mock(escrow_handler(store)).await; - let client = RecoveryClient::new(session_for(&base), &base); + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); let master = [0x55u8; 32]; let old_secret = b"the-old-lost-secret"; @@ -741,7 +1090,7 @@ mod tests { #[tokio::test] async fn guided_rewrap_no_shamir_when_not_enrolled() { let base = start_mock(escrow_handler(EscrowStore::default())).await; - let client = RecoveryClient::new(session_for(&base), &base); + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); let master = [0x66u8; 32]; client.store_escrow(&wrap(&master, b"old")).await.unwrap(); @@ -752,6 +1101,202 @@ mod tests { assert!(rewrap.shamir.is_none()); } + /// The mock's route guard is live: a client pointed one segment off the documented path + /// gets a loud `Unexpected`, not a plausible-looking `NotEnrolled`. + /// + /// This is the regression this slice exists for, asserted as a property of the *test + /// harness*: without it the mock would answer any path at all, and a wrong route would + /// once again read as "this account has escrowed nothing" — which is what let + /// `backup/escrow` survive a contract re-source. + #[tokio::test] + async fn a_call_off_the_documented_route_is_not_mistaken_for_an_empty_escrow() { + let base = start_mock(escrow_handler(EscrowStore::default())).await; + let client = + RecoveryClient::new(session_for(&base), &format!("{base}/not-the-contract")).unwrap(); + let error = client + .fetch_escrow() + .await + .expect_err("a path the server does not serve is not an empty escrow"); + assert!( + matches!(error, RecoveryError::Unexpected { status: 501, .. }), + "got {error:?}" + ); + } + + /// A `400` refusal becomes the typed `Malformed` and carries **the server's own** code — + /// not the `Unexpected { status }` the hand-written path used to collapse it into, and not + /// a constant this module guessed. + #[tokio::test] + async fn a_refused_blob_is_malformed_with_the_servers_code() { + let handler: Handler = Arc::new(|_req| { + Box::pin(async move { + MockResponse::problem( + 400, + error_codes::ESCROW_MALFORMED, + "the escrow blob is empty", + ) + }) + }); + let base = start_mock(handler).await; + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); + let error = client + .store_escrow(&wrap(&[0x77u8; 32], b"whatever")) + .await + .expect_err("the server refused the blob"); + assert!( + matches!(error, RecoveryError::Malformed { .. }), + "got {error:?}" + ); + assert_eq!(error.error_code(), Some(error_codes::ESCROW_MALFORMED)); + } + + /// The store answering `500` is its own variant carrying `error.escrow.unavailable`, not a + /// bare transport failure: a client that localizes codes has one to show, and the cadence + /// can tell "the server is unwell" from "the network is gone". + #[tokio::test] + async fn an_unavailable_store_keeps_its_catalog_code() { + let handler: Handler = Arc::new(|_req| { + Box::pin(async move { + MockResponse::problem( + 500, + error_codes::ESCROW_UNAVAILABLE, + "the escrow could not be read", + ) + }) + }); + let base = start_mock(handler).await; + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); + let error = client.fetch_escrow().await.expect_err("the store is down"); + assert!( + matches!(error, RecoveryError::Unavailable { .. }), + "got {error:?}" + ); + assert_eq!(error.error_code(), Some(error_codes::ESCROW_UNAVAILABLE)); + } + + /// **An unreachable server is not an expired session.** reqwest reports a refused + /// connection as a *request* error, which the generated taxonomy files under + /// `RequestConstruction` next to the genuine pre-flight failures — so classifying that + /// whole class as an auth failure would tell an offline device to sign in again, the one + /// remedy that cannot work without a network. + #[tokio::test] + async fn an_unreachable_endpoint_is_a_transport_failure_not_an_auth_one() { + // Bind, read the port, then drop the listener: the address is now certain to refuse. + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + drop(listener); + + // The session is built against a live mock, so the session itself is healthy and the + // only thing wrong is the escrow origin. + let live = start_mock(escrow_handler(EscrowStore::default())).await; + let client = RecoveryClient::new(session_for(&live), &format!("http://{addr}")).unwrap(); + + let error = client + .fetch_escrow() + .await + .expect_err("nothing is listening there"); + assert!( + matches!(error, RecoveryError::Transport(_)), + "an unreachable endpoint must be transport, got {error:?}" + ); + assert_eq!(error.error_code(), None); + } + + /// **A session that cannot mint a bearer is an auth failure, not a network one.** + /// + /// This is the other side of `an_unreachable_endpoint_is_a_transport_failure_not_an_auth_one` + /// and it pins the discrimination that separates them. Both arrive as the generated + /// taxonomy's `RequestConstruction`; the only thing telling them apart is the boxed source, + /// which is the runtime's own `AuthError` when — and only when — the bearer provider is + /// what failed. Without this case, a spargen change to how a provider failure is boxed + /// would silently demote every expired refresh token to `Transport`, and the FFI would tell + /// a user to retry where it must tell them to sign in again. The escrow calls themselves + /// never leave the process here: there is no bearer to send them with. + #[tokio::test] + async fn a_session_that_cannot_mint_a_bearer_is_an_auth_failure() { + let base = start_mock(escrow_handler(EscrowStore::default())).await; + let client = RecoveryClient::new(dead_session_for(&base), &base).unwrap(); + + let error = client + .fetch_escrow() + .await + .expect_err("the session cannot produce a token"); + assert!( + matches!(error, RecoveryError::Unauthorized { code: None, .. }), + "a dead session must reach the caller as an auth failure, got {error:?}" + ); + assert_eq!( + error.error_code(), + None, + "no server answered, so there is no catalog code to carry" + ); + + // And the same on the write path, which has a body to construct and still fails before + // it is sent. + let error = client + .store_escrow(&wrap(&[0x99u8; 32], b"whatever")) + .await + .expect_err("the session cannot produce a token"); + assert!( + matches!(error, RecoveryError::Unauthorized { code: None, .. }), + "got {error:?}" + ); + } + + /// A refusal whose body is not the coded problem the document promises — an intermediary + /// answering a bare `404`, say — is a broken path, not an empty escrow. + /// + /// This is the deliberate class change the route fix rests on: the hand-written client + /// read *any* `404` as `NotEnrolled`, which is exactly why a wrong route looked like an + /// account that had escrowed nothing. + #[tokio::test] + async fn an_uncoded_404_is_not_read_as_an_empty_escrow() { + let handler: Handler = Arc::new(|_req| { + Box::pin(async move { MockResponse::bytes(404, b"not found".to_vec()) }) + }); + let base = start_mock(handler).await; + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); + let error = client + .fetch_escrow() + .await + .expect_err("an unparseable refusal is not an enrollment state"); + assert!( + matches!(error, RecoveryError::Transport(_)), + "got {error:?}" + ); + } + + /// A refused credential carries the problem body's code straight through, so a client + /// localizes what the *server* said rather than what this module assumed. + #[tokio::test] + async fn a_refused_credential_keeps_the_problem_code() { + // No bearer reaches the mock's handler at all: it answers `401` at the door, which is + // precisely the shape a revoked token produces. + let handler: Handler = Arc::new(|_req| { + Box::pin(async move { + MockResponse::problem( + 401, + error_codes::REQUEST_UNAUTHENTICATED, + "the access token was refused", + ) + }) + }); + let base = start_mock(handler).await; + let client = RecoveryClient::new(session_for(&base), &base).unwrap(); + let error = client + .fetch_escrow() + .await + .expect_err("the credential was refused"); + assert!( + matches!(error, RecoveryError::Unauthorized { .. }), + "got {error:?}" + ); + assert_eq!( + error.error_code(), + Some(error_codes::REQUEST_UNAUTHENTICATED) + ); + } + /// The minted secret clears the ≥128-bit entropy floor (256-bit) and never prints /// its material. #[test] diff --git a/capsule-sdk/src/upgrade.rs b/capsule-sdk/src/upgrade.rs new file mode 100644 index 00000000..f779a6eb --- /dev/null +++ b/capsule-sdk/src/upgrade.rs @@ -0,0 +1,596 @@ +//! Proposing an **album upgrade** — the client half of the ceremony's one server-side step +//! (slice `S-C24`, over the `S-D28` wire). +//! +//! [Versioning — Album Upgrade Ceremony] is a client ceremony carried on MLS application +//! messages the server cannot read. Four of its steps are the server's, and the first is the +//! one this module drives: `POST /v1/albums/{album_id}/upgrade` hands the server a **signed +//! `UpgradeIntent`**, which quiesces the album (a v_old client that never saw the proposal is +//! precisely the party that will not stop writing on its own) and starts the deadline on the +//! server's own clock (so a skewed member clock can neither extend nor shorten the window). +//! +//! Two rules shape this module, and both are the directory client's rules for the same reason: +//! +//! - **The signed bytes travel verbatim.** `intent_cbor` is the canonical CBOR +//! `capsule_core::crypto::upgrade::SignedUpgradeIntent` produced, and it is written to the +//! body unchanged. Re-encoding it here would detach it from the signature the server checks +//! against the proposing device's DSK in the account's published directory, and the failure +//! would look like a forged proposal. +//! - **Nothing cryptographic happens here.** The intent is built and signed in `capsule-core`; +//! this module is the wire and its refusals. +//! +//! # Why hand-written, and what would retire it +//! +//! The request body is `application/cbor`, and `spargen` 0.4's `classify_media` does not know +//! that media type, so `capsule-sdk/build.rs` narrows the operation out of the generated client +//! (`S-D28`) — the *surface* is narrowed, the document is never mutilated. This module is +//! therefore the orchestration `AGENTS.md` permits over a wire it cannot generate, and it is +//! the fourth and last such client: `capsule_sdk::directory` hand-writes two and +//! [`crate::verify::StorageVerifyClient::fetch_receipt`] the third. Teaching spargen the media +//! type retires all four; nothing in this repository can. +//! +//! The **other two** operations on this path — `GET` (read the phase) and `DELETE` (end the +//! ceremony) — are plain JSON and *are* generated. Call them through +//! [`crate::client::AuthenticatedClient`]; this module deliberately does not duplicate them. +//! +//! [Versioning — Album Upgrade Ceremony]: https://docs/design/versioning/#album-upgrade-ceremony + +use jiff::Timestamp; +use serde::Deserialize; +use tracing::instrument; +use uuid::Uuid; + +use crate::auth::{AuthError, Session}; + +/// The media type the intent is *signed* in, and therefore the only one it may be sent as. +const CBOR: &str = "application/cbor"; + +/// The phase response's media type — the answer is a plain JSON document. +const JSON: &str = "application/json"; + +// ─── Errors ─────────────────────────────────────────────────────────────────── + +/// Everything a proposal can fail with. Callers switch on the typed variant, or on its stable +/// `error.*` code, and never on a bare status. +/// +/// Every refusal carries the code the **server** stamped rather than one this module inferred +/// from the status, because the ceremony's refusals are the ones a user actually reads: "the +/// album is already upgrading" and "your device is not an admin" are different sentences. +#[derive(Debug, thiserror::Error)] +pub enum UpgradeError { + /// The authenticated request itself failed (transport, session expiry, refresh). + #[error(transparent)] + Auth(#[from] AuthError), + /// The server refused the body as one it cannot read as a signed intent (`400`), refused + /// the media type (`415`), or refused its size (`413`). Retrying the same bytes changes + /// nothing — rebuild and re-sign the intent. + #[error("the server rejected the upgrade intent: {detail}")] + Malformed { + /// The stable `error.*` catalog code the refusal carried. + code: Option, + /// English detail from the problem body. + detail: String, + }, + /// The credential was refused (`401`). + #[error("the upgrade surface refused the credential: {detail}")] + Unauthorized { + /// The stable `error.*` catalog code the refusal carried. + code: Option, + /// English detail from the problem body. + detail: String, + }, + /// The intent is not signed by a device in the caller's **published** directory (`403`), + /// so the server cannot tell that an admin device really asked for this. Publish the + /// directory holding the proposing device first ([`crate::directory`]). + #[error("the upgrade intent's proposer could not be verified: {detail}")] + NotProposer { + /// The stable `error.*` catalog code the refusal carried. + code: Option, + /// English detail from the problem body. + detail: String, + }, + /// No such album, or not this caller's (`404`) — one answer for both, deliberately, so the + /// surface discloses nothing about albums the caller does not own. + #[error("no such album: {detail}")] + NotFound { + /// The stable `error.*` catalog code the refusal carried. + code: Option, + /// English detail from the problem body. + detail: String, + }, + /// A different ceremony already holds this album (`409`), and only one may. Read the phase + /// (the generated `GET` on the same path) and either join that ceremony or wait for its + /// deadline; a fresh proposal *replaces* an expired one rather than conflicting with it. + #[error("album is already upgrading under {intent_id:?}: {detail}")] + InFlight { + /// The ceremony that holds the album, as the server reported it. + intent_id: Option, + /// The stable `error.*` catalog code the refusal carried. + code: Option, + /// English detail from the problem body. + detail: String, + }, + /// A collaborator could not answer (`500`). Transient. + #[error("the upgrade could not be recorded: {detail}")] + Unavailable { + /// The stable `error.*` catalog code the refusal carried. + code: Option, + /// English detail from the problem body. + detail: String, + }, + /// The response body was not the phase document the contract declares. + #[error("malformed upgrade phase response: {0}")] + MalformedResponse(String), + /// The server returned an unmodeled status. + #[error("unexpected {status} response from the album-upgrade endpoint")] + Unexpected { + /// The HTTP status code the server returned. + status: u16, + }, +} + +impl UpgradeError { + /// The stable `error.*` catalog code a client localizes, when one applies. The English + /// [`Display`](std::fmt::Display) form stays the developer/log detail. + #[must_use] + pub fn error_code(&self) -> Option<&str> { + match self { + Self::Auth(auth) => auth.error_code(), + Self::Malformed { code, .. } + | Self::Unauthorized { code, .. } + | Self::NotProposer { code, .. } + | Self::NotFound { code, .. } + | Self::InFlight { code, .. } + | Self::Unavailable { code, .. } => code.as_deref(), + _ => None, + } + } +} + +// ─── Wire DTOs (mirror the server's transport JSON) ─────────────────────────── + +/// `UpgradePhaseResponse`, as the server serializes it. The three optional members are absent +/// when no ceremony is in flight — which also covers *expired*, because the deadline passing +/// aborts the upgrade and leaves nothing to be in. +#[derive(Debug, Deserialize)] +struct UpgradePhaseWire { + album_id: String, + #[serde(default)] + intent_id: Option, + #[serde(default)] + to_protocol_version: Option, + #[serde(default)] + expires_at: Option, + in_flight: u64, +} + +/// The members this module reads off an RFC 9457 problem body. `intent_id` is the `409`'s +/// extension; the rest are the coded-problem shape every Capsule refusal renders. +#[derive(Debug, Default, Deserialize)] +struct ProblemWire { + #[serde(default)] + code: Option, + #[serde(default)] + detail: Option, + #[serde(default)] + intent_id: Option, +} + +/// The ceremony an album is in, as the proposal answered it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpgradePhase { + /// The album, echoed by the server. + pub album_id: Uuid, + /// The ceremony now in flight, or `None` when the album is in normal operation. + pub intent_id: Option, + /// The protocol version the fork will be pinned to, when a ceremony is in flight. + pub to_protocol_version: Option, + /// When the window closes, on the **server's** clock — never the client's. + pub expires_at: Option, + /// How many upload sessions are still in flight against this album. + /// + /// The drain signal of the ceremony's step 3: the proposer waits for zero. A count and not + /// a listing, because the proposer needs to know *whether* to wait and has no business + /// seeing other members' upload identifiers to find out. + pub in_flight: u64, +} + +// ─── Client ─────────────────────────────────────────────────────────────────── + +/// The album-upgrade proposal client. Borrows an authenticated [`Session`], so every call +/// rides the SDK's bearer/refresh machinery and no token is handled here. +#[derive(Clone)] +pub struct UpgradeClient { + session: Session, + base_url: String, +} + +impl UpgradeClient { + /// Build a client against the **API root** — the origin the operation paths hang off (e.g. + /// `https://api.example.com`), the same base [`crate::client::AuthenticatedClient`], + /// [`crate::sync::SyncConsumer`] and [`crate::recovery::RecoveryClient`] take. + /// + /// Note that [`crate::directory`] and [`crate::verify`] take a *deeper* base instead. That + /// divergence is real and is noticed in `capsule-server/tests/sdk_client.rs`; it is not + /// this module's to close, and a new module choosing the root is how it narrows. + #[must_use] + pub fn new(session: Session, api_base_url: &str) -> Self { + Self { + session, + base_url: api_base_url.trim_end_matches('/').to_owned(), + } + } + + /// Propose the upgrade `intent_cbor` describes for `album_id`, returning the ceremony the + /// server now holds. + /// + /// `intent_cbor` is the canonical CBOR of a signed `UpgradeIntent` and is sent **verbatim** + /// — this method never re-encodes it, because the signature is over exactly those bytes. + /// + /// # Errors + /// + /// [`UpgradeError::InFlight`] when another ceremony already holds the album (only one may), + /// [`UpgradeError::NotProposer`] when the signing device is not in the published directory, + /// and the rest of [`UpgradeError`] for the remaining refusals. + #[instrument(skip(self, intent_cbor), fields(album_id = %album_id, bytes = intent_cbor.len()))] + pub async fn begin( + &self, + album_id: Uuid, + intent_cbor: &[u8], + ) -> Result { + let url = format!( + "{}/v1/albums/{}/upgrade", + self.base_url, + album_id.hyphenated() + ); + let body = intent_cbor.to_vec(); + let response = self + .session + .execute(|http| { + http.post(&url) + .header(reqwest::header::CONTENT_TYPE, CBOR) + .header(reqwest::header::ACCEPT, JSON) + .body(body.clone()) + }) + .await?; + + let status = response.status(); + if !status.is_success() { + let problem = response.json::().await.unwrap_or_default(); + let error = refusal(status.as_u16(), problem); + tracing::warn!( + status = status.as_u16(), + code = ?error.error_code(), + "album-upgrade proposal refused" + ); + return Err(error); + } + + let wire: UpgradePhaseWire = response + .json() + .await + .map_err(|e| UpgradeError::MalformedResponse(e.to_string()))?; + let phase = decode_phase(wire)?; + tracing::info!( + intent_id = ?phase.intent_id, + in_flight = phase.in_flight, + expires_at = ?phase.expires_at, + "album upgrade proposed; the album is quiesced" + ); + Ok(phase) + } +} + +/// Map a refusal onto its typed variant, keeping the code the server stamped. +/// +/// One readable status table rather than a match buried in the request path. +/// +/// `413` is the transport's body backstop and carries no problem body at all, so it has no +/// code — and this client does not mint one. Every code here is the code the *server* stamped; +/// a code invented on this side would assert that the server said something it did not, and a +/// client localizing it would read the SDK's guess as the server's judgement. The variant +/// already carries the actionable half ("these bytes will not do"), and the English detail +/// carries the reason. +fn refusal(status: u16, problem: ProblemWire) -> UpgradeError { + let ProblemWire { + code, + detail, + intent_id, + } = problem; + let detail = detail.unwrap_or_default(); + match status { + 400 | 415 => UpgradeError::Malformed { code, detail }, + 401 => UpgradeError::Unauthorized { code, detail }, + 403 => UpgradeError::NotProposer { code, detail }, + 404 => UpgradeError::NotFound { code, detail }, + 409 => UpgradeError::InFlight { + intent_id, + code, + detail, + }, + 413 => UpgradeError::Malformed { + code: None, + detail: "the signed upgrade intent exceeds the server's body limit".to_owned(), + }, + 500 => UpgradeError::Unavailable { code, detail }, + other => UpgradeError::Unexpected { status: other }, + } +} + +/// Parse the phase document into its typed shape. +/// +/// The ids become [`Uuid`]s and the deadline a [`jiff::Timestamp`], so a caller compares +/// instants rather than strings — the deadline is the one field in this ceremony where a +/// string comparison would be a correctness bug rather than an inconvenience. +fn decode_phase(wire: UpgradePhaseWire) -> Result { + let album_id = Uuid::parse_str(&wire.album_id) + .map_err(|e| UpgradeError::MalformedResponse(format!("response album_id: {e}")))?; + let intent_id = wire + .intent_id + .as_deref() + .map(Uuid::parse_str) + .transpose() + .map_err(|e| UpgradeError::MalformedResponse(format!("response intent_id: {e}")))?; + let expires_at = wire + .expires_at + .as_deref() + .map(str::parse::) + .transpose() + .map_err(|e| UpgradeError::MalformedResponse(format!("response expires_at: {e}")))?; + Ok(UpgradePhase { + album_id, + intent_id, + to_protocol_version: wire.to_protocol_version, + expires_at, + in_flight: wire.in_flight, + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use capsule_i18n::error_codes; + + use super::*; + use crate::auth::{AuthClient, PersistedSession}; + use crate::testmock::{MockRequest, MockResponse, MockServer}; + + const ALBUM: &str = "018f3f1e-4b7a-7c9d-8e2f-1a2b3c4d5e60"; + const INTENT: &str = "019a0000-0000-7000-8000-00000000cafe"; + + /// The bytes a real client would hand over: opaque, and deliberately not valid UTF-8 so a + /// re-encoding anywhere on the path would show up. + fn signed_intent() -> Vec { + let mut bytes = b"signed-upgrade-intent".to_vec(); + bytes.extend_from_slice(&[0x00, 0xff, 0xa5]); + bytes + } + + fn album() -> Uuid { + Uuid::parse_str(ALBUM).expect("the literal is a uuid") + } + + /// A session over `base` with a far-future token, so no refresh ever fires and the mock + /// needs no `/refresh` endpoint. + fn session_for(base: &str) -> Session { + AuthClient::new(base) + .expect("a base url") + .resume(PersistedSession { + access_token: "test-access".to_string().into(), + refresh_token: "test-refresh".to_string().into(), + access_expires_at_unix: jiff::Timestamp::now().as_second() + 3_600, + }) + .expect("a session resumes from any pair") + } + + fn phase_json() -> String { + serde_json::json!({ + "album_id": ALBUM, + "intent_id": INTENT, + "to_protocol_version": "2030-01-01", + "expires_at": "2030-01-01T00:05:00Z", + "in_flight": 0, + }) + .to_string() + } + + /// An RFC 9457 problem, as `capsule-server`'s interceptor renders one. + fn problem(status: u16, reason: &str, code: &str, detail: &str) -> MockResponse { + MockResponse::new(status, reason).json_body( + serde_json::json!({ + "type": "about:blank", + "title": reason, + "status": status, + "detail": detail, + "code": code, + }) + .to_string(), + ) + } + + /// The signed bytes reach the documented path, in the documented media type, unchanged — + /// and the phase decodes into typed ids and a real instant. + #[tokio::test] + async fn a_proposal_sends_the_signed_bytes_verbatim_and_decodes_the_phase() { + let intent = signed_intent(); + let expected = intent.clone(); + let seen = Arc::new(AtomicUsize::new(0)); + let counter = seen.clone(); + let server = MockServer::start(move |req: &MockRequest| { + counter.fetch_add(1, Ordering::SeqCst); + assert_eq!(req.method, "POST"); + assert_eq!(req.path, format!("/v1/albums/{ALBUM}/upgrade")); + assert_eq!( + req.header("content-type"), + Some(CBOR), + "the intent must be sent in the media type it was signed in" + ); + assert_eq!( + req.body, expected, + "a re-encoded intent no longer verifies under the proposer's DSK" + ); + assert!( + req.header("authorization") + .is_some_and(|v| v.starts_with("Bearer ")), + "the proposal is owner-scoped" + ); + MockResponse::new(200, "OK").json_body(phase_json()) + }) + .await; + + let client = UpgradeClient::new(session_for(&server.base_url()), &server.base_url()); + let phase = client + .begin(album(), &intent) + .await + .expect("the proposal is accepted"); + + assert_eq!(seen.load(Ordering::SeqCst), 1, "exactly one request"); + assert_eq!(phase.album_id, album()); + assert_eq!( + phase.intent_id, + Some(Uuid::parse_str(INTENT).expect("a uuid")) + ); + assert_eq!(phase.to_protocol_version.as_deref(), Some("2030-01-01")); + assert_eq!( + phase.expires_at, + Some("2030-01-01T00:05:00Z".parse().expect("an instant")) + ); + assert_eq!(phase.in_flight, 0); + } + + /// A second ceremony is refused with the live `intent_id` and the code a client localizes + /// — the refusal an admin actually reads, so neither may be flattened into a status. + #[tokio::test] + async fn a_second_proposal_is_refused_with_the_live_ceremony() { + let server = MockServer::start(move |_req: &MockRequest| { + MockResponse::new(409, "Conflict").json_body( + serde_json::json!({ + "type": "about:blank", + "title": "Upgrade in flight", + "status": 409, + "detail": format!("album is already upgrading under {INTENT}"), + "code": error_codes::ALBUM_UPGRADE_IN_FLIGHT, + "intent_id": INTENT, + }) + .to_string(), + ) + }) + .await; + + let client = UpgradeClient::new(session_for(&server.base_url()), &server.base_url()); + let error = client + .begin(album(), &signed_intent()) + .await + .expect_err("only one ceremony may hold an album"); + let UpgradeError::InFlight { intent_id, .. } = &error else { + panic!("expected an in-flight refusal, got {error:?}"); + }; + assert_eq!(intent_id.as_deref(), Some(INTENT)); + assert_eq!( + error.error_code(), + Some(error_codes::ALBUM_UPGRADE_IN_FLIGHT) + ); + } + + /// Every refusal the operation declares maps to its own variant and keeps the server's + /// code. A status collapsed into the wrong variant would tell an admin to fix the wrong + /// thing — re-sign an intent that was fine, or wait out a ceremony that does not exist. + #[tokio::test] + async fn each_declared_refusal_keeps_its_own_identity() { + for (status, reason, code) in [ + (400u16, "Bad Request", error_codes::ALBUM_UPGRADE_MALFORMED), + (403, "Forbidden", error_codes::ALBUM_UPGRADE_PROPOSER), + (404, "Not Found", error_codes::ALBUM_UPGRADE_NOT_FOUND), + (500, "Internal Server Error", error_codes::ALBUM_UNAVAILABLE), + ] { + let server = + MockServer::start(move |_req: &MockRequest| problem(status, reason, code, "no")) + .await; + let client = UpgradeClient::new(session_for(&server.base_url()), &server.base_url()); + let error = client + .begin(album(), &signed_intent()) + .await + .expect_err("the server refused"); + assert_eq!(error.error_code(), Some(code), "status {status}: {error:?}"); + let matched = match status { + 400 => matches!(error, UpgradeError::Malformed { .. }), + 403 => matches!(error, UpgradeError::NotProposer { .. }), + 404 => matches!(error, UpgradeError::NotFound { .. }), + 500 => matches!(error, UpgradeError::Unavailable { .. }), + _ => false, + }; + assert!(matched, "status {status} took the wrong variant: {error:?}"); + } + } + + /// The body-size backstop carries no problem body, so the client supplies the variant — + /// and **no code**. Minting one here would put words in the server's mouth, and every other + /// code this module reports is the server's own. + #[tokio::test] + async fn a_body_too_large_carries_no_invented_code() { + let server = MockServer::start(move |_req: &MockRequest| { + MockResponse::new(413, "Payload Too Large") + }) + .await; + let client = UpgradeClient::new(session_for(&server.base_url()), &server.base_url()); + let error = client + .begin(album(), &signed_intent()) + .await + .expect_err("the server refused the size"); + assert!( + matches!(error, UpgradeError::Malformed { code: None, .. }), + "got {error:?}" + ); + assert_eq!( + error.error_code(), + None, + "a code the server never sent is not this client's to supply" + ); + } + + /// An undeclared status is surfaced as itself rather than guessed at. + #[tokio::test] + async fn an_undeclared_status_is_surfaced_as_unexpected() { + let server = + MockServer::start(move |_req: &MockRequest| MockResponse::new(418, "I'm a teapot")) + .await; + let client = UpgradeClient::new(session_for(&server.base_url()), &server.base_url()); + let error = client + .begin(album(), &signed_intent()) + .await + .expect_err("418 is not in the contract"); + assert!( + matches!(error, UpgradeError::Unexpected { status: 418 }), + "got {error:?}" + ); + assert_eq!(error.error_code(), None); + } + + /// A phase document whose deadline is not an instant is a malformed *response*, not a + /// silent `None` — a dropped deadline would make a client think the ceremony never expires. + #[tokio::test] + async fn an_unparseable_deadline_is_a_malformed_response() { + let server = MockServer::start(move |_req: &MockRequest| { + MockResponse::new(200, "OK").json_body( + serde_json::json!({ + "album_id": ALBUM, + "intent_id": INTENT, + "expires_at": "next tuesday", + "in_flight": 0, + }) + .to_string(), + ) + }) + .await; + let client = UpgradeClient::new(session_for(&server.base_url()), &server.base_url()); + let error = client + .begin(album(), &signed_intent()) + .await + .expect_err("a deadline that is not an instant is not a deadline"); + assert!( + matches!(error, UpgradeError::MalformedResponse(_)), + "got {error:?}" + ); + } +} diff --git a/capsule-server/src/upload/policy.rs b/capsule-server/src/upload/policy.rs index 4ade86de..e5e5f7e1 100644 --- a/capsule-server/src/upload/policy.rs +++ b/capsule-server/src/upload/policy.rs @@ -23,11 +23,44 @@ use jiff::Timestamp; +/// The still-derivative content types the thumbnail ladder uploads. +/// +/// **Source of truth: +/// [Thumbnails and Previews](../../../capsule-docs/src/content/docs/design/thumbnails.md).** Its +/// tier table is the closed set: a JXL master with AVIF and WebP as the delivery variants. The +/// doc is explicit that "every receiver (and every federated peer) compares the +/// `DerivativeManifest.format` value against this list, and an unknown value is a structural +/// rejection" — and this server is such a receiver, so its accept-list has to carry every value +/// that list admits *and no more*. +/// +/// The `original` sentinel is deliberately **absent**. It is a recognised `format` value, not a +/// content type: a tier that references the original carries no bytes of its own, so it opens no +/// upload session and never presents a `content_type` at all. Admitting it here would widen the +/// closed enum for a blob that cannot exist. +/// +/// # Why this is a list here and not the enum +/// +/// It should be `capsule_core::derivative_format::DerivativeFormat::STILL_DELIVERY_ORDER` mapped +/// through `mime()` — one set, evaluated by producer and receiver alike. That module is not +/// reachable from this crate yet: it exists only on the branch of #436, is on neither `master` +/// nor this branch's base, and `capsule-core` is not this change's to edit. So the set is +/// restated once, in one place, with the swap named — and the test below fails the moment this +/// list and the accept-list disagree, which is the failure that produced #470. +pub const DERIVATIVE_CONTENT_TYPES: &[&str] = &["image/jxl", "image/avif", "image/webp"]; + /// The closed `content_type` enum for the current protocol version (invariant 5). /// /// Frozen for a given `protocol_version` and server-tunable across versions. Metadata, /// provenance and backup blobs are opaque CBOR or ciphertext and declare /// `application/octet-stream`. +/// +/// It carries two disjoint things: the **originals** a client imports, and the **derivatives** +/// its ladder generates ([`DERIVATIVE_CONTENT_TYPES`], plus `video/mp4` for the H.264 baseline +/// video preview, which the stills-only derivative set does not model). `image/jxl` is here for +/// the second reason only — nothing imports a JXL original today — and its absence is #470: +/// every still larger than the 256 px thumbnail cap failed its T1 upload with +/// `400 error.upload.unsupported_content_type`, because the ladder encodes that tier as JXL and +/// the server had never been told the format existed. pub const DEFAULT_CONTENT_TYPES: &[&str] = &[ "image/jpeg", "image/png", @@ -35,6 +68,7 @@ pub const DEFAULT_CONTENT_TYPES: &[&str] = &[ "image/heif", "image/webp", "image/avif", + "image/jxl", "image/gif", "image/tiff", "video/mp4", @@ -214,6 +248,37 @@ mod tests { ); } + #[test] + fn every_committed_derivative_format_is_accepted() { + // #470: the ladder encodes the thumbnail tier as JXL, the SDK uploads each derivative + // with `content_type = derivative.format`, and the server answered + // `400 error.upload.unsupported_content_type` — so every still larger than the 256 px + // cap failed. The defect was not the missing string, it was that nothing tied the + // accept-list to the closed set it is supposed to mirror. This is that tie. + let policy = UploadPolicy::default(); + let accepted = policy.content_types(); + for format in DERIVATIVE_CONTENT_TYPES { + assert!( + accepted.contains(format), + "{format} is a committed derivative format the ladder uploads, and the closed \ + content-type enum refuses it" + ); + } + } + + #[test] + fn the_original_sentinel_is_not_a_content_type() { + // It is a recognised `DerivativeManifest.format` value and nothing more: a tier that + // references the original carries no bytes, opens no upload session, and presents no + // `content_type`. Admitting it would widen the closed enum for a blob that cannot exist. + assert!(!DERIVATIVE_CONTENT_TYPES.contains(&"original")); + assert!( + !UploadPolicy::default() + .content_types() + .contains(&"original") + ); + } + #[test] fn a_deployment_can_narrow_every_tunable() { let policy = UploadPolicy::default() diff --git a/capsule-server/tests/sdk_client.rs b/capsule-server/tests/sdk_client.rs index 8984f968..55d80600 100644 --- a/capsule-server/tests/sdk_client.rs +++ b/capsule-server/tests/sdk_client.rs @@ -30,9 +30,12 @@ mod support; use capsule_sdk::auth::AuthClient; use capsule_sdk::sync::{ChangeKind, SyncConsumer, SyncCursor, SyncError, SyncState}; +use capsule_server::App; +use capsule_server::app::Modules; use capsule_server::blob::{BlobStore, ContentAddress}; use capsule_server::index::{AssetIndex, BlobRecord, PendingAsset}; use capsule_server::store::{AssetId, BlobRole, Clock}; +use capsule_server::upload::{UploadContext, UploadPolicy}; use jiff::Timestamp; use support::{EMAIL, Fixture, PASSWORD, PROTOCOL_VERSION, album, owner}; @@ -44,7 +47,12 @@ const CLIENT_MAX_PROTOCOL: &str = "2099-12-31"; /// The listener serves the **same** context the fixture holds handles on, so an asset seeded /// through `fixture.index` is an asset this server serves. async fn serve(fixture: &Fixture) -> String { - let service = capsule_server::service(fixture.app()).expect("the router builds"); + serve_app(fixture.app()).await +} + +/// Bind `app` to an ephemeral port and return its base URL. +async fn serve_app(app: App) -> String { + let service = capsule_server::service(app).expect("the router builds"); let bound = kynos::server::Server::new(service) .bind(("127.0.0.1", 0)) .prepare() @@ -287,3 +295,386 @@ async fn the_sdk_completes_a_real_second_factor_over_a_socket() { .expect("the code completes the sign-in"); assert!(session.is_authenticated().await); } + +/// The escrow round trip, over a socket, against the router that actually serves it. +/// +/// This is the case the slice was missing. `capsule_sdk::recovery` used to build +/// `{api_root}/backup/escrow` by hand — the Salvo document's path — and its own in-module mock +/// answered whatever path it was handed, so every escrow test passed while no real server had +/// that route. Only a client pointed at the router can tell the difference, and the bytes are +/// the ones a KDF runs against: a wrap that comes back re-encoded is a lost master key. +/// +/// **The route is asserted, not inferred.** Both halves are cross-checked against +/// `/v1/auth/escrow` through the fixture's own in-process client: what the SDK stored is read +/// back at that path, and what was seeded at that path is what the SDK fetches. A client +/// talking to some other path could satisfy neither, so this does not depend on how the +/// router happens to render a `404` for a path it does not serve. +#[tokio::test] +async fn the_sdk_stores_and_fetches_an_escrow_over_a_socket() { + use capsule_core::crypto::primitives::Argon2Params; + use capsule_core::crypto::pwkdf; + use capsule_sdk::recovery::{RecoveryClient, RecoveryError}; + + // Fast Argon2id params: the crypto is `capsule-core`'s and proven there; what is under test + // is the wire. + let params = Argon2Params { + mem_kib: 64, + t_cost: 1, + p_cost: 1, + }; + const SECRET: &[u8] = b"correct horse battery staple"; + + let fixture = Fixture::working(); + let bearer = fixture.bearer().await; + let base_url = serve(&fixture).await; + let client = + RecoveryClient::new(session(&base_url).await, &base_url).expect("an API root parses"); + + // Nothing stored yet: the typed refusal a cadence reads as "enroll first". + let missing = client + .fetch_escrow() + .await + .expect_err("a fresh account has escrowed nothing"); + assert!( + matches!(missing, RecoveryError::NotEnrolled), + "got {missing:?}" + ); + + // ── The SDK writes; the contract's route is where it landed ─────────────────────────── + let master = [0x5Au8; 32]; + let blob = pwkdf::wrap_with(&master, SECRET, params).expect("the master key wraps"); + client.store_escrow(&blob).await.expect("the escrow stores"); + + let response = fixture + .client + .get("/v1/auth/escrow") + .header("authorization", &bearer) + .send() + .await; + let seen = response.assert_status(kynos::http::StatusCode::OK).bytes(); + assert_eq!( + seen.as_ref(), + capsule_core::cbor::to_canonical_vec(&blob) + .expect("the wrap encodes") + .as_slice(), + "the bytes the SDK stored must be readable at `/v1/auth/escrow` — the path the \ + committed document declares, which is the assertion the old tests could not make" + ); + + // ── The SDK reads back what that route holds, byte for byte ─────────────────────────── + let cache = client.fetch_escrow().await.expect("and comes back"); + assert_eq!( + cache.blob(), + &blob, + "the escrow is ciphertext served verbatim; a re-encoded wrap no longer opens" + ); + assert_eq!( + capsule_core::backup::recover_master_key(cache.blob(), SECRET) + .expect("the fetched wrap opens"), + master, + ); + + // A rotation seeded at the contract's route is the one the SDK sees next — the read half + // pinned to the same path, without relying on a refusal to prove it. + let rotated = pwkdf::wrap_with(&master, b"a different secret entirely", params) + .expect("the master key re-wraps"); + fixture + .client + .put("/v1/auth/escrow") + .header("authorization", &bearer) + .header("accept", "application/json") + .body( + "application/octet-stream", + capsule_core::cbor::to_canonical_vec(&rotated).expect("the wrap encodes"), + ) + .send() + .await + .assert_status(kynos::http::StatusCode::OK); + assert_eq!( + client + .fetch_escrow() + .await + .expect("the rotated escrow comes back") + .blob(), + &rotated, + "the SDK reads the resource `/v1/auth/escrow` addresses, not some other path that \ + happens to answer" + ); +} + +/// **`S-D17`'s Done-when, against the server that decides.** A token the client still believes +/// in and the server has stopped honouring is refreshed once and the call replayed once. +/// +/// The unit tests cover the layer against a mock; this covers the one thing a mock cannot rule +/// out — that the two ends disagree about when an access token dies. The server validates `exp` +/// against its **injected** clock (`capsule_server::auth::tokens`, deliberately, so a test can +/// walk over an expiry), so advancing the fixture's clock past `ACCESS_TOKEN_TTL` revokes the +/// access token for real while the session and its refresh token remain live. +/// +/// The client is then handed the same token pair with a far-future expiry, which is exactly the +/// state a client is in whenever it trusted a server-supplied deadline and the server changed +/// its mind first — a revocation, a clock skew, a rotated signing key. Because the client sees +/// no reason to refresh, the pre-flight half cannot fire, so a call that succeeds here succeeded +/// through the reactive layer and nothing else. +#[tokio::test] +async fn a_token_the_server_stopped_honouring_is_refreshed_and_the_call_replayed() { + use capsule_sdk::auth::PersistedSession; + use capsule_sdk::client::AuthenticatedClient; + use secrecy::ExposeSecret as _; + + let fixture = Fixture::working(); + let base_url = serve(&fixture).await; + let signed_in = session(&base_url).await; + let pair = signed_in.export().await.expect("a live session exports"); + let stale_access = pair.access_token.expose_secret().to_owned(); + + // Past the access token's life, well inside the session's. The refresh token still works; + // the access token does not. + fixture.clock.advance( + capsule_server::auth::ACCESS_TOKEN_TTL + .checked_add(jiff::SignedDuration::from_secs(60)) + .expect("a representable instant"), + ); + + // The same pair, with a deadline the client has no reason to doubt. + let session = AuthClient::new(&format!("{base_url}/v1/auth")) + .expect("a base url") + .resume(PersistedSession { + access_token: stale_access.clone().into(), + refresh_token: pair.refresh_token, + access_expires_at_unix: Timestamp::now().as_second() + 3600, + }) + .expect("a session resumes from any pair"); + let client = AuthenticatedClient::new(&base_url, session).expect("an API root parses"); + + // A generated operation, called straight through the Deref — nothing about this call site + // knows a retry layer exists, which is the point of putting it at the transport seam. + let quota = client + .get_quota(capsule_core::crypto::primitives::PROTOCOL_VERSION, None) + .await + .expect("the 401 is recovered and the call replayed") + .into_inner(); + assert_eq!(quota.state.as_str(), "ok"); + + let after = client + .session() + .export() + .await + .expect("the session is still live"); + assert_ne!( + after.access_token.expose_secret(), + stale_access.as_str(), + "the replay must have ridden a rotated token; an unchanged one would mean the server \ + accepted a token it had already stopped honouring" + ); +} + +/// The album-upgrade proposal, over a socket, against the server that verifies the signature. +/// +/// This one cannot be proven against a mock at all. The intent is signed with the proposing +/// device's DSK and verified against the account's **published** device directory, so a mock +/// that answered `200` would prove only that the client can post bytes. Here the directory is +/// anchored, the album provisioned, and the intent signed by `capsule-core` with the *same* +/// types the server verifies with — so what is asserted is that the bytes the SDK put on the +/// wire are the bytes that verify. +/// +/// The `409` half matters as much: only one ceremony may hold an album, and a client that read +/// that refusal as "malformed, re-sign" would have an admin re-signing intents forever. +#[tokio::test] +async fn the_sdk_proposes_an_album_upgrade_over_a_socket() { + use capsule_sdk::upgrade::{UpgradeClient, UpgradeError}; + use support::{ + device, identity_header, identity_key, signed_directory_with_device, signed_upgrade_intent, + }; + use uuid::Uuid; + + let intent_id = Uuid::parse_str("019a0000-0000-7000-8000-00000000cafe").expect("a uuid"); + let fixture = Fixture::working(); + let bearer = fixture.bearer().await; + let identity = identity_key(); + let device_key = identity_key(); + + // Anchor the directory holding the proposing device, then provision the album. Without the + // first, every proposal is `403` however well signed — the directory *is* the trust anchor. + fixture + .client + .post("/v1/auth/devices/directory") + .header("authorization", &bearer) + .header("x-capsule-identity-key", &identity_header(&identity)) + .body( + "application/cbor", + signed_directory_with_device( + &identity, + 1, + device(), + &device_key, + "1970-01-01T00:00:00Z", + ), + ) + .send() + .await + .assert_status(kynos::http::StatusCode::OK); + fixture + .client + .post("/v1/albums") + .header("authorization", &bearer) + .header("accept", "application/json") + .json(&serde_json::json!({ "album_id": album().as_str() })) + .send() + .await + .assert_status(kynos::http::StatusCode::CREATED); + + let base_url = serve(&fixture).await; + let client = UpgradeClient::new(session(&base_url).await, &base_url); + let album_id = Uuid::parse_str(album().as_str()).expect("the seeded album id is a uuid"); + let intent = signed_upgrade_intent(&device_key, device(), intent_id, "2030-01-01", 300); + + let phase = client + .begin(album_id, &intent) + .await + .expect("a signed proposal from an anchored device is accepted"); + assert_eq!(phase.album_id, album_id); + assert_eq!( + phase.intent_id, + Some(intent_id), + "the ceremony the server now holds is the one the client proposed" + ); + assert_eq!(phase.to_protocol_version.as_deref(), Some("2030-01-01")); + assert_eq!(phase.in_flight, 0, "nothing is draining on a fresh album"); + assert!( + phase.expires_at.is_some(), + "the deadline is the server's to set, and it must reach the client as an instant" + ); + + // A second ceremony under a different id is refused with the live one — and with the code + // a client localizes, parsed out of a problem body that crossed a socket. + let second = Uuid::parse_str("019a0000-0000-7000-8000-00000000beef").expect("a uuid"); + let error = client + .begin( + album_id, + &signed_upgrade_intent(&device_key, device(), second, "2030-01-01", 300), + ) + .await + .expect_err("only one ceremony may hold an album"); + let UpgradeError::InFlight { + intent_id: live, .. + } = &error + else { + panic!("expected an in-flight refusal, got {error:?}"); + }; + assert_eq!(live.as_deref(), Some(intent_id.to_string().as_str())); + assert_eq!( + error.error_code(), + Some("error.album.upgrade_in_flight"), + "got {error:?}" + ); +} + +/// **Issue #404 meets the escrow route.** A write from outside the server's protocol window is +/// the gate's `426`, and the SDK reports it as [`RecoveryError::Unexpected`] carrying the +/// gate's own code — never as a verdict on the blob. +/// +/// The window is the one knob a second `App` over the fixture's stores turns: every context +/// but `upload` is the fixture's own (so the session minted on the default-window listener is +/// live on the windowed one), and `upload` carries a policy whose window this build's protocol +/// date falls outside. Two listeners, one account, one sessions store. +#[tokio::test] +async fn an_escrow_write_outside_the_servers_window_is_a_426_the_sdk_reports_with_its_code() { + use capsule_core::crypto::primitives::Argon2Params; + use capsule_core::crypto::pwkdf; + use capsule_sdk::recovery::{RecoveryClient, RecoveryError}; + use kynos::di::Provides as _; + + const VERSION_UNSUPPORTED: &str = "error.protocol.version_unsupported"; + + let fixture = Fixture::working(); + let app = fixture.app(); + let windowed = App::new(Modules { + auth: app.provide(), + totp: app.provide(), + upload: UploadContext::new( + fixture.uploads.clone(), + fixture.blobs.clone(), + fixture.index.clone(), + fixture.authority.clone(), + fixture.clock.clone(), + UploadPolicy::default().with_protocol_window("2000-01-01", "2000-01-01"), + ), + sync: app.provide(), + serve: app.provide(), + verify: app.provide(), + directories: app.provide(), + albums: app.provide(), + quota: app.provide(), + attestation: app.provide(), + discovery: app.provide(), + escrow: app.provide(), + enrollment: app.provide(), + moderation: app.provide(), + share: app.provide(), + drops: app.provide(), + counters: app.provide(), + }); + + // Sign in where the window admits this build; write where it does not. + let admitting = serve(&fixture).await; + let session = session(&admitting).await; + let refusing = serve_app(windowed).await; + let client = RecoveryClient::new(session.clone(), &refusing).expect("an API root parses"); + + let blob = pwkdf::wrap_with( + &[0x5Au8; 32], + b"correct horse battery staple", + Argon2Params { + mem_kib: 64, + t_cost: 1, + p_cost: 1, + }, + ) + .expect("the master key wraps"); + let refused = client + .store_escrow(&blob) + .await + .expect_err("a write from outside the window is refused"); + match &refused { + RecoveryError::Unexpected { status, code } => { + assert_eq!(*status, 426); + assert_eq!(code.as_deref(), Some(VERSION_UNSUPPORTED)); + } + other => panic!("expected the gate's 426, got {other:?}"), + } + assert_eq!( + refused.error_code(), + Some(VERSION_UNSUPPORTED), + "the code a client localizes is the server's, not one this client minted" + ); + + // The window rides the refusing listener's responses, so the client can say which build + // would be admitted. + let response = session + .execute(|http| http.get(format!("{refusing}/v1/version"))) + .await + .expect("the exempt read answers"); + let header = |name: &str| { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + assert_eq!( + header("x-capsule-protocol-min").as_deref(), + Some("2000-01-01") + ); + assert_eq!( + header("x-capsule-protocol-max").as_deref(), + Some("2000-01-01") + ); + + // The same write on the admitting listener lands: the window was the only difference. + RecoveryClient::new(session, &admitting) + .expect("an API root parses") + .store_escrow(&blob) + .await + .expect("this build's protocol date is inside the default window"); +} diff --git a/capsule-wasm/src/lib.rs b/capsule-wasm/src/lib.rs index d8889fd5..5e625e94 100644 --- a/capsule-wasm/src/lib.rs +++ b/capsule-wasm/src/lib.rs @@ -30,6 +30,18 @@ //! fragment. Byte-identical to the server's stored verifier, so the guest proves possession //! without transmitting the passphrase (SSoT: [Web Upload] — Optional passphrase abuse gate). //! +//! The crate additionally carries the **LQIP decode** entry point (slices `S-B14`/`S-B1`), which +//! is the one surface here that is not about crypto: +//! +//! 6. [`decode_lqip`] (`decodeLqip`) — render a sidecar `lqip` record to packed RGBA the viewer +//! can hand straight to `CanvasRenderingContext2D.putImageData`. Infallible: a placeholder is +//! cosmetic, so a malformed record paints a fallback fill rather than failing a gallery. The placeholder lives inside +//! the *encrypted* metadata blob, so the browser only ever holds it after opening a share +//! link — which is why this belongs in the same crate as the open path rather than beside a +//! server route. It is the same [`capsule_core::lqip`] implementation the import pipeline +//! encodes with and the native apps decode with, so a photo's placeholder does not depend on +//! which client is painting it. +//! //! The drop surface is deliberately **contribute-only**: there is no open/decapsulate/decrypt //! entry point for drops (only the provisioning user's *native* client, holding the Drop Key //! private half, can adopt). Keep new browser entry points here behind the same thin-glue @@ -51,6 +63,7 @@ use capsule_core::crypto::encryption::stream::{NONCE_PREFIX_LEN, decrypt_asset_v use capsule_core::crypto::primitives::Argon2Params; use capsule_core::crypto::pwkdf; use capsule_core::drop::{SealedDrop, seal_drop, seal_drop_derand}; +use capsule_core::lqip::{RgbaImage, render as lqip_render}; use capsule_core::sharing::{ LINK_SECRET_LEN, OPAQUE_ID_LEN, ScopeMaterial, SharingError, WrappedScope, open_scope, }; @@ -372,6 +385,105 @@ pub fn drop_passphrase_proof( // `JsValue`. The wasm-side behaviour of the `#[wasm_bindgen]` entry points is covered by // `capsule-web`'s bun KATs. +// ─────────────────────────── LQIP placeholder decode (slice S-B14) ────────────────────────── + +/// A decoded LQIP placeholder: packed RGBA8, ready for `putImageData`. +/// +/// A struct rather than a bare `Vec` because the caller cannot know the dimensions in +/// advance: `decode_capped` returns the largest frame that fits *inside* the requested box while +/// preserving the source aspect ratio, so the answer is `(width, height, rgba)` or nothing. +#[wasm_bindgen] +pub struct WasmLqipImage { + image: RgbaImage, +} + +#[wasm_bindgen] +impl WasmLqipImage { + /// Frame width in pixels — at most the `maxWidth` that was requested. + #[wasm_bindgen(getter)] + #[must_use] + pub fn width(&self) -> u32 { + self.image.width + } + + /// Frame height in pixels — at most the `maxHeight` that was requested. + #[wasm_bindgen(getter)] + #[must_use] + pub fn height(&self) -> u32 { + self.image.height + } + + /// Packed RGBA8 samples, `width * height * 4` bytes — the exact layout + /// `new ImageData(rgba, width, height)` takes. + #[wasm_bindgen(getter)] + #[must_use] + pub fn rgba(&self) -> Vec { + self.image.rgba.clone() + } +} + +/// Render a sidecar `lqip` record to a paintable placeholder, band-limited to the box being +/// painted. +/// +/// - `format_version` — the record's `lqip.format_version`. +/// - `chromahash` — the record's `lqip.chromahash` payload (32 bytes at the committed tier). +/// - `dominant_color` — the record's `lqip.dominant_color`, exactly 3 bytes (opaque RGB). +/// - `max_width` / `max_height` — the box the caller is about to paint. Decoding to the box +/// rather than to a fixed size is the point of `decode_capped`: a grid cell never scales down +/// a larger decode. +/// +/// **Infallible.** An unrecognised `format_version`, a payload the parser rejects, *and* a +/// `dominant_color` that is not three bytes all yield a solid fallback fill rather than a throw. +/// +/// A placeholder is cosmetic: it never damages a library and its absence never loses data, so +/// failing a viewer over one trades a blurry square for a broken gallery. The earlier version +/// threw on a malformed `dominant_color` while the native FFI painted black for the same input — +/// one record, two behaviours, decided by which client opened it. That is exactly the +/// client-dependent divergence `capsule-core::lqip` exists to prevent, so both surfaces now +/// paint the same fallback fill (black, the conventional empty cell) and say so. +#[wasm_bindgen(js_name = decodeLqip)] +pub fn decode_lqip( + format_version: u16, + chromahash: &[u8], + dominant_color: &[u8], + max_width: u32, + max_height: u32, +) -> WasmLqipImage { + WasmLqipImage { + image: render_lqip_record( + format_version, + chromahash, + dominant_color, + max_width, + max_height, + ), + } +} + +/// The colour a malformed record paints when it carries no usable `dominant_color`. +/// +/// Black, the conventional empty-cell fill, and identical to what `capsule-core-ffi`'s +/// `render_lqip` paints for the same input — the two surfaces are asserted to agree. +const FALLBACK_FILL: [u8; 3] = [0, 0, 0]; + +/// [`decode_lqip`] without the JS boundary — the whole of its logic, so the host unit tests can +/// exercise it. +/// +/// The split is not ceremony: `JsError` cannot be *constructed* off-wasm (its host shim aborts), +/// so a test reaching an error arm through the exported function would abort the test binary +/// rather than fail an assertion. It survives the move to an infallible signature because the +/// `#[wasm_bindgen]` wrapper is still unusable from a host test. +fn render_lqip_record( + format_version: u16, + chromahash: &[u8], + dominant_color: &[u8], + max_width: u32, + max_height: u32, +) -> RgbaImage { + let fill: [u8; 3] = dominant_color.try_into().unwrap_or(FALLBACK_FILL); + lqip_render(format_version, chromahash, fill, max_width, max_height) +} + #[cfg(test)] mod tests { use std::collections::HashSet; @@ -487,6 +599,95 @@ mod tests { assert_eq!(decoded, bytes); } + // ── LQIP decode (slice S-B14) ─────────────────────────────────────────── + + /// A gradient frame, the shape `Lqip::encode` takes. + fn gradient(width: u32, height: u32) -> Vec { + let (w, h) = (width as usize, height as usize); + let mut rgba = Vec::with_capacity(w * h * 4); + for y in 0..h { + for x in 0..w { + rgba.extend_from_slice(&[ + (x * 255 / w) as u8, + (y * 255 / h) as u8, + ((x + y) * 255 / (w + h)) as u8, + 255, + ]); + } + } + rgba + } + + /// The browser decodes the same bytes the import pipeline encoded, to the same pixels the + /// core `decode_capped` produces — the `S-B14` cross-surface criterion, at the one boundary + /// where a second implementation could have crept in. + #[test] + fn decode_lqip_matches_the_core_decoder_for_a_real_payload() { + use capsule_core::lqip::{Gamut, LQIP_FORMAT_V1, Lqip}; + + let lqip = Lqip::encode(200, 150, &gradient(200, 150), Gamut::Srgb).expect("encode"); + let payload = lqip.as_bytes(); + assert_eq!(payload.len(), 32, "the committed tier is 32 bytes"); + + let decoded = render_lqip_record(LQIP_FORMAT_V1, payload, &lqip.dominant_color(), 64, 64); + assert_eq!( + decoded, + lqip.decode_capped(64, 64), + "byte-identical to the core decoder" + ); + assert_eq!( + decoded.rgba.len() as u32, + decoded.width * decoded.height * 4, + "the buffer is exactly what `new ImageData(rgba, w, h)` requires" + ); + assert!( + decoded.width <= 64 && decoded.height <= 64, + "the decode is capped to the box being painted, not to a fixed size" + ); + } + + /// An unrecognised version and an undecodable payload both paint the stored fallback colour + /// rather than throwing: a reader must never misrender, and a missing placeholder is not + /// worth failing a gallery over. + #[test] + fn decode_lqip_falls_back_to_the_dominant_colour_instead_of_throwing() { + use capsule_core::lqip::LQIP_FORMAT_V1; + + let fill = [12u8, 34, 56]; + for (version, payload) in [ + (LQIP_FORMAT_V1 + 999, vec![0xDE, 0xAD, 0xBE, 0xEF]), + (LQIP_FORMAT_V1, vec![0xDE, 0xAD, 0xBE, 0xEF]), + (LQIP_FORMAT_V1, Vec::new()), + ] { + let decoded = render_lqip_record(version, &payload, &fill, 32, 32); + assert_eq!((decoded.width, decoded.height), (1, 1)); + assert_eq!(decoded.rgba, vec![12, 34, 56, 255]); + } + } + + /// A malformed `dominant_color` paints [`FALLBACK_FILL`] rather than throwing — the same + /// answer `capsule-core-ffi`'s `render_lqip` gives, so a viewer's behaviour does not depend + /// on which client opened the record. + #[test] + fn decode_lqip_paints_the_fallback_fill_for_a_malformed_dominant_colour() { + use capsule_core::lqip::LQIP_FORMAT_V1; + + for fill in [&[][..], &[1][..], &[1, 2][..], &[1, 2, 3, 4][..]] { + let decoded = render_lqip_record(LQIP_FORMAT_V1, &[0; 32], fill, 16, 16); + assert_eq!( + (decoded.width, decoded.height), + (1, 1), + "a {}-byte dominant_color paints the fill, not an error", + fill.len() + ); + assert_eq!( + decoded.rgba, + vec![FALLBACK_FILL[0], FALLBACK_FILL[1], FALLBACK_FILL[2], 255], + "and it is black — identical to the native surface" + ); + } + } + #[test] fn decode_wrapped_round_trips_a_canonical_wrapped_scope() { let wrapped = WrappedScope::LinkOnly { diff --git a/mise.toml b/mise.toml index d4f7fb04..07cd924c 100644 --- a/mise.toml +++ b/mise.toml @@ -154,6 +154,8 @@ run = "cargo doc --no-deps --document-private-items -p capsule-core -p capsule-c # nextest: cross-binary parallel scheduling + process-per-test isolation. Two # invocations — the workspace (default features), then capsule-core's FFI surface. +# The workspace run includes `capsule-e2e`, the bounded E2E cases over the real SDK, +# library and server composition root (no container, no env gate; issue #409). # nextest does not run doctests; the workspace has none (if you add one, also add a # `cargo test --doc` step). CI selects the `ci` profile via NEXTEST_PROFILE=ci. [tasks.test-rust]