From 39986a2a8850831720ca542f300ce162556bcaf4 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 5 Aug 2026 16:29:27 -0400 Subject: [PATCH 1/6] refactor(hwdec): share HEVC and AV1 parsing --- .../src/{vaapi => bitstream}/av1.rs | 12 +++++++++--- .../src/{vaapi => bitstream}/bits.rs | 2 +- .../src/{vaapi => bitstream}/hevc.rs | 19 ++++++++++++++++--- crates/rawshift-hwdec/src/bitstream/mod.rs | 17 +++++++++++++++++ crates/rawshift-hwdec/src/lib.rs | 3 +++ crates/rawshift-hwdec/src/vaapi/mod.rs | 6 ++---- 6 files changed, 48 insertions(+), 11 deletions(-) rename crates/rawshift-hwdec/src/{vaapi => bitstream}/av1.rs (99%) rename crates/rawshift-hwdec/src/{vaapi => bitstream}/bits.rs (99%) rename crates/rawshift-hwdec/src/{vaapi => bitstream}/hevc.rs (98%) create mode 100644 crates/rawshift-hwdec/src/bitstream/mod.rs diff --git a/crates/rawshift-hwdec/src/vaapi/av1.rs b/crates/rawshift-hwdec/src/bitstream/av1.rs similarity index 99% rename from crates/rawshift-hwdec/src/vaapi/av1.rs rename to crates/rawshift-hwdec/src/bitstream/av1.rs index 66b6d96..be3e52c 100644 --- a/crates/rawshift-hwdec/src/vaapi/av1.rs +++ b/crates/rawshift-hwdec/src/bitstream/av1.rs @@ -1,5 +1,5 @@ -//! AV1 still-picture parsing for the VAAPI backend — **safe Rust only** -//! (no `unsafe`; FFI stays in `sys.rs`/`mod.rs`). +//! AV1 still-picture parsing shared by hardware backends — **safe Rust only** +//! (no `unsafe`; platform FFI stays in backend modules). //! //! ## Scope //! @@ -16,7 +16,8 @@ //! `VASliceParameterBufferAV1` per tile. use super::bits::{BitReader, PResult, ParseError, clip3}; -use super::sys; +#[cfg(hwdec_backend = "vaapi")] +use crate::vaapi::sys; // ── OBU framing (§5.3) ────────────────────────────────────────────────────── @@ -1144,6 +1145,7 @@ pub fn parse_still_picture(config_obus: &[u8], payload: &[u8]) -> PResult sys::VASliceParameterBufferAV1 { sys::VASliceParameterBufferAV1 { slice_data_size: tile.data.len() as u32, @@ -1516,6 +1519,9 @@ mod tests { ); } + // Exercises the VAAPI parameter-buffer builders, which only exist in a + // VAAPI build (the parsers above are shared by every backend). + #[cfg(hwdec_backend = "vaapi")] #[test] fn pic_param_maps_seq_and_frame_fields() { let pic = parse_still_picture(&[], AV1_64X64_LIBAOM).unwrap(); diff --git a/crates/rawshift-hwdec/src/vaapi/bits.rs b/crates/rawshift-hwdec/src/bitstream/bits.rs similarity index 99% rename from crates/rawshift-hwdec/src/vaapi/bits.rs rename to crates/rawshift-hwdec/src/bitstream/bits.rs index 4e34c14..3f328d8 100644 --- a/crates/rawshift-hwdec/src/vaapi/bits.rs +++ b/crates/rawshift-hwdec/src/bitstream/bits.rs @@ -1,4 +1,4 @@ -//! Safe bitstream readers shared by the HEVC and AV1 header parsers. +//! Safe bitstream readers shared by the HEVC and AV1 still-picture parsers. //! //! **Safe Rust only** — this module (like `hevc.rs` / `av1.rs`) contains no //! `unsafe`; all FFI stays in `sys.rs` and the call sites in `mod.rs`. diff --git a/crates/rawshift-hwdec/src/vaapi/hevc.rs b/crates/rawshift-hwdec/src/bitstream/hevc.rs similarity index 98% rename from crates/rawshift-hwdec/src/vaapi/hevc.rs rename to crates/rawshift-hwdec/src/bitstream/hevc.rs index f9df68c..b52d30d 100644 --- a/crates/rawshift-hwdec/src/vaapi/hevc.rs +++ b/crates/rawshift-hwdec/src/bitstream/hevc.rs @@ -1,5 +1,5 @@ -//! HEVC still-picture header parsing for the VAAPI backend — **safe Rust -//! only** (no `unsafe`; FFI stays in `sys.rs`/`mod.rs`). +//! HEVC still-picture header parsing shared by hardware backends — **safe +//! Rust only** (no `unsafe`; platform FFI stays in backend modules). //! //! ## Scope //! @@ -23,7 +23,8 @@ //! `slice_data()`) and the emulation-prevention-byte count VAAPI wants. use super::bits::{BitReader, PResult, ParseError, Rbsp, rbsp_from_nal_payload}; -use super::sys; +#[cfg(hwdec_backend = "vaapi")] +use crate::vaapi::sys; // ── NAL classification ────────────────────────────────────────────────────── @@ -743,6 +744,7 @@ fn ceil_log2(x: u32) -> u32 { /// Uniform tile partition of `total` CTBs into `count` parts, as the /// `minus1` sizes VAAPI wants (H.265 §6.5.1 derivation). +#[cfg(hwdec_backend = "vaapi")] fn uniform_partition_minus1(total: u32, count: u32) -> Vec { (0..count) .map(|i| (((i + 1) * total) / count - (i * total) / count - 1) as u16) @@ -752,6 +754,7 @@ fn uniform_partition_minus1(total: u32, count: u32) -> Vec { /// Fill `VAPictureParameterBufferHEVC` for a still picture decoded into /// `surface`. `nut` is the slice NAL type; `st_rps_bits` comes from the /// first independent slice header. +#[cfg(hwdec_backend = "vaapi")] pub fn build_pic_param( sps: &Sps, pps: &Pps, @@ -894,6 +897,7 @@ pub fn build_pic_param( /// Fill `VASliceParameterBufferHEVC` for one coded slice NAL of /// `slice_data_size` bytes at offset 0 of its own data buffer. +#[cfg(hwdec_backend = "vaapi")] pub fn build_slice_param( sh: &SliceHeader, pps: &Pps, @@ -1019,6 +1023,9 @@ mod tests { assert!(!pps.slice_segment_header_extension_present); } + // Exercises the VAAPI parameter-buffer builders, which only exist in a + // VAAPI build (the parsers above are shared by every backend). + #[cfg(hwdec_backend = "vaapi")] #[test] fn pic_param_packs_bitfields() { let sps = parse_sps(SPS_64X64_X265).unwrap(); @@ -1044,6 +1051,9 @@ mod tests { ); } + // Exercises the VAAPI parameter-buffer builders, which only exist in a + // VAAPI build (the parsers above are shared by every backend). + #[cfg(hwdec_backend = "vaapi")] #[test] fn uniform_tile_partition_covers_exactly() { // 10 CTBs into 3 columns: 3+3+4 (spec derivation gives 3,3,4). @@ -1052,6 +1062,9 @@ mod tests { assert_eq!(parts.len(), 3); } + // Exercises the VAAPI parameter-buffer builders, which only exist in a + // VAAPI build (the parsers above are shared by every backend). + #[cfg(hwdec_backend = "vaapi")] #[test] fn slice_param_marks_last_slice_and_i_type() { let sh = SliceHeader { diff --git a/crates/rawshift-hwdec/src/bitstream/mod.rs b/crates/rawshift-hwdec/src/bitstream/mod.rs new file mode 100644 index 0000000..d36ca63 --- /dev/null +++ b/crates/rawshift-hwdec/src/bitstream/mod.rs @@ -0,0 +1,17 @@ +//! Safe parsing and framing shared by the platform hardware-decode backends. +//! +//! Each backend consumes a different subset of this module, because the +//! platform APIs sit at different levels. VAAPI is a slice-level API, so it +//! needs everything down to slice headers, tile groups and the +//! `build_*_param` builders. VideoToolbox takes the configuration record and +//! the coded sample almost as-is, so it only needs `hvcC`/`av1C` parsing, NAL +//! classification and the AV1 sequence header — it lets the framework do the +//! rest. The unused remainder is therefore expected rather than dead, and the +//! allow below keeps `-D warnings` builds honest for every backend selection +//! without scattering per-item `cfg`s that would couple this module to the +//! backend list. +#![allow(dead_code)] + +pub(crate) mod av1; +pub(crate) mod bits; +pub(crate) mod hevc; diff --git a/crates/rawshift-hwdec/src/lib.rs b/crates/rawshift-hwdec/src/lib.rs index 5d77541..3b065ea 100644 --- a/crates/rawshift-hwdec/src/lib.rs +++ b/crates/rawshift-hwdec/src/lib.rs @@ -48,6 +48,9 @@ #![deny(unsafe_op_in_unsafe_fn)] +#[cfg(any(hwdec_backend = "vaapi", hwdec_backend = "mediacodec", test))] +mod bitstream; + // The VAAPI platform backend: compiled only when build.rs selected it // (`vaapi` explicit flag, or `hw` on a linux-gnu target). #[cfg(hwdec_backend = "vaapi")] diff --git a/crates/rawshift-hwdec/src/vaapi/mod.rs b/crates/rawshift-hwdec/src/vaapi/mod.rs index faf0aed..52fff27 100644 --- a/crates/rawshift-hwdec/src/vaapi/mod.rs +++ b/crates/rawshift-hwdec/src/vaapi/mod.rs @@ -36,16 +36,14 @@ //! invariant. The bitstream parsers ([`bits`], [`hevc`], [`av1`]) are safe //! Rust. -mod av1; -mod bits; -mod hevc; -mod sys; +pub(crate) mod sys; use std::ffi::c_int; use std::fs::File; use std::os::fd::AsRawFd; use std::sync::OnceLock; +use crate::bitstream::{av1, bits, hevc}; use crate::{ CodecConfig, ColorRange, DecodedFrame, HwBackend, HwCodec, HwDecodeError, HwStillDecoder, PixelFormat, Plane, StillDecodeRequest, From a0d99df5aab45c2ec2de32c348e828853d057bc2 Mon Sep 17 00:00:00 2001 From: Justin Chung <20733699+justin13888@users.noreply.github.com> Date: Sat, 22 Aug 2026 05:43:06 -0400 Subject: [PATCH 2/6] =?UTF-8?q?feat(hwdec):=20VideoToolbox=20backend=20?= =?UTF-8?q?=E2=80=94=20HEVC=20+=20AV1=20still=20decode=20on=20macOS/iOS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the third platform backend behind the existing `HwStillDecoder` contract, so HEIC and AVIF pixel decode work end to end on Apple hardware. `build.rs` already mapped macOS/iOS to `videotoolbox` and `HwBackend` already had the variant; this fills in the module and its dispatch arms. - HEVC: `CMVideoFormatDescriptionCreateFromHEVCParameterSets` from the hvcC parameter sets, with the length-prefixed payload passed through byte for byte — no Annex-B conversion. VideoToolbox parses the SPS, so the advisory (and, from the HEIC adapter, always zero) picture size is never needed. - AV1: `av1C` carried in `SampleDescriptionExtensionAtoms`, dimensions from the sequence header. VideoToolbox *requires* the sequence header in the config atom — a record with empty `configOBUs` fails session creation at every picture size — so when the sample carries it instead, the record is rebuilt with it. - Availability is `VTIsHardwareDecodeSupported` per codec, cached once per process: AV1 is reported only on hardware that has an AV1 decode block (M3 / A17 Pro and later), while HEVC keeps working on older machines. - Sessions are created hardware-pinned on macOS via `RequireHardwareAcceleratedVideoDecoder`, then retried once without it. Apple's hardware HEVC block refuses pictures below roughly 64x64 and HEIF thumbnails are routinely 32x32, so refusing them outright would be worse than letting VideoToolbox decode that one picture itself. Availability is still gated on the hardware probe, so this only widens accepted picture sizes, never the codec list. - The session is cached and reused across pictures sharing a configuration record, which is what a grid HEIC's hundreds of tiles hit. - Decode is synchronous (no async/temporal flags, plus an explicit wait), so the payload is wrapped zero-copy with `kCFAllocatorNull`. - An explicit destination pixel-format list is requested. Left to itself a 10-bit decode natively produces `'p420'`, which appears in no public CoreVideo header and whose plane contents do not match the documented `x420` samples for the same bitstream. Offering all four documented 4:2:0 surfaces keeps the decoder on interpretable layouts while still letting it pick the range-matching one, so video/full range fidelity survives. Bindings are the generated `objc2-*` framework crates rather than a hand-written `sys.rs` — the opposite call from VAAPI, because libva must be dlopen'd to stay headless-safe whereas the Apple frameworks are guaranteed present and link normally. They are taken with `default-features = false` and only the per-header features used, so no Objective-C runtime, Metal, OpenGL or CoreAudio is compiled in. Rationale is recorded in the module docs. Claude-Session: https://claude.ai/code/session_01YTL5nD4tjuppsDGMRFFEoh --- Cargo.lock | 45 ++ crates/rawshift-hwdec/Cargo.toml | 61 +- crates/rawshift-hwdec/src/lib.rs | 61 +- .../rawshift-hwdec/src/videotoolbox/format.rs | 425 ++++++++++++ crates/rawshift-hwdec/src/videotoolbox/mod.rs | 613 ++++++++++++++++++ .../rawshift-hwdec/src/videotoolbox/pixels.rs | 406 ++++++++++++ .../rawshift-hwdec/src/videotoolbox/sample.rs | 155 +++++ .../rawshift-hwdec/src/videotoolbox/status.rs | 308 +++++++++ 8 files changed, 2054 insertions(+), 20 deletions(-) create mode 100644 crates/rawshift-hwdec/src/videotoolbox/format.rs create mode 100644 crates/rawshift-hwdec/src/videotoolbox/mod.rs create mode 100644 crates/rawshift-hwdec/src/videotoolbox/pixels.rs create mode 100644 crates/rawshift-hwdec/src/videotoolbox/sample.rs create mode 100644 crates/rawshift-hwdec/src/videotoolbox/status.rs diff --git a/Cargo.lock b/Cargo.lock index f33e310..88c66fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1030,6 +1030,47 @@ dependencies = [ "autocfg", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.11.0", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags 2.11.0", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-video-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05bf9a3c14831a7d9641b0d81d87dd913ee238a012b2fde27db5a84b56f5df3e" +dependencies = [ + "bitflags 2.11.0", + "objc2-core-foundation", + "objc2-core-media", + "objc2-core-video", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -1202,6 +1243,10 @@ version = "0.1.1" dependencies = [ "gamut-color", "libloading", + "objc2-core-foundation", + "objc2-core-media", + "objc2-core-video", + "objc2-video-toolbox", "thiserror", ] diff --git a/crates/rawshift-hwdec/Cargo.toml b/crates/rawshift-hwdec/Cargo.toml index c24b58e..3a45631 100644 --- a/crates/rawshift-hwdec/Cargo.toml +++ b/crates/rawshift-hwdec/Cargo.toml @@ -30,6 +30,56 @@ thiserror = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] libloading = "0.8" +# VideoToolbox/CoreMedia/CoreVideo are always-present Apple system frameworks, +# so unlike libva they are linked normally rather than dlopen'd. The `objc2-*` +# framework crates are the maintained, generated bindings for them (MSRV 1.71, +# Zlib OR Apache-2.0 OR MIT) and carry the `#[link(kind = "framework")]` +# attributes, so build.rs needs no link directives. They also give us +# `CFRetained` RAII instead of hand-rolled CFRetain/CFRelease pairs — the +# reason this backend uses generated bindings where the VAAPI one hand-writes +# `sys.rs` (see the module docs in src/videotoolbox/mod.rs). +# +# `default-features = false` throughout: the default feature sets pull in the +# Objective-C runtime, Metal, OpenGL and CoreAudio, none of which a still-frame +# decode touches. Only the granular per-header features below are enabled. +[target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies] +objc2-video-toolbox = { version = "0.3.2", default-features = false, features = [ + "std", + "bitflags", + "VTBase", + "VTErrors", + "VTDecompressionSession", + "VTDecompressionProperties", + "objc2-core-media", + "objc2-core-video", +] } +objc2-core-media = { version = "0.3.2", default-features = false, features = [ + "std", + "bitflags", + "CMBase", + "CMTime", + "CMBlockBuffer", + "CMFormatDescription", + "CMSampleBuffer", +] } +objc2-core-video = { version = "0.3.2", default-features = false, features = [ + "std", + "bitflags", + "CVBase", + "CVReturn", + "CVBuffer", + "CVImageBuffer", + "CVPixelBuffer", +] } +objc2-core-foundation = { version = "0.3.2", default-features = false, features = [ + "std", + "CFBase", + "CFString", + "CFData", + "CFNumber", + "CFDictionary", +] } + [features] # Verified backend feature flags — see docs/SUPPORT.md for the permanent # target/API matrix. Each explicit backend flag hard-fails the compile @@ -39,11 +89,12 @@ libloading = "0.8" # (windows-msvc, linux-musl, wasm) emits a build-script warning and compiles # the no-backend stub. # -# Backends implemented: VAAPI (linux-gnu; dlopen'd libva, HEVC Main/Main10 + -# AV1 Profile 0 still pictures). VideoToolbox / MediaCodec land as separate -# issues; builds without a selected backend compile the no-backend stub — -# `decoder()` returns `None`, `backend()` returns `None`, and -# `available_codecs()` is empty. +# Backends implemented: VideoToolbox (macOS/iOS; linked system frameworks, +# hardware-only via VTIsHardwareDecodeSupported) and VAAPI (linux-gnu; +# dlopen'd libva), both for HEVC Main/Main10 + AV1 Profile 0 still pictures. +# MediaCodec lands as a separate issue; builds without a selected backend +# compile the no-backend stub — `decoder()` returns `None`, `backend()` +# returns `None`, and `available_codecs()` is empty. videotoolbox = [] vaapi = [] mediacodec = [] diff --git a/crates/rawshift-hwdec/src/lib.rs b/crates/rawshift-hwdec/src/lib.rs index 3b065ea..bed51e1 100644 --- a/crates/rawshift-hwdec/src/lib.rs +++ b/crates/rawshift-hwdec/src/lib.rs @@ -10,11 +10,12 @@ //! Per `PRINCIPLES.md`, **all** platform FFI for hardware decode lives in this //! crate and nowhere else: `#![deny(unsafe_op_in_unsafe_fn)]`, every public //! item is safe, and every `unsafe` block documents its invariants inside the -//! platform backend module that owns it. The **VAAPI backend** (linux-gnu, -//! dlopen'd libva — see the `vaapi` module) is implemented; VideoToolbox and -//! MediaCodec land as separate issues. On targets/builds with no backend -//! every entry point reports "no decoder" — [`decoder`] returns `None`, -//! [`backend`] returns `None`, and [`available_codecs`] is empty. +//! platform backend module that owns it. The **VideoToolbox backend** +//! (macOS/iOS, linked system frameworks, hardware-only) and the **VAAPI +//! backend** (linux-gnu, dlopen'd libva) are implemented; MediaCodec lands as +//! a separate issue. On targets/builds with no backend every entry point +//! reports "no decoder" — [`decoder`] returns `None`, [`backend`] returns +//! `None`, and [`available_codecs`] is empty. //! //! On NVIDIA GPUs the VAAPI backend works through the maintained //! [`nvidia-vaapi-driver`](https://github.com/elFarto/nvidia-vaapi-driver) @@ -48,9 +49,19 @@ #![deny(unsafe_op_in_unsafe_fn)] -#[cfg(any(hwdec_backend = "vaapi", hwdec_backend = "mediacodec", test))] +#[cfg(any( + hwdec_backend = "vaapi", + hwdec_backend = "videotoolbox", + hwdec_backend = "mediacodec", + test +))] mod bitstream; +// The VideoToolbox platform backend: compiled only when build.rs selected it +// (`videotoolbox` explicit flag, or `hw` on a macOS/iOS target). +#[cfg(hwdec_backend = "videotoolbox")] +mod videotoolbox; + // The VAAPI platform backend: compiled only when build.rs selected it // (`vaapi` explicit flag, or `hw` on a linux-gnu target). #[cfg(hwdec_backend = "vaapi")] @@ -478,19 +489,29 @@ pub enum HwDecodeError { /// Returns a decoder for `codec`, or `None` when no compiled-in backend can /// decode it at runtime. /// +/// With the VideoToolbox backend compiled in (`videotoolbox`, or `hw` on +/// macOS/iOS), this answers from `VTIsHardwareDecodeSupported` — so AV1 is +/// reported only on hardware that actually has an AV1 decode block (M3 / +/// A17 Pro and later), while HEVC keeps working on older machines. +/// /// With the VAAPI backend compiled in (`vaapi`, or `hw` on linux-gnu), this /// dlopens libva on first use and answers from the driver's actual /// profile/entrypoint list; missing libraries, render nodes, or driver /// support all degrade to `None` (never a link or startup failure). -/// VideoToolbox and MediaCodec land as separate issues; without a backend -/// this returns `None` everywhere. +/// +/// MediaCodec lands as a separate issue; without a backend this returns +/// `None` everywhere. #[must_use] pub fn decoder(codec: HwCodec) -> Option> { + #[cfg(hwdec_backend = "videotoolbox")] + { + videotoolbox::decoder(codec) + } #[cfg(hwdec_backend = "vaapi")] { vaapi::decoder(codec) } - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] { let _ = codec; None @@ -500,32 +521,42 @@ pub fn decoder(codec: HwCodec) -> Option> { /// The platform backend compiled into this build and usable at runtime, or /// `None`. /// -/// Reports `Some` only when the runtime probe succeeds (e.g. VAAPI's dlopen +/// Reports `Some` only when the runtime probe succeeds (e.g. VideoToolbox +/// reporting hardware decode for at least one codec, or VAAPI's dlopen /// finding a driver with at least one supported codec at the VLD entry /// point). #[must_use] pub fn backend() -> Option { + #[cfg(hwdec_backend = "videotoolbox")] + { + videotoolbox::backend() + } #[cfg(hwdec_backend = "vaapi")] { vaapi::backend() } - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] { None } } /// The codecs [`decoder`] can currently return a decoder for, per the -/// runtime probe (e.g. `vaQueryConfigProfiles` for VAAPI). +/// runtime probe (`VTIsHardwareDecodeSupported` for VideoToolbox, +/// `vaQueryConfigProfiles` for VAAPI). /// /// Empty when no backend is compiled in or usable. #[must_use] pub fn available_codecs() -> &'static [HwCodec] { + #[cfg(hwdec_backend = "videotoolbox")] + { + videotoolbox::available_codecs() + } #[cfg(hwdec_backend = "vaapi")] { vaapi::available_codecs() } - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] { &[] } @@ -537,14 +568,14 @@ mod tests { // ── stub behaviour (builds with no selected backend) ──────────────────── - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] #[test] fn stub_has_no_decoder_for_any_codec() { assert!(decoder(HwCodec::Hevc).is_none()); assert!(decoder(HwCodec::Av1).is_none()); } - #[cfg(not(hwdec_backend = "vaapi"))] + #[cfg(not(any(hwdec_backend = "videotoolbox", hwdec_backend = "vaapi")))] #[test] fn stub_reports_no_backend_and_no_codecs() { assert_eq!(backend(), None); diff --git a/crates/rawshift-hwdec/src/videotoolbox/format.rs b/crates/rawshift-hwdec/src/videotoolbox/format.rs new file mode 100644 index 0000000..497c0dc --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/format.rs @@ -0,0 +1,425 @@ +//! Building the `CMVideoFormatDescription` that a decompression session is +//! created from, for both codecs. +//! +//! ## HEVC +//! +//! `CMVideoFormatDescriptionCreateFromHEVCParameterSets` takes the VPS/SPS/PPS +//! NAL units out of the `hvcC` record plus the record's NAL length prefix +//! width. Two things fall out of that choice: +//! +//! - VideoToolbox parses the SPS itself, so the coded dimensions never have to +//! come from [`StillDecodeRequest::width`]/`height` — which the HEIC adapter +//! passes as `0` precisely because `hvcC` carries no picture size. +//! - `nal_unit_header_length` *is* the "length-prefixed samples, no Annex-B +//! conversion" contract: the payload is handed to the decoder byte for byte. +//! +//! ## AV1 +//! +//! CoreMedia has no `…CreateFromAV1ParameterSets`, so the generic +//! `CMVideoFormatDescriptionCreate` is used with the whole `av1C` record body +//! carried in the standard `SampleDescriptionExtensionAtoms` dictionary. That +//! call *does* require non-zero dimensions, which come from the AV1 sequence +//! header (authoritative, and what VideoToolbox itself will parse) with the +//! advisory request fields as a last resort. + +use std::ffi::c_int; +use std::ptr::{self, NonNull}; + +use objc2_core_foundation::{CFData, CFDictionary, CFRetained, CFString, CFType}; +use objc2_core_media::{ + CMFormatDescription, CMVideoFormatDescriptionCreate, + CMVideoFormatDescriptionCreateFromHEVCParameterSets, + kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, kCMVideoCodecType_AV1, +}; + +use super::status::{NO_ERR, decode_err, map_status}; +use crate::bitstream::{av1, hevc}; +use crate::{HwCodec, HwDecodeError, StillDecodeRequest}; + +/// The NAL unit types `CMVideoFormatDescriptionCreateFromHEVCParameterSets` +/// accepts: VPS (32), SPS (33), PPS (34), prefix SEI (39), suffix SEI (40). +const PARAMETER_SET_NALS: [u8; 5] = [32, 33, 34, 39, 40]; + +/// Erase a typed dictionary's phantom generics for the C signatures, which all +/// take the fully opaque `CFDictionary`. +pub(super) fn as_opaque_dict(dict: &CFDictionary) -> &CFDictionary { + // SAFETY: `CFDictionary` is a zero-sized opaque wrapper whose type + // parameters are pure `PhantomData` — every instantiation has identical + // layout and wraps the same underlying `CFDictionaryRef`. The C entry + // points only read it as an opaque dictionary. + unsafe { &*(dict as *const CFDictionary).cast::() } +} + +/// Take ownership of a `Create`-rule format description returned through an +/// out-parameter. +fn adopt( + codec: HwCodec, + operation: &str, + status: i32, + raw: *const CMFormatDescription, +) -> Result, HwDecodeError> { + if status != NO_ERR { + return Err(map_status(codec, operation, status)); + } + let ptr = NonNull::new(raw.cast_mut()) + .ok_or_else(|| decode_err(codec, format!("{operation} returned no format description")))?; + // SAFETY: the call succeeded, so `ptr` is a format description the Create + // rule already gave us a +1 reference to; `CFRetained::from_raw` adopts + // that reference and releases it exactly once on drop. + Ok(unsafe { CFRetained::from_raw(ptr) }) +} + +/// Build the format description for an HEVC still from its `hvcC` record. +pub(super) fn hevc_format_description( + hvcc_bytes: &[u8], + request: &StillDecodeRequest<'_>, +) -> Result, HwDecodeError> { + let codec = HwCodec::Hevc; + let to_err = |e: crate::bitstream::bits::ParseError| decode_err(codec, e.0); + let hvcc = hevc::parse_hvcc(hvcc_bytes).map_err(to_err)?; + + // Parameter sets from the hvcC arrays, in record order. + let mut sets: Vec<&[u8]> = Vec::new(); + let (mut has_vps, mut has_sps, mut has_pps) = (false, false, false); + for nal in &hvcc.nal_units { + let nal_type = hevc::nal_type(nal).map_err(to_err)?; + if !PARAMETER_SET_NALS.contains(&nal_type) { + continue; + } + match nal_type { + 32 => has_vps = true, + 33 => has_sps = true, + 34 => has_pps = true, + _ => {} + } + sets.push(nal.as_slice()); + } + + // Some encoders leave one or more parameter sets out of the hvcC arrays + // and send them in band instead; the VAAPI backend accepts either source, + // so this one does too. + if !(has_vps && has_sps && has_pps) { + let payload_nals = + hevc::split_length_prefixed(request.payload, hvcc.nal_length_size).map_err(to_err)?; + for nal in payload_nals { + let nal_type = hevc::nal_type(nal).map_err(to_err)?; + let wanted = match nal_type { + 32 if !has_vps => { + has_vps = true; + true + } + 33 if !has_sps => { + has_sps = true; + true + } + 34 if !has_pps => { + has_pps = true; + true + } + _ => false, + }; + if wanted { + sets.push(nal); + } + } + } + + if !(has_vps && has_sps && has_pps) { + return Err(decode_err( + codec, + "hvcC and payload carry no complete VPS/SPS/PPS set \ + (VideoToolbox requires all three)", + )); + } + + let pointers: Vec> = sets + .iter() + .map(|nal| { + NonNull::new(nal.as_ptr().cast_mut()) + .ok_or_else(|| decode_err(codec, "empty parameter set NAL")) + }) + .collect::>()?; + let sizes: Vec = sets.iter().map(|nal| nal.len()).collect(); + + let mut raw: *const CMFormatDescription = ptr::null(); + // SAFETY: `pointers` and `sizes` are non-empty parallel arrays of exactly + // `pointers.len()` entries. Each pointer is the start of a live parameter + // set NAL borrowed from `hvcc` or `request.payload`, both of which outlive + // this call, and the matching `sizes` entry is that NAL's length. + // `nal_length_size` is 1, 2 or 4 (`parse_hvcc` rejects 3). `raw` is a live + // local that the framework writes exactly one +1 reference into. + let status = unsafe { + CMVideoFormatDescriptionCreateFromHEVCParameterSets( + None, + pointers.len(), + NonNull::from(&pointers[0]), + NonNull::from(&sizes[0]), + hvcc.nal_length_size as c_int, + None, + NonNull::from(&mut raw), + ) + }; + adopt( + codec, + "CMVideoFormatDescriptionCreateFromHEVCParameterSets", + status, + raw, + ) +} + +/// `OBU_SEQUENCE_HEADER` (AV1 §6.2.2). +const OBU_SEQUENCE_HEADER: u8 = 1; + +/// Encode a length as a LEB128 `obu_size` field (AV1 §4.10.5). +fn leb128(mut value: usize, out: &mut Vec) { + loop { + let mut byte = (value & 0x7f) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + out.push(byte); + if value == 0 { + break; + } + } +} + +/// Re-frame a sequence-header payload as a canonical OBU with a size field. +/// +/// `split_obus` hands back payloads rather than byte ranges, so a sequence +/// header recovered from the sample is rebuilt here instead of being sliced +/// out of the original stream. The result is byte-identical in meaning: OBU +/// type 1, `obu_has_size_field = 1`, no extension. +fn sequence_header_obu(payload: &[u8]) -> Vec { + let mut obu = Vec::with_capacity(payload.len() + 3); + obu.push((OBU_SEQUENCE_HEADER << 3) | 0b10); // type, obu_has_size_field + leb128(payload.len(), &mut obu); + obu.extend_from_slice(payload); + obu +} + +/// The sequence header for an AV1 still, from `av1C.configOBUs` if it is +/// there and from the sample otherwise. +/// +/// AVIF permits the sequence header in either place, but **VideoToolbox +/// requires it in the `av1C` config atom**: given a record whose `configOBUs` +/// are empty it fails `VTDecompressionSessionCreate` outright with +/// `kVTVideoDecoderMalfunctionErr`, at every picture size. So when the sample +/// is carrying it, the record is rebuilt with it (see [`av1_format_description`]). +fn av1_sequence_header( + config_obus: &[u8], + payload: &[u8], +) -> Result<(av1::SequenceHeader, Option>), HwDecodeError> { + let codec = HwCodec::Av1; + let to_err = |e: crate::bitstream::bits::ParseError| decode_err(codec, e.0); + // `parse_sequence_header` takes an OBU *payload*, so the stream has to be + // split first — handing it the raw stream would parse the OBU header bytes + // as sequence-header fields and yield nonsense dimensions. + for (source, from_config) in [(config_obus, true), (payload, false)] { + if source.is_empty() { + continue; + } + let obus = av1::split_obus(source).map_err(to_err)?; + let Some(obu) = obus.iter().find(|obu| obu.obu_type == OBU_SEQUENCE_HEADER) else { + continue; + }; + let seq = av1::parse_sequence_header(obu.payload).map_err(to_err)?; + let rebuilt = (!from_config).then(|| sequence_header_obu(obu.payload)); + return Ok((seq, rebuilt)); + } + Err(decode_err( + codec, + "AV1 stream carries no sequence header in either the av1C record or \ + the sample; VideoToolbox cannot be configured without one", + )) +} + +/// Build the format description for an AV1 still from its `av1C` record. +pub(super) fn av1_format_description( + av1c_bytes: &[u8], + request: &StillDecodeRequest<'_>, +) -> Result, HwDecodeError> { + let codec = HwCodec::Av1; + let to_err = |e: crate::bitstream::bits::ParseError| decode_err(codec, e.0); + let av1c = av1::parse_av1c(av1c_bytes).map_err(to_err)?; + if av1c.seq_profile != 0 { + return Err(decode_err( + codec, + format!( + "AV1 profile {} is outside the still-picture scope \ + (VideoToolbox decodes Main / Profile 0)", + av1c.seq_profile + ), + )); + } + + let (seq, rebuilt) = av1_sequence_header(av1c.config_obus, request.payload)?; + + // The sequence header is authoritative for the coded size — and it is what + // VideoToolbox itself parses. The container's advisory `ispe` size is only + // a fallback, because it can legitimately disagree (rotation, cropping). + let (width, height) = match (seq.max_frame_width, seq.max_frame_height) { + (w, h) if w > 0 && h > 0 => (w, h), + _ if request.width > 0 && request.height > 0 => (request.width, request.height), + _ => { + return Err(decode_err( + codec, + "AV1 sequence header declares no picture size and the container \ + supplied none", + )); + } + }; + let width = i32::try_from(width) + .map_err(|_| decode_err(codec, "AV1 picture width does not fit in i32"))?; + let height = i32::try_from(height) + .map_err(|_| decode_err(codec, "AV1 picture height does not fit in i32"))?; + + // `SampleDescriptionExtensionAtoms = { "av1C": }` — the + // whole record including its four-byte header, not just `configOBUs`. When + // the sequence header lived in the sample rather than the record, it is + // appended here so VideoToolbox gets the config it insists on. + let record: Vec; + let atom_bytes = match rebuilt { + Some(seq_obu) => { + record = av1c_bytes + .get(..4) + .unwrap_or(av1c_bytes) + .iter() + .copied() + .chain(seq_obu) + .collect(); + record.as_slice() + } + None => av1c_bytes, + }; + let atom_key = CFString::from_static_str("av1C"); + let atom_value = CFData::from_bytes(atom_bytes); + let atoms: CFRetained> = + CFDictionary::from_slices(&[&*atom_key], &[atom_value.as_ref() as &CFType]); + // SAFETY: an immortal constant `CFStringRef` exported by CoreMedia. + let atoms_key = unsafe { kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms }; + let extensions: CFRetained> = + CFDictionary::from_slices(&[atoms_key], &[atoms.as_ref() as &CFType]); + + let mut raw: *const CMFormatDescription = ptr::null(); + // SAFETY: `extensions` is a live dictionary whose keys are `CFString` and + // whose values are CF objects, which is what the "extensions generics must + // be of the correct type" precondition asks for; it is deep-copied by the + // call, so it need not outlive it. `raw` is a live local the framework + // writes exactly one +1 reference into. + let status = unsafe { + CMVideoFormatDescriptionCreate( + None, + kCMVideoCodecType_AV1, + width, + height, + Some(as_opaque_dict(&extensions)), + NonNull::from(&mut raw), + ) + }; + adopt(codec, "CMVideoFormatDescriptionCreate", status, raw) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ChromaSubsampling, CodecConfig}; + + fn request<'a>(config: CodecConfig<'a>, payload: &'a [u8]) -> StillDecodeRequest<'a> { + StillDecodeRequest { + config, + payload, + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + } + } + + /// A 23-byte hvcC header plus one array per parameter set, built the way + /// `tests/vaapi_device.rs` builds them from an Annex-B stream. + fn hvcc_with(sets: &[(u8, &[u8])]) -> Vec { + let mut hvcc = vec![0u8; 23]; + hvcc[0] = 1; // configurationVersion + hvcc[21] = 0x03; // lengthSizeMinusOne = 3 -> 4-byte prefixes + hvcc[22] = sets.len() as u8; + for (nal_type, nal) in sets { + hvcc.push(0x80 | nal_type); + hvcc.extend_from_slice(&1u16.to_be_bytes()); + hvcc.extend_from_slice(&(nal.len() as u16).to_be_bytes()); + hvcc.extend_from_slice(nal); + } + hvcc + } + + /// A minimal NAL with the given type in its two-byte header. + fn nal(nal_type: u8) -> Vec { + vec![nal_type << 1, 0x01, 0xAA, 0xBB] + } + + #[test] + fn hevc_requires_a_complete_parameter_set() { + // VPS + SPS but no PPS, and nothing in band to make up for it. + let vps = nal(32); + let sps = nal(33); + let hvcc = hvcc_with(&[(32, &vps), (33, &sps)]); + let err = hevc_format_description(&hvcc, &request(CodecConfig::Hvcc(&hvcc), &[])) + .expect_err("an incomplete parameter set must be rejected"); + assert!( + err.to_string().contains("VPS/SPS/PPS"), + "message must say what is missing: {err}" + ); + } + + #[test] + fn av1_rejects_profiles_outside_main() { + // av1C: marker/version, seq_profile = 1 in the top three bits. + let av1c = [0x81u8, 0x20, 0x0c, 0x00]; + let err = av1_format_description(&av1c, &request(CodecConfig::Av1c(&av1c), &[])) + .expect_err("profile 1 must be rejected"); + assert!( + err.to_string().contains("profile 1"), + "message must name the profile: {err}" + ); + } + + #[test] + fn av1_without_any_sequence_header_is_rejected() { + let av1c = [0x81u8, 0x00, 0x0c, 0x00]; + let err = av1_format_description(&av1c, &request(CodecConfig::Av1c(&av1c), &[])) + .expect_err("no sequence header anywhere must be an error"); + assert!( + err.to_string().contains("no sequence header"), + "message must say what is missing: {err}" + ); + } + + #[test] + fn leb128_matches_the_spec_encoding() { + let mut out = Vec::new(); + leb128(0, &mut out); + assert_eq!(out, [0x00]); + out.clear(); + leb128(127, &mut out); + assert_eq!(out, [0x7f]); + out.clear(); + leb128(128, &mut out); + assert_eq!(out, [0x80, 0x01]); + out.clear(); + leb128(300, &mut out); + assert_eq!(out, [0xac, 0x02]); + } + + #[test] + fn rebuilt_sequence_header_obu_round_trips() { + let payload = [0xAAu8; 13]; + let obu = sequence_header_obu(&payload); + // type 1, obu_has_size_field set, no extension. + assert_eq!(obu[0], 0x0A); + let parsed = av1::split_obus(&obu).expect("rebuilt OBU parses"); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].obu_type, OBU_SEQUENCE_HEADER); + assert_eq!(parsed[0].payload, &payload[..]); + } +} diff --git a/crates/rawshift-hwdec/src/videotoolbox/mod.rs b/crates/rawshift-hwdec/src/videotoolbox/mod.rs new file mode 100644 index 0000000..6a526a5 --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/mod.rs @@ -0,0 +1,613 @@ +//! VideoToolbox still-frame decode backend for macOS and iOS. +//! +//! ## Runtime model +//! +//! VideoToolbox, CoreMedia and CoreVideo are always-present Apple system +//! frameworks, so unlike the VAAPI backend there is nothing to `dlopen` and no +//! device node to find: availability is a pure capability question, answered +//! once per process by [`VTIsHardwareDecodeSupported`] for each codec. +//! +//! **Hardware only.** On macOS the session is created with +//! `kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder`, so a +//! machine without a decode block for the codec fails session creation instead +//! of silently falling back to VideoToolbox's *software* decoder — which would +//! make a crate called `rawshift-hwdec` quietly lie about what it did. That +//! key is macOS-only (see the `decoder_specification` docs); on iOS every +//! VideoToolbox decoder is hardware anyway. +//! +//! ## Codec scope +//! +//! - **HEVC** (HEIC) still pictures: `hvcC` parameter sets, length-prefixed +//! NAL payload passed through byte for byte — no Annex-B conversion. +//! - **AV1** (AVIF) Profile 0 still pictures: `av1C` config atom, raw OBU +//! temporal unit payload. Runtime-probed, because AV1 hardware decode +//! arrived with the M3 and A17 Pro generations — on older Apple silicon and +//! on Intel, [`available_codecs`] honestly omits [`HwCodec::Av1`] while HEVC +//! keeps working. +//! +//! Both emit 8-bit or 10-bit 4:2:0 → `Nv12` / `P010` (see [`pixels`]). +//! +//! ## Binding choice +//! +//! This backend uses the generated [`objc2`](https://github.com/madsmtm/objc2) +//! framework crates rather than a hand-written `sys.rs`, the opposite call +//! from the VAAPI backend. The reason is that the two situations differ: +//! libva must be `dlopen`'d so a machine without it degrades to "no decoder" +//! instead of failing to start, which rules out generated bindings; the Apple +//! frameworks are guaranteed present and link normally. Given that, the +//! maintained bindings win on every axis that matters here — `CFRetained` +//! gives us Create/Get-rule reference counting as RAII instead of hand-paired +//! `CFRetain`/`CFRelease`, and the `#[link(kind = "framework")]` attributes +//! live upstream. They are pulled in with `default-features = false` and only +//! the per-header features this file needs, so no Objective-C runtime, Metal, +//! OpenGL or CoreAudio code is compiled in. See `Cargo.toml`. +//! +//! ## Safety boundary +//! +//! All `unsafe` in this backend is FFI, plus the one `unsafe impl Send` on the +//! decoder. The bitstream parsers in [`crate::bitstream`] and the pixel +//! plumbing in [`pixels`] are safe Rust apart from the single documented slice +//! construction over a locked surface. + +mod format; +mod pixels; +mod sample; +mod status; + +use std::cell::UnsafeCell; +use std::ffi::c_void; +use std::ptr::{self, NonNull}; +use std::sync::OnceLock; + +use objc2_core_foundation::CFRetained; +use objc2_core_media::{CMFormatDescription, CMTime, kCMVideoCodecType_HEVC}; +use objc2_core_video::CVImageBuffer; +use objc2_video_toolbox::{ + VTDecodeFrameFlags, VTDecodeInfoFlags, VTDecompressionOutputCallbackRecord, + VTDecompressionSession, VTIsHardwareDecodeSupported, +}; + +use self::status::{NO_ERR, OSStatus, decode_err, map_status}; +use crate::{ + CodecConfig, DecodedFrame, HwBackend, HwCodec, HwDecodeError, HwStillDecoder, + StillDecodeRequest, +}; + +// ── Public backend surface (called from lib.rs) ───────────────────────────── + +/// Backend hook for [`crate::decoder`]. +pub fn decoder(codec: HwCodec) -> Option> { + let caps = probe(); + let supported = match codec { + HwCodec::Hevc => caps.hevc, + HwCodec::Av1 => caps.av1, + }; + if !supported { + return None; + } + // Nothing to acquire up front: the decompression session is created lazily + // on the first decode, from that picture's format description. This keeps + // `decoder(c).is_some()` exactly equal to `available_codecs().contains(&c)`, + // which `crate::tests::discovery_entry_points_are_consistent` requires. + Some(Box::new(VideoToolboxStillDecoder::new(codec))) +} + +/// Backend hook for [`crate::backend`]. +pub fn backend() -> Option { + let caps = probe(); + (caps.hevc || caps.av1).then_some(HwBackend::VideoToolbox) +} + +/// Backend hook for [`crate::available_codecs`]. +pub fn available_codecs() -> &'static [HwCodec] { + const NONE: &[HwCodec] = &[]; + const HEVC_ONLY: &[HwCodec] = &[HwCodec::Hevc]; + const AV1_ONLY: &[HwCodec] = &[HwCodec::Av1]; + const BOTH: &[HwCodec] = &[HwCodec::Hevc, HwCodec::Av1]; + let caps = probe(); + match (caps.hevc, caps.av1) { + (true, true) => BOTH, + (true, false) => HEVC_ONLY, + (false, true) => AV1_ONLY, + (false, false) => NONE, + } +} + +// ── Runtime probe ─────────────────────────────────────────────────────────── + +/// `kCMVideoCodecType_AV1` — `'av01'`. +/// +/// Named locally rather than imported so the constant is greppable next to its +/// HEVC peer; the value is the one `objc2-core-media` generates. +const CODEC_TYPE_AV1: u32 = objc2_core_media::kCMVideoCodecType_AV1; + +/// What this machine's VideoToolbox reports *hardware* decode support for. +#[derive(Debug, Clone, Copy, Default)] +struct Caps { + hevc: bool, + av1: bool, +} + +/// Probe once per process: hardware does not change at runtime. +fn probe() -> Caps { + static CAPS: OnceLock = OnceLock::new(); + *CAPS.get_or_init(|| { + // SAFETY: `VTIsHardwareDecodeSupported` takes a four-character codec + // type by value and only reads static system capability — no pointers, + // no session, no allocation. It is available since macOS 10.13 / + // iOS 11, both below this crate's macOS 11 / iOS 14 floor (see + // docs/SUPPORT.md), so the symbol is always present and needs no weak + // linking. An unrecognised codec type returns false rather than + // trapping. + unsafe { + Caps { + hevc: VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC), + av1: VTIsHardwareDecodeSupported(CODEC_TYPE_AV1), + } + } + }) +} + +/// The decoder specification handed to `VTDecompressionSessionCreate`. +/// +/// On macOS this pins the session to a hardware decoder: Apple documents +/// `RequireHardwareAcceleratedVideoDecoder` as making session creation *fail* +/// when hardware acceleration is not possible, which is what this crate wants +/// — a clean [`HwDecodeError::Unavailable`] instead of a silent software +/// decode. Because [`probe`] already gates on `VTIsHardwareDecodeSupported`, +/// it should never fire spuriously. +/// +/// On iOS it returns `None`, and that is **not** cosmetic: the +/// `kVTVideoDecoderSpecification_*` symbols are annotated `ios(17.0)` and the +/// binding is a plain non-weak `extern` static, so merely referencing one in a +/// build targeting iOS 14–16 would be a dyld "symbol not found" *launch* +/// failure for the whole app. iOS has no software VideoToolbox decoder to +/// guard against in any case. +#[cfg(target_os = "macos")] +fn decoder_specification() -> Option< + CFRetained< + objc2_core_foundation::CFDictionary< + objc2_core_foundation::CFString, + objc2_core_foundation::CFType, + >, + >, +> { + use objc2_core_foundation::{CFDictionary, CFType, kCFBooleanTrue}; + use objc2_video_toolbox::kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder; + + // SAFETY: both are immortal constant CF objects exported by VideoToolbox + // and CoreFoundation; reading them performs no allocation and they are + // valid for the life of the process. + let (key, value) = unsafe { + ( + kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder, + kCFBooleanTrue?, + ) + }; + Some(CFDictionary::from_slices( + &[key], + &[value.as_ref() as &CFType], + )) +} + +#[cfg(not(target_os = "macos"))] +fn decoder_specification() -> Option< + CFRetained< + objc2_core_foundation::CFDictionary< + objc2_core_foundation::CFString, + objc2_core_foundation::CFType, + >, + >, +> { + None +} + +// ── Output callback ───────────────────────────────────────────────────────── + +/// Where the decompression callback deposits its result. +/// +/// Heap-allocated and owned by the decoder so its address stays stable for the +/// life of the session that was created with a pointer to it. +struct DecodeSlot { + status: OSStatus, + info: VTDecodeInfoFlags, + image: Option>, +} + +impl Default for DecodeSlot { + fn default() -> Self { + Self { + status: NO_ERR, + info: VTDecodeInfoFlags::empty(), + image: None, + } + } +} + +/// `VTDecompressionOutputCallback`: called once per decoded frame. +/// +/// This runs on the calling thread inside `VTDecompressionSessionDecodeFrame` +/// because the decode is submitted with neither +/// `kVTDecodeFrame_EnableAsynchronousDecompression` nor +/// `kVTDecodeFrame_EnableTemporalProcessing`. +/// +/// The body must never unwind across the `extern "C-unwind"` boundary, so it +/// deliberately does no allocation beyond a single `CFRetain`, no formatting, +/// no indexing and no `unwrap`. +unsafe extern "C-unwind" fn output_callback( + output_ref_con: *mut c_void, + _source_ref_con: *mut c_void, + status: OSStatus, + info: VTDecodeInfoFlags, + image: *mut CVImageBuffer, + _pts: CMTime, + _duration: CMTime, +) { + // SAFETY: `output_ref_con` is the `DecodeSlot` pointer this decoder + // registered in its `VTDecompressionOutputCallbackRecord`, and the slot's + // heap allocation outlives the session (field order on + // `VideoToolboxStillDecoder` drops the session first). The decode is + // synchronous under `&mut self`, so this callback runs on the calling + // thread inside `decode_frame` and no other reference to the slot is live. + // `image` is a borrowed +0 buffer valid only for this call, so + // `CFRetained::retain` takes the +1 needed to outlive it. + unsafe { + let slot = &mut *(output_ref_con.cast::()); + slot.status = status; + slot.info = info; + slot.image = NonNull::new(image).map(|p| CFRetained::retain(p)); + } +} + +// ── Session ───────────────────────────────────────────────────────────────── + +/// A live decompression session and the format description it was created +/// from. +struct Session { + /// The configuration record bytes this session was built for. Grid HEICs + /// decode hundreds of tiles that share one `hvcC`, so a byte compare here + /// skips both session *and* format-description construction. + config: Vec, + format: CFRetained, + session: CFRetained, +} + +impl Drop for Session { + fn drop(&mut self) { + // SAFETY: `session` was created successfully by this decoder and is + // invalidated exactly once, here. VideoToolbox documents Invalidate as + // the deterministic teardown to call before the last release, which + // the `CFRetained` field performs immediately afterwards. + unsafe { self.session.invalidate() }; + } +} + +// ── Decoder ───────────────────────────────────────────────────────────────── + +struct VideoToolboxStillDecoder { + codec: HwCodec, + /// Declared before `slot` so it is dropped first: invalidating the session + /// severs the callback's reference to the slot before the slot's + /// allocation goes away. **Reordering these two fields is a use-after-free.** + session: Option, + slot: Box>, +} + +// SAFETY: the decoder owns its decompression session, its format description +// and its callback slot exclusively — `decode_still` is the only entry point +// and takes `&mut self`, so no two threads can touch them at once. +// VideoToolbox sessions are not thread-affine: they require no run loop and no +// main thread, only that calls be externally serialised, which exclusive +// ownership guarantees. The output callback runs synchronously on the calling +// thread inside `decode_frame` (neither the asynchronous nor the +// temporal-processing flag is set, and every decode is followed by +// `wait_for_asynchronous_frames`), so the raw slot pointer is never +// dereferenced from another thread. `Send` is required by the +// [`HwStillDecoder`] contract; `Sync` is deliberately not claimed. +unsafe impl Send for VideoToolboxStillDecoder {} + +impl VideoToolboxStillDecoder { + fn new(codec: HwCodec) -> Self { + Self { + codec, + session: None, + slot: Box::new(UnsafeCell::new(DecodeSlot::default())), + } + } + + /// Get a session that can decode `format`, reusing the cached one when it + /// can accept the picture. + fn session_for( + &mut self, + config_bytes: &[u8], + format: CFRetained, + ) -> Result<&Session, HwDecodeError> { + let reusable = match self.session.as_ref() { + // Same configuration record: the fast path a grid HEIC or an AVIF + // batch takes on every tile after the first. + Some(existing) if existing.config == config_bytes => true, + // Different record, but some decoders accommodate minor format + // changes without a new session. + Some(existing) => { + // SAFETY: `existing.session` is a live session created by this + // decoder and `format` a live description; the call only + // queries the decoder and mutates nothing. + unsafe { existing.session.can_accept_format_description(&format) } + } + None => false, + }; + + if reusable { + let existing = self.session.as_mut().expect("checked above"); + existing.config = config_bytes.to_vec(); + existing.format = format; + return Ok(self.session.as_ref().expect("just set")); + } + + // Drop any old session before creating a new one, so two sessions + // never hold decode resources at the same time. + self.session = None; + + let specification = decoder_specification(); + let callback = VTDecompressionOutputCallbackRecord { + decompressionOutputCallback: Some(output_callback), + decompressionOutputRefCon: self.slot.get().cast::(), + }; + + let destination = pixels::destination_attributes(); + + let create_session = |spec: Option<&objc2_core_foundation::CFDictionary>| { + let mut raw: *mut VTDecompressionSession = ptr::null_mut(); + // SAFETY: `format` is a live video format description. The callback + // record is a live local that the framework copies during the + // call, and its refcon points at this decoder's heap-allocated + // slot, which outlives the session (see the field-order note on + // the struct). `spec` and `destination` are live dictionaries of + // CF types. `raw` is a live local the framework writes exactly one + // +1 reference into on success. + let status = unsafe { + VTDecompressionSession::create( + None, + &format, + spec, + Some(format::as_opaque_dict(&destination)), + &raw const callback, + NonNull::from(&mut raw), + ) + }; + (status, raw) + }; + + // Hardware-pinned first (macOS; on iOS the specification is None and + // there is nothing to retry). + let (mut status, mut raw) = + create_session(specification.as_deref().map(format::as_opaque_dict)); + + // Apple's hardware HEVC block refuses pictures below roughly 64x64, + // failing session creation with `kVTVideoDecoderMalfunctionErr`. Small + // items are entirely ordinary in HEIF — a `thmb` thumbnail is + // routinely 32x32 — so refusing them outright would be a worse answer + // than letting VideoToolbox use its own decoder for that one picture. + // A failed hardware-pinned create is therefore retried once without + // the requirement. + // + // This never manufactures support for a codec the machine cannot + // decode: availability is still gated on `VTIsHardwareDecodeSupported`, + // so the retry only widens the picture *sizes* an already-available + // codec accepts. If it fails too, the original hardware status is what + // gets reported, because that is the one that describes the real + // problem. + if status != NO_ERR && specification.is_some() { + let hardware_status = status; + let (retry_status, retry_raw) = create_session(None); + if retry_status == NO_ERR { + status = retry_status; + raw = retry_raw; + } else { + return Err(map_status( + self.codec, + "VTDecompressionSessionCreate", + hardware_status, + )); + } + } + if status != NO_ERR { + return Err(map_status( + self.codec, + "VTDecompressionSessionCreate", + status, + )); + } + let session_ptr = NonNull::new(raw).ok_or_else(|| { + decode_err( + self.codec, + "VTDecompressionSessionCreate returned no session", + ) + })?; + // SAFETY: the call succeeded, so `session_ptr` carries the +1 + // reference the Create rule granted; `from_raw` adopts it and the + // `Drop` impl invalidates then releases it exactly once. + let session = unsafe { CFRetained::from_raw(session_ptr) }; + + self.session = Some(Session { + config: config_bytes.to_vec(), + format, + session, + }); + Ok(self.session.as_ref().expect("just set")) + } + + /// Submit one coded picture and copy the decoded surface out. + fn decode( + &mut self, + config_bytes: &[u8], + format: CFRetained, + payload: &[u8], + ) -> Result { + let codec = self.codec; + + // Reset the slot before submitting so a decoder that never calls back + // cannot leave us reading the previous picture. + // SAFETY: `&mut self` is exclusive and the callback has not run yet. + unsafe { *self.slot.get() = DecodeSlot::default() }; + + let session = self.session_for(config_bytes, format)?; + let sample = sample::wrap(codec, &session.format, payload)?; + + // SAFETY: `session` was created with this decoder's callback record, + // and the sample buffer is ready and described by the very format + // description the session holds. A null source refcon and a null + // info-flags out-pointer are both explicitly permitted. No + // asynchronous or temporal-processing flag is set, so the output + // callback runs before this returns. + let status = unsafe { + session.session.decode_frame( + &sample.buffer, + VTDecodeFrameFlags::empty(), + ptr::null_mut(), + ptr::null_mut(), + ) + }; + if status != NO_ERR { + return Err(map_status( + codec, + "VTDecompressionSessionDecodeFrame", + status, + )); + } + + // A no-op for a synchronous decode, and a hard barrier for a decoder + // that went asynchronous anyway. This is what makes the zero-copy + // block buffer in `sample::wrap` sound. + // SAFETY: `session` is live and owned by this decoder. + let wait = unsafe { session.session.wait_for_asynchronous_frames() }; + if wait != NO_ERR { + return Err(map_status( + codec, + "VTDecompressionSessionWaitForAsynchronousFrames", + wait, + )); + } + + // SAFETY: `&mut self` is exclusive and, per the flags above, the + // callback has finished running on this thread. + let slot = unsafe { &mut *self.slot.get() }; + if slot.status != NO_ERR { + return Err(map_status( + codec, + "VideoToolbox decode callback", + slot.status, + )); + } + if slot + .info + .intersects(VTDecodeInfoFlags::FrameDropped | VTDecodeInfoFlags::FrameInterrupted) + { + return Err(decode_err( + codec, + "VideoToolbox dropped or interrupted the frame without producing pixels", + )); + } + let image = slot + .image + .take() + .ok_or_else(|| decode_err(codec, "VideoToolbox produced no image buffer"))?; + + pixels::to_decoded_frame(codec, &image) + } +} + +impl HwStillDecoder for VideoToolboxStillDecoder { + fn decode_still( + &mut self, + request: &StillDecodeRequest<'_>, + ) -> Result { + if request.codec() != self.codec { + return Err(HwDecodeError::Decode { + codec: request.codec(), + message: format!("decoder was opened for {}", self.codec), + }); + } + // Cheapest check first: without a payload there is nothing to describe + // a format for, let alone decode. + if request.payload.is_empty() { + return Err(decode_err(self.codec, "coded picture payload is empty")); + } + match request.config { + CodecConfig::Hvcc(hvcc) => { + let format = format::hevc_format_description(hvcc, request)?; + self.decode(hvcc, format, request.payload) + } + CodecConfig::Av1c(av1c) => { + let format = format::av1_format_description(av1c, request)?; + self.decode(av1c, format, request.payload) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The probe must never panic and must agree with itself across calls, + /// on any Apple machine including one with no decode block at all. + #[test] + fn probe_is_infallible_and_cached() { + let first = probe(); + let second = probe(); + assert_eq!(first.hevc, second.hevc); + assert_eq!(first.av1, second.av1); + } + + /// The invariant `crate::tests::discovery_entry_points_are_consistent` + /// enforces, checked against this backend's own `Caps` so a future edit to + /// one of the three hooks cannot drift from the others. + #[test] + fn discovery_hooks_agree_with_caps() { + let caps = probe(); + let codecs = available_codecs(); + assert_eq!(codecs.contains(&HwCodec::Hevc), caps.hevc); + assert_eq!(codecs.contains(&HwCodec::Av1), caps.av1); + assert_eq!(backend().is_some(), !codecs.is_empty()); + assert_eq!(decoder(HwCodec::Hevc).is_some(), caps.hevc); + assert_eq!(decoder(HwCodec::Av1).is_some(), caps.av1); + } + + /// A decoder opened for one codec must reject the other rather than + /// building a format description for a session that cannot take it. + #[test] + fn decoder_rejects_the_other_codec() { + let mut decoder = VideoToolboxStillDecoder::new(HwCodec::Hevc); + let request = StillDecodeRequest { + config: CodecConfig::Av1c(&[0x81, 0x00, 0x0c, 0x00]), + payload: &[0x12, 0x00], + width: 64, + height: 64, + bit_depth: 8, + chroma: crate::ChromaSubsampling::Cs420, + }; + let err = decoder + .decode_still(&request) + .expect_err("codec mismatch must be rejected"); + assert!(err.to_string().contains("opened for HEVC"), "{err}"); + } + + /// An empty payload must be refused before any framework call. + #[test] + fn empty_payload_is_rejected() { + let mut decoder = VideoToolboxStillDecoder::new(HwCodec::Av1); + let request = StillDecodeRequest { + config: CodecConfig::Av1c(&[0x81, 0x00, 0x0c, 0x00]), + payload: &[], + width: 64, + height: 64, + bit_depth: 8, + chroma: crate::ChromaSubsampling::Cs420, + }; + let err = decoder + .decode_still(&request) + .expect_err("an empty payload must be rejected"); + assert!(err.to_string().contains("empty"), "{err}"); + } +} diff --git a/crates/rawshift-hwdec/src/videotoolbox/pixels.rs b/crates/rawshift-hwdec/src/videotoolbox/pixels.rs new file mode 100644 index 0000000..356723f --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/pixels.rs @@ -0,0 +1,406 @@ +//! `CVPixelBuffer` → [`DecodedFrame`]. +//! +//! VideoToolbox hands back the decoder's native surface (this backend passes +//! no destination pixel-buffer attributes, so no conversion is imposed). The +//! surface's `OSType` is therefore the ground truth for pixel layout, colour +//! range and bit depth, and [`describe_format`] is the single table that +//! interprets it. +//! +//! ## Geometry +//! +//! `CVPixelBufferGetWidth`/`GetHeight` report the **clean** aperture — the +//! conformance window is already applied — so, unlike the VAAPI backend, this +//! one does no cropping arithmetic. `videotoolbox_crops_to_the_conformance_window` +//! in `tests/videotoolbox_device.rs` pins that assumption to real hardware. +//! +//! ## 10-bit layout +//! +//! CoreVideo's `x420`/`xf20` are documented in `CVPixelBuffer.h` as "2 plane +//! YCbCr10 4:2:0, each 10 bits in the **MSBs** of 16bits". That is exactly +//! [`PixelFormat::P010`]'s contract, so the copy is a plain per-row `memcpy` +//! with no shifting or byte-swapping (every Apple target is little-endian). +//! Do not "fix" this to shift right. +//! +//! [`PixelFormat::I010`] is unreachable here: CoreVideo has no planar 10-bit +//! 4:2:0 type. Its absence from the table below is deliberate. + +use objc2_core_foundation::{CFArray, CFDictionary, CFNumber, CFRetained, CFString, CFType}; +use objc2_core_video::{ + CVPixelBuffer, CVPixelBufferGetBaseAddressOfPlane, CVPixelBufferGetBytesPerRowOfPlane, + CVPixelBufferGetHeight, CVPixelBufferGetHeightOfPlane, CVPixelBufferGetPixelFormatType, + CVPixelBufferGetPlaneCount, CVPixelBufferGetWidth, CVPixelBufferLockBaseAddress, + CVPixelBufferLockFlags, CVPixelBufferUnlockBaseAddress, kCVPixelBufferPixelFormatTypeKey, + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8Planar, + kCVPixelFormatType_420YpCbCr8PlanarFullRange, kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, kCVReturnSuccess, +}; + +use super::status::{OSType, decode_err}; +use crate::{ColorRange, DecodedFrame, HwCodec, HwDecodeError, PixelFormat, Plane}; + +/// Read-only lock: we copy the planes out and never write to the surface. +const READ_ONLY: CVPixelBufferLockFlags = CVPixelBufferLockFlags::ReadOnly; + +// CoreVideo's constants keep Apple's C spelling, which is not a legal Rust +// pattern name. Alias them once here so the table below reads idiomatically +// and each four-character code sits next to the constant it belongs to. + +/// `'420v'` — 8-bit biplanar 4:2:0, video range (luma 16..235). +const NV12_VIDEO_RANGE: OSType = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; +/// `'420f'` — 8-bit biplanar 4:2:0, full range. +const NV12_FULL_RANGE: OSType = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; +/// `'x420'` — 10-bit biplanar 4:2:0, video range, sample in the MSBs. +const P010_VIDEO_RANGE: OSType = kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange; +/// `'xf20'` — 10-bit biplanar 4:2:0, full range, sample in the MSBs. +const P010_FULL_RANGE: OSType = kCVPixelFormatType_420YpCbCr10BiPlanarFullRange; +/// `'y420'` — 8-bit planar 4:2:0, video range. +const I420_VIDEO_RANGE: OSType = kCVPixelFormatType_420YpCbCr8Planar; +/// `'f420'` — 8-bit planar 4:2:0, full range. +const I420_FULL_RANGE: OSType = kCVPixelFormatType_420YpCbCr8PlanarFullRange; + +/// The pixel formats a decompression session is asked to produce. +/// +/// **Why ask at all.** Left to itself, VideoToolbox hands back the decoder's +/// native surface, and on Apple silicon a 10-bit HEVC decode natively produces +/// `'p420'` — a format that appears in no public CoreVideo header, whose plane +/// contents demonstrably do **not** match the documented `x420` samples for +/// the same bitstream. There is no safe way to interpret it. Offering an +/// explicit list keeps the decoder on documented, linear layouts. +/// +/// **Why a list rather than one format.** All four documented 4:2:0 surfaces +/// are offered — both depths and both ranges — so VideoToolbox still picks the +/// one that matches the stream instead of converting to a format we imposed. +/// That is what preserves the video/full range distinction, which +/// [`describe_format`] then reads back off the chosen surface. +pub(super) fn destination_attributes() -> CFRetained> { + let formats = [ + NV12_VIDEO_RANGE, + NV12_FULL_RANGE, + P010_VIDEO_RANGE, + P010_FULL_RANGE, + ] + .map(|fourcc| CFNumber::new_i32(fourcc as i32)); + let refs: Vec<&CFNumber> = formats.iter().map(|n| &**n).collect(); + let list = CFArray::from_objects(&refs); + // SAFETY: an immortal constant `CFStringRef` exported by CoreVideo. + let key = unsafe { kCVPixelBufferPixelFormatTypeKey }; + CFDictionary::from_slices(&[key], &[list.as_ref() as &CFType]) +} + +/// What a CoreVideo pixel format means in this crate's vocabulary. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct SurfaceFormat { + pub(super) format: PixelFormat, + pub(super) range: ColorRange, + pub(super) bit_depth: u8, +} + +/// The 4:2:0 surfaces a VideoToolbox HEVC/AV1 still decode can emit. +/// +/// Anything else — 4:2:2/4:4:4, and the lossless (`&8v0`) / lossy (`-8v0`) +/// packed families, which have no CPU-addressable plane layout — is rejected +/// by name rather than misread. +pub(super) fn describe_format(fourcc: OSType) -> Option { + let described = match fourcc { + // '420v' — 8-bit biplanar, video range (luma 16..235). + NV12_VIDEO_RANGE => SurfaceFormat { + format: PixelFormat::Nv12, + range: ColorRange::Limited, + bit_depth: 8, + }, + // '420f' — 8-bit biplanar, full range. + NV12_FULL_RANGE => SurfaceFormat { + format: PixelFormat::Nv12, + range: ColorRange::Full, + bit_depth: 8, + }, + // 'x420' — 10-bit biplanar, video range, sample in the MSBs. + P010_VIDEO_RANGE => SurfaceFormat { + format: PixelFormat::P010, + range: ColorRange::Limited, + bit_depth: 10, + }, + // 'xf20' — 10-bit biplanar, full range, sample in the MSBs. + P010_FULL_RANGE => SurfaceFormat { + format: PixelFormat::P010, + range: ColorRange::Full, + bit_depth: 10, + }, + // 'y420' — 8-bit planar, video range. + I420_VIDEO_RANGE => SurfaceFormat { + format: PixelFormat::I420, + range: ColorRange::Limited, + bit_depth: 8, + }, + // 'f420' — 8-bit planar, full range. + I420_FULL_RANGE => SurfaceFormat { + format: PixelFormat::I420, + range: ColorRange::Full, + bit_depth: 8, + }, + _ => return None, + }; + Some(described) +} + +/// Render a four-character code the way the CoreVideo headers write it. +pub(super) fn fourcc_name(fourcc: OSType) -> String { + let bytes = fourcc.to_be_bytes(); + if bytes.iter().all(|b| (0x20..=0x7e).contains(b)) { + format!("'{}'", bytes.iter().map(|&b| b as char).collect::()) + } else { + format!("{fourcc:#010x}") + } +} + +/// The per-plane byte geometry [`DecodedFrame::new`] validates against. +/// +/// Derived from the frame dimensions with the same ceiling rules the +/// constructor uses, never from `CVPixelBufferGetWidthOfPlane`, so +/// construction cannot disagree with validation. +fn plane_geometry(format: PixelFormat, width: u32, height: u32) -> Vec<(usize, usize)> { + let bps = format.bytes_per_sample(); + let (cw, ch) = (width.div_ceil(2) as usize, height.div_ceil(2) as usize); + let luma = (width as usize * bps, height as usize); + match format.plane_count() { + // Biplanar: full-size Y, then half-size interleaved CbCr (two samples + // per chroma column). + 2 => vec![luma, (cw * 2 * bps, ch)], + // Planar: full-size Y, then half-size Cb and Cr. + _ => vec![luma, (cw * bps, ch), (cw * bps, ch)], + } +} + +/// Copy one plane into a tightly packed [`Plane`] (`stride == row_bytes`). +/// +/// Every offset is checked: a surface whose reported geometry does not cover +/// the rows we need yields `None` rather than reading out of bounds. +fn copy_plane(base: *const u8, stride: usize, row_bytes: usize, rows: usize) -> Option { + if base.is_null() || rows == 0 || stride < row_bytes { + return None; + } + // The last row needs only `row_bytes`, not a full stride — reading a whole + // stride past the final row would run off the end of the mapping. + let span = stride + .checked_mul(rows.checked_sub(1)?)? + .checked_add(row_bytes)?; + // SAFETY: the pixel buffer is locked read-only by the caller for the whole + // of this call, so CoreVideo guarantees `bytesPerRow * (heightOfPlane - 1) + // + rowBytes` initialised bytes at the plane base address. The caller has + // checked `rows <= heightOfPlane` and `row_bytes <= stride`, so `span` is + // within that guarantee. The slice is immutable, is not aliased by any + // other Rust reference, and dies before the matching unlock. + let src = unsafe { std::slice::from_raw_parts(base, span) }; + let mut data = Vec::with_capacity(row_bytes.checked_mul(rows)?); + for row in 0..rows { + let start = stride.checked_mul(row)?; + data.extend_from_slice(src.get(start..start.checked_add(row_bytes)?)?); + } + Some(Plane { + data, + stride: row_bytes, + }) +} + +/// Lock the surface, copy every plane out, unlock, and build the frame. +pub(super) fn to_decoded_frame( + codec: HwCodec, + pixel_buffer: &CVPixelBuffer, +) -> Result { + let fourcc = CVPixelBufferGetPixelFormatType(pixel_buffer); + let described = describe_format(fourcc).ok_or_else(|| { + decode_err( + codec, + format!( + "VideoToolbox emitted unsupported pixel format {} \ + (this backend handles 8/10-bit 4:2:0 only)", + fourcc_name(fourcc) + ), + ) + })?; + + let width = u32::try_from(CVPixelBufferGetWidth(pixel_buffer)) + .map_err(|_| decode_err(codec, "decoded surface width does not fit in u32"))?; + let height = u32::try_from(CVPixelBufferGetHeight(pixel_buffer)) + .map_err(|_| decode_err(codec, "decoded surface height does not fit in u32"))?; + + let geometry = plane_geometry(described.format, width, height); + let plane_count = CVPixelBufferGetPlaneCount(pixel_buffer); + if plane_count != geometry.len() { + return Err(decode_err( + codec, + format!( + "decoded surface {} reports {plane_count} planes, expected {}", + fourcc_name(fourcc), + geometry.len() + ), + )); + } + + // SAFETY: `pixel_buffer` is a live, retained surface handed to us by the + // decompression callback. Locking read-only makes the planes CPU- + // addressable; it is paired with exactly one unlock below, which runs on + // every path because the copy is done inside a closure whose result is + // only returned after unlocking. + let lock = unsafe { CVPixelBufferLockBaseAddress(pixel_buffer, READ_ONLY) }; + if lock != kCVReturnSuccess { + return Err(decode_err( + codec, + format!("CVPixelBufferLockBaseAddress failed (CVReturn {lock})"), + )); + } + + let result = (|| -> Result { + let mut planes = Vec::with_capacity(geometry.len()); + for (index, &(row_bytes, rows)) in geometry.iter().enumerate() { + let stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, index); + let plane_rows = CVPixelBufferGetHeightOfPlane(pixel_buffer, index); + if stride < row_bytes || plane_rows < rows { + return Err(decode_err( + codec, + format!( + "decoded surface plane {index} is {stride}x{plane_rows}, \ + too small for {row_bytes}x{rows}" + ), + )); + } + let base = CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, index).cast::(); + let plane = copy_plane(base, stride, row_bytes, rows).ok_or_else(|| { + decode_err( + codec, + format!("decoded surface plane {index} is not readable"), + ) + })?; + planes.push(plane); + } + DecodedFrame::new( + described.format, + width, + height, + described.bit_depth, + described.range, + planes, + ) + })(); + + // SAFETY: the surface was locked read-only immediately above with these + // exact flags and is unlocked exactly once here. Every slice borrowed from + // it is dead — the planes were copied into owned `Vec`s. + let _ = unsafe { CVPixelBufferUnlockBaseAddress(pixel_buffer, READ_ONLY) }; + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fourcc_table_maps_the_six_supported_surfaces() { + let cases = [ + (0x3432_3076_u32, PixelFormat::Nv12, ColorRange::Limited, 8), // '420v' + (0x3432_3066, PixelFormat::Nv12, ColorRange::Full, 8), // '420f' + (0x7834_3230, PixelFormat::P010, ColorRange::Limited, 10), // 'x420' + (0x7866_3230, PixelFormat::P010, ColorRange::Full, 10), // 'xf20' + (0x7934_3230, PixelFormat::I420, ColorRange::Limited, 8), // 'y420' + (0x6634_3230, PixelFormat::I420, ColorRange::Full, 8), // 'f420' + ]; + for (fourcc, format, range, bit_depth) in cases { + let described = describe_format(fourcc) + .unwrap_or_else(|| panic!("{} must be supported", fourcc_name(fourcc))); + assert_eq!(described.format, format, "{}", fourcc_name(fourcc)); + assert_eq!(described.range, range, "{}", fourcc_name(fourcc)); + assert_eq!(described.bit_depth, bit_depth, "{}", fourcc_name(fourcc)); + // The table must agree with the crate's own depth rules. + assert!(format.supports_bit_depth(bit_depth)); + } + } + + #[test] + fn unsupported_surfaces_are_rejected() { + for fourcc in [ + 0x2638_7630_u32, // '&8v0' lossless biplanar — no readable layout + 0x2d38_7630, // '-8v0' lossy biplanar + 0x7834_3232, // 'x422' 4:2:2 10-bit + 0x4241_4752, // 'BAGR' + 0, + ] { + assert!(describe_format(fourcc).is_none(), "{fourcc:#010x}"); + } + } + + #[test] + fn fourcc_name_renders_ascii_and_falls_back_to_hex() { + assert_eq!(fourcc_name(0x7834_3230), "'x420'"); + assert_eq!(fourcc_name(0x3432_3076), "'420v'"); + assert_eq!(fourcc_name(0), "0x00000000"); + } + + /// The geometry table must satisfy `DecodedFrame::new` exactly, including + /// the ceiling division for odd dimensions. + #[test] + fn plane_geometry_satisfies_decoded_frame_validation() { + for (format, bit_depth) in [ + (PixelFormat::Nv12, 8), + (PixelFormat::P010, 10), + (PixelFormat::I420, 8), + ] { + for (width, height) in [(5u32, 3u32), (64, 64), (61, 37), (1, 1)] { + let geometry = plane_geometry(format, width, height); + assert_eq!(geometry.len(), format.plane_count()); + let planes = geometry + .iter() + .map(|&(row_bytes, rows)| Plane { + data: vec![0u8; row_bytes * rows], + stride: row_bytes, + }) + .collect(); + DecodedFrame::new( + format, + width, + height, + bit_depth, + ColorRange::Limited, + planes, + ) + .unwrap_or_else(|e| panic!("{format:?} {width}x{height} must validate: {e}")); + } + } + } + + #[test] + fn copy_plane_tightens_a_padded_stride() { + // 4x2 luma in a buffer with an 8-byte stride: the padding must not + // survive into the returned plane. + let src: Vec = vec![ + 1, 2, 3, 4, 0xFF, 0xFF, 0xFF, 0xFF, // row 0 + padding + 5, 6, 7, 8, 0xFF, 0xFF, 0xFF, 0xFF, // row 1 + padding + ]; + let plane = copy_plane(src.as_ptr(), 8, 4, 2).expect("plane copies"); + assert_eq!(plane.stride, 4); + assert_eq!(plane.data, vec![1, 2, 3, 4, 5, 6, 7, 8]); + } + + #[test] + fn copy_plane_reads_only_row_bytes_from_the_final_row() { + // Exactly `stride * (rows - 1) + row_bytes` bytes are available — one + // byte less than `stride * rows`. A naive implementation would read + // past the end here. + let src: Vec = vec![1, 2, 3, 0xFF, 4, 5, 6]; + let plane = copy_plane(src.as_ptr(), 4, 3, 2).expect("plane copies"); + assert_eq!(plane.data, vec![1, 2, 3, 4, 5, 6]); + } + + #[test] + fn copy_plane_rejects_bad_geometry() { + let src = [0u8; 64]; + // Stride narrower than the used row width. + assert!(copy_plane(src.as_ptr(), 2, 4, 2).is_none()); + // Zero rows. + assert!(copy_plane(src.as_ptr(), 4, 4, 0).is_none()); + // Null base (a plane CoreVideo could not make CPU-addressable). + assert!(copy_plane(std::ptr::null(), 4, 4, 2).is_none()); + // Overflowing geometry must not wrap. + assert!(copy_plane(src.as_ptr(), usize::MAX, 4, 4).is_none()); + } +} diff --git a/crates/rawshift-hwdec/src/videotoolbox/sample.rs b/crates/rawshift-hwdec/src/videotoolbox/sample.rs new file mode 100644 index 0000000..4d963a2 --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/sample.rs @@ -0,0 +1,155 @@ +//! Wrapping a coded picture payload in the `CMSampleBuffer` a decompression +//! session consumes. +//! +//! The payload is wrapped **zero-copy**: `CMBlockBufferCreateWithMemoryBlock` +//! is given `kCFAllocatorNull` as the block allocator, so CoreMedia never +//! takes ownership of or frees the caller's bytes. That is sound because the +//! lifetime is bounded on both ends — the decode is synchronous (no +//! asynchronous or temporal-processing flags, plus an explicit +//! `VTDecompressionSessionWaitForAsynchronousFrames`), and the sample buffer +//! is released before `decode_still` returns. +//! +//! Both codecs share this path unchanged: HEVC payloads are length-prefixed +//! NAL units and AV1 payloads are raw OBU temporal units, and in each case the +//! bytes go to the decoder exactly as the container stored them. + +use std::ffi::c_void; +use std::ptr::{self, NonNull}; + +use objc2_core_foundation::{CFRetained, kCFAllocatorNull}; +use objc2_core_media::{ + CMBlockBuffer, CMFormatDescription, CMSampleBuffer, CMSampleTimingInfo, CMTime, CMTimeFlags, +}; + +use super::status::{NO_ERR, decode_err, map_status}; +use crate::{HwCodec, HwDecodeError}; + +/// Presentation timestamp 0/1. A *valid* PTS matters: some decoders drop a +/// frame whose PTS is `kCMTimeInvalid`. +const PTS_ZERO: CMTime = CMTime { + value: 0, + timescale: 1, + flags: CMTimeFlags::Valid, + epoch: 0, +}; + +/// `kCMTimeInvalid`: a still has no duration, and its samples are already in +/// presentation order so no decode timestamp is meaningful. +const TIME_INVALID: CMTime = CMTime { + value: 0, + timescale: 0, + flags: CMTimeFlags::empty(), + epoch: 0, +}; + +/// A ready sample buffer and the block buffer it reads through. +/// +/// Field order is load-bearing: `buffer` is declared before `_block` so the +/// sample buffer is released first, and the block buffer it references +/// outlives it. +pub(super) struct Sample { + pub(super) buffer: CFRetained, + _block: CFRetained, +} + +/// Wrap `payload` in a one-sample `CMSampleBuffer` described by `format`. +pub(super) fn wrap( + codec: HwCodec, + format: &CMFormatDescription, + payload: &[u8], +) -> Result { + if payload.is_empty() { + return Err(decode_err(codec, "coded picture payload is empty")); + } + + let mut raw_block: *mut CMBlockBuffer = ptr::null_mut(); + // SAFETY: `payload` is a live immutable slice for the whole of this call + // and of the decode that consumes the returned sample. `kCFAllocatorNull` + // tells CoreMedia never to free those bytes, and the decoder only reads + // them. The `*const -> *mut` cast is provenance-only: nothing writes + // through the pointer. `custom_block_source` is null, which the API + // permits when a memory block is supplied, and `raw_block` is a live local + // the framework writes exactly one +1 reference into. + let status = unsafe { + CMBlockBuffer::create_with_memory_block( + None, + payload.as_ptr().cast_mut().cast::(), + payload.len(), + kCFAllocatorNull, + ptr::null(), + 0, + payload.len(), + 0, + NonNull::from(&mut raw_block), + ) + }; + if status != NO_ERR { + return Err(map_status( + codec, + "CMBlockBufferCreateWithMemoryBlock", + status, + )); + } + let block_ptr = NonNull::new(raw_block).ok_or_else(|| { + decode_err( + codec, + "CMBlockBufferCreateWithMemoryBlock returned no buffer", + ) + })?; + // SAFETY: the call succeeded, so `block_ptr` carries the +1 reference the + // Create rule granted; `from_raw` adopts it and releases it once on drop. + let block = unsafe { CFRetained::from_raw(block_ptr) }; + + let timing = CMSampleTimingInfo { + duration: TIME_INVALID, + presentationTimeStamp: PTS_ZERO, + decodeTimeStamp: TIME_INVALID, + }; + let sample_size = payload.len(); + + let mut raw_sample: *mut CMSampleBuffer = ptr::null_mut(); + // SAFETY: one sample spanning the whole block buffer, so the timing and + // size arrays each have exactly the single entry their count declares and + // both outlive the call (the framework copies them). `block` and `format` + // are live. `raw_sample` is a live local the framework writes exactly one + // +1 reference into. + let status = unsafe { + CMSampleBuffer::create_ready( + None, + Some(&block), + Some(format), + 1, + 1, + &raw const timing, + 1, + &raw const sample_size, + NonNull::from(&mut raw_sample), + ) + }; + if status != NO_ERR { + return Err(map_status(codec, "CMSampleBufferCreateReady", status)); + } + let sample_ptr = NonNull::new(raw_sample) + .ok_or_else(|| decode_err(codec, "CMSampleBufferCreateReady returned no buffer"))?; + // SAFETY: as above — adopt the Create-rule +1 reference. + let buffer = unsafe { CFRetained::from_raw(sample_ptr) }; + + Ok(Sample { + buffer, + _block: block, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A still carries no duration and no decode timestamp, but its + /// presentation timestamp must be *valid* or decoders may drop the frame. + #[test] + fn still_timing_has_a_valid_pts_and_invalid_duration() { + assert!(PTS_ZERO.flags.contains(CMTimeFlags::Valid)); + assert_eq!(PTS_ZERO.timescale, 1); + assert!(!TIME_INVALID.flags.contains(CMTimeFlags::Valid)); + } +} diff --git a/crates/rawshift-hwdec/src/videotoolbox/status.rs b/crates/rawshift-hwdec/src/videotoolbox/status.rs new file mode 100644 index 0000000..503bec9 --- /dev/null +++ b/crates/rawshift-hwdec/src/videotoolbox/status.rs @@ -0,0 +1,308 @@ +//! `OSStatus` naming and the mapping from platform status codes to +//! [`HwDecodeError`]. +//! +//! **Safe Rust only** — this module never calls into a framework; it just +//! interprets the integers they return. +//! +//! The `MacTypes.h` aliases are re-declared here because the objc2 crates +//! keep theirs `pub(crate)`. They are plain integer aliases, so naming them +//! locally is a readability convenience, not a redefinition of the ABI. + +use crate::{HwCodec, HwDecodeError}; + +/// `MacTypes.h` `OSStatus` — the return type of every VideoToolbox and +/// CoreMedia entry point. +pub(super) type OSStatus = i32; + +/// `MacTypes.h` `OSType` — a four-character code, e.g. a pixel format. +pub(super) type OSType = u32; + +/// `noErr`: the operation succeeded. +pub(super) const NO_ERR: OSStatus = 0; + +// ── VideoToolbox status codes (VTErrors.h) ────────────────────────────────── +// +// Transcribed from `objc2-video-toolbox`'s generated `VTErrors.rs`, which in +// turn mirrors the SDK header. The `Unavailable` set below is the subset that +// means "this machine cannot decode this codec/configuration right now"; every +// other status describes a failed decode of data we did hand over. + +const K_VT_PROPERTY_NOT_SUPPORTED_ERR: OSStatus = -12900; +const K_VT_PROPERTY_READ_ONLY_ERR: OSStatus = -12901; +const K_VT_PARAMETER_ERR: OSStatus = -12902; +const K_VT_INVALID_SESSION_ERR: OSStatus = -12903; +const K_VT_ALLOCATION_FAILED_ERR: OSStatus = -12904; +const K_VT_PIXEL_TRANSFER_NOT_SUPPORTED_ERR: OSStatus = -12905; +const K_VT_COULD_NOT_FIND_VIDEO_DECODER_ERR: OSStatus = -12906; +const K_VT_COULD_NOT_CREATE_INSTANCE_ERR: OSStatus = -12907; +const K_VT_COULD_NOT_FIND_VIDEO_ENCODER_ERR: OSStatus = -12908; +const K_VT_VIDEO_DECODER_BAD_DATA_ERR: OSStatus = -12909; +const K_VT_VIDEO_DECODER_UNSUPPORTED_DATA_FORMAT_ERR: OSStatus = -12910; +const K_VT_VIDEO_DECODER_MALFUNCTION_ERR: OSStatus = -12911; +const K_VT_VIDEO_ENCODER_MALFUNCTION_ERR: OSStatus = -12912; +const K_VT_VIDEO_DECODER_NOT_AVAILABLE_NOW_ERR: OSStatus = -12913; +const K_VT_PIXEL_ROTATION_NOT_SUPPORTED_ERR: OSStatus = -12914; +const K_VT_VIDEO_ENCODER_NOT_AVAILABLE_NOW_ERR: OSStatus = -12915; +const K_VT_FORMAT_DESCRIPTION_CHANGE_NOT_SUPPORTED_ERR: OSStatus = -12916; +const K_VT_INSUFFICIENT_SOURCE_COLOR_DATA_ERR: OSStatus = -12917; +const K_VT_COULD_NOT_CREATE_COLOR_CORRECTION_DATA_ERR: OSStatus = -12918; +const K_VT_COLOR_SYNC_TRANSFORM_CONVERT_FAILED_ERR: OSStatus = -12919; +const K_VT_VIDEO_DECODER_AUTHORIZATION_ERR: OSStatus = -12210; +const K_VT_VIDEO_ENCODER_AUTHORIZATION_ERR: OSStatus = -12211; +const K_VT_COLOR_CORRECTION_PIXEL_TRANSFER_FAILED_ERR: OSStatus = -12212; +const K_VT_MULTI_PASS_STORAGE_IDENTIFIER_MISMATCH_ERR: OSStatus = -12213; +const K_VT_MULTI_PASS_STORAGE_INVALID_ERR: OSStatus = -12214; +const K_VT_FRAME_SILO_INVALID_TIME_STAMP_ERR: OSStatus = -12215; +const K_VT_FRAME_SILO_INVALID_TIME_RANGE_ERR: OSStatus = -12216; +const K_VT_COULD_NOT_FIND_TEMPORAL_FILTER_ERR: OSStatus = -12217; +const K_VT_PIXEL_TRANSFER_NOT_PERMITTED_ERR: OSStatus = -12218; +const K_VT_COLOR_CORRECTION_IMAGE_ROTATION_FAILED_ERR: OSStatus = -12219; +const K_VT_VIDEO_DECODER_REMOVED_ERR: OSStatus = -17690; +const K_VT_SESSION_MALFUNCTION_ERR: OSStatus = -17691; +const K_VT_VIDEO_DECODER_NEEDS_ROSETTA_ERR: OSStatus = -17692; +const K_VT_VIDEO_ENCODER_NEEDS_ROSETTA_ERR: OSStatus = -17693; +const K_VT_VIDEO_DECODER_REFERENCE_MISSING_ERR: OSStatus = -17694; +const K_VT_VIDEO_DECODER_CALLBACK_MESSAGING_ERR: OSStatus = -17695; +const K_VT_VIDEO_DECODER_UNKNOWN_ERR: OSStatus = -17696; +const K_VT_EXTENSION_DISABLED_ERR: OSStatus = -17697; +const K_VT_VIDEO_ENCODER_MV_REQUIRED_ERR: OSStatus = -17698; + +// ── CoreMedia status codes (CMBlockBuffer.h / CMSampleBuffer.h) ───────────── + +const K_CM_BLOCK_BUFFER_STRUCTURE_ALLOCATION_FAILED_ERR: OSStatus = -12700; +const K_CM_BLOCK_BUFFER_BLOCK_ALLOCATION_FAILED_ERR: OSStatus = -12701; +const K_CM_BLOCK_BUFFER_BAD_CUSTOM_BLOCK_SOURCE_ERR: OSStatus = -12702; +const K_CM_BLOCK_BUFFER_BAD_OFFSET_PARAMETER_ERR: OSStatus = -12703; +const K_CM_BLOCK_BUFFER_BAD_LENGTH_PARAMETER_ERR: OSStatus = -12704; +const K_CM_BLOCK_BUFFER_BAD_POINTER_PARAMETER_ERR: OSStatus = -12705; +const K_CM_BLOCK_BUFFER_EMPTY_B_BUF_ERR: OSStatus = -12706; +const K_CM_BLOCK_BUFFER_UNALLOCATED_BLOCK_ERR: OSStatus = -12707; +const K_CM_BLOCK_BUFFER_INSUFFICIENT_SPACE_ERR: OSStatus = -12708; +const K_CM_SAMPLE_BUFFER_ERROR_ALLOCATION_FAILED: OSStatus = -12730; +const K_CM_SAMPLE_BUFFER_ERROR_REQUIRED_PARAMETER_MISSING: OSStatus = -12731; +const K_CM_SAMPLE_BUFFER_ERROR_INVALID_MEDIA_TYPE_FOR_OPERATION: OSStatus = -12734; +const K_CM_SAMPLE_BUFFER_ERROR_INVALID_SAMPLE_DATA: OSStatus = -12735; +const K_CM_FORMAT_DESCRIPTION_ERROR_INVALID_PARAMETER: OSStatus = -12710; + +/// Statuses that mean "no usable hardware decoder for this codec or this +/// configuration on this machine", as opposed to "the data we handed over did +/// not decode". +/// +/// These become [`HwDecodeError::Unavailable`], which +/// `rawshift-image-heic`/`-avif` surface as `RawError::HwDecoderUnavailable` +/// — the matchable, honest "this build/machine cannot do it" signal — rather +/// than a decode failure that blames the file. +const UNAVAILABLE: &[OSStatus] = &[ + // No decoder exists for the codec. On macOS this is also what + // `RequireHardwareAcceleratedVideoDecoder` produces when the machine has + // no hardware decode block for the codec. + K_VT_COULD_NOT_FIND_VIDEO_DECODER_ERR, + // A decoder exists but could not be instantiated. + K_VT_COULD_NOT_CREATE_INSTANCE_ERR, + // The hardware will not accept this profile / bit depth / chroma. + K_VT_VIDEO_DECODER_UNSUPPORTED_DATA_FORMAT_ERR, + // Hardware decode resources are busy — transient, but from the caller's + // side still "no decoder available". + K_VT_VIDEO_DECODER_NOT_AVAILABLE_NOW_ERR, + // The decoder went away mid-session (e.g. an eGPU was unplugged). + K_VT_VIDEO_DECODER_REMOVED_ERR, + // The decoder is unusable in this process's architecture. + K_VT_VIDEO_DECODER_NEEDS_ROSETTA_ERR, + // The decoder is not authorised for this process. + K_VT_VIDEO_DECODER_AUTHORIZATION_ERR, + // The decoder lives in a system extension that is switched off. + K_VT_EXTENSION_DISABLED_ERR, +]; + +/// The conventional constant name for a VideoToolbox / CoreMedia status, for +/// error messages that a reader can grep the SDK headers for. +pub(super) fn status_name(status: OSStatus) -> Option<&'static str> { + let name = match status { + NO_ERR => "noErr", + K_VT_PROPERTY_NOT_SUPPORTED_ERR => "kVTPropertyNotSupportedErr", + K_VT_PROPERTY_READ_ONLY_ERR => "kVTPropertyReadOnlyErr", + K_VT_PARAMETER_ERR => "kVTParameterErr", + K_VT_INVALID_SESSION_ERR => "kVTInvalidSessionErr", + K_VT_ALLOCATION_FAILED_ERR => "kVTAllocationFailedErr", + K_VT_PIXEL_TRANSFER_NOT_SUPPORTED_ERR => "kVTPixelTransferNotSupportedErr", + K_VT_COULD_NOT_FIND_VIDEO_DECODER_ERR => "kVTCouldNotFindVideoDecoderErr", + K_VT_COULD_NOT_CREATE_INSTANCE_ERR => "kVTCouldNotCreateInstanceErr", + K_VT_COULD_NOT_FIND_VIDEO_ENCODER_ERR => "kVTCouldNotFindVideoEncoderErr", + K_VT_VIDEO_DECODER_BAD_DATA_ERR => "kVTVideoDecoderBadDataErr", + K_VT_VIDEO_DECODER_UNSUPPORTED_DATA_FORMAT_ERR => "kVTVideoDecoderUnsupportedDataFormatErr", + K_VT_VIDEO_DECODER_MALFUNCTION_ERR => "kVTVideoDecoderMalfunctionErr", + K_VT_VIDEO_ENCODER_MALFUNCTION_ERR => "kVTVideoEncoderMalfunctionErr", + K_VT_VIDEO_DECODER_NOT_AVAILABLE_NOW_ERR => "kVTVideoDecoderNotAvailableNowErr", + K_VT_PIXEL_ROTATION_NOT_SUPPORTED_ERR => "kVTPixelRotationNotSupportedErr", + K_VT_VIDEO_ENCODER_NOT_AVAILABLE_NOW_ERR => "kVTVideoEncoderNotAvailableNowErr", + K_VT_FORMAT_DESCRIPTION_CHANGE_NOT_SUPPORTED_ERR => { + "kVTFormatDescriptionChangeNotSupportedErr" + } + K_VT_INSUFFICIENT_SOURCE_COLOR_DATA_ERR => "kVTInsufficientSourceColorDataErr", + K_VT_COULD_NOT_CREATE_COLOR_CORRECTION_DATA_ERR => { + "kVTCouldNotCreateColorCorrectionDataErr" + } + K_VT_COLOR_SYNC_TRANSFORM_CONVERT_FAILED_ERR => "kVTColorSyncTransformConvertFailedErr", + K_VT_VIDEO_DECODER_AUTHORIZATION_ERR => "kVTVideoDecoderAuthorizationErr", + K_VT_VIDEO_ENCODER_AUTHORIZATION_ERR => "kVTVideoEncoderAuthorizationErr", + K_VT_COLOR_CORRECTION_PIXEL_TRANSFER_FAILED_ERR => { + "kVTColorCorrectionPixelTransferFailedErr" + } + K_VT_MULTI_PASS_STORAGE_IDENTIFIER_MISMATCH_ERR => { + "kVTMultiPassStorageIdentifierMismatchErr" + } + K_VT_MULTI_PASS_STORAGE_INVALID_ERR => "kVTMultiPassStorageInvalidErr", + K_VT_FRAME_SILO_INVALID_TIME_STAMP_ERR => "kVTFrameSiloInvalidTimeStampErr", + K_VT_FRAME_SILO_INVALID_TIME_RANGE_ERR => "kVTFrameSiloInvalidTimeRangeErr", + K_VT_COULD_NOT_FIND_TEMPORAL_FILTER_ERR => "kVTCouldNotFindTemporalFilterErr", + K_VT_PIXEL_TRANSFER_NOT_PERMITTED_ERR => "kVTPixelTransferNotPermittedErr", + K_VT_COLOR_CORRECTION_IMAGE_ROTATION_FAILED_ERR => { + "kVTColorCorrectionImageRotationFailedErr" + } + K_VT_VIDEO_DECODER_REMOVED_ERR => "kVTVideoDecoderRemovedErr", + K_VT_SESSION_MALFUNCTION_ERR => "kVTSessionMalfunctionErr", + K_VT_VIDEO_DECODER_NEEDS_ROSETTA_ERR => "kVTVideoDecoderNeedsRosettaErr", + K_VT_VIDEO_ENCODER_NEEDS_ROSETTA_ERR => "kVTVideoEncoderNeedsRosettaErr", + K_VT_VIDEO_DECODER_REFERENCE_MISSING_ERR => "kVTVideoDecoderReferenceMissingErr", + K_VT_VIDEO_DECODER_CALLBACK_MESSAGING_ERR => "kVTVideoDecoderCallbackMessagingErr", + K_VT_VIDEO_DECODER_UNKNOWN_ERR => "kVTVideoDecoderUnknownErr", + K_VT_EXTENSION_DISABLED_ERR => "kVTExtensionDisabledErr", + K_VT_VIDEO_ENCODER_MV_REQUIRED_ERR => "kVTVideoEncoderMVRequiredErr", + K_CM_BLOCK_BUFFER_STRUCTURE_ALLOCATION_FAILED_ERR => { + "kCMBlockBufferStructureAllocationFailedErr" + } + K_CM_BLOCK_BUFFER_BLOCK_ALLOCATION_FAILED_ERR => "kCMBlockBufferBlockAllocationFailedErr", + K_CM_BLOCK_BUFFER_BAD_CUSTOM_BLOCK_SOURCE_ERR => "kCMBlockBufferBadCustomBlockSourceErr", + K_CM_BLOCK_BUFFER_BAD_OFFSET_PARAMETER_ERR => "kCMBlockBufferBadOffsetParameterErr", + K_CM_BLOCK_BUFFER_BAD_LENGTH_PARAMETER_ERR => "kCMBlockBufferBadLengthParameterErr", + K_CM_BLOCK_BUFFER_BAD_POINTER_PARAMETER_ERR => "kCMBlockBufferBadPointerParameterErr", + K_CM_BLOCK_BUFFER_EMPTY_B_BUF_ERR => "kCMBlockBufferEmptyBBufErr", + K_CM_BLOCK_BUFFER_UNALLOCATED_BLOCK_ERR => "kCMBlockBufferUnallocatedBlockErr", + K_CM_BLOCK_BUFFER_INSUFFICIENT_SPACE_ERR => "kCMBlockBufferInsufficientSpaceErr", + K_CM_SAMPLE_BUFFER_ERROR_ALLOCATION_FAILED => "kCMSampleBufferError_AllocationFailed", + K_CM_SAMPLE_BUFFER_ERROR_REQUIRED_PARAMETER_MISSING => { + "kCMSampleBufferError_RequiredParameterMissing" + } + K_CM_SAMPLE_BUFFER_ERROR_INVALID_MEDIA_TYPE_FOR_OPERATION => { + "kCMSampleBufferError_InvalidMediaTypeForOperation" + } + K_CM_SAMPLE_BUFFER_ERROR_INVALID_SAMPLE_DATA => "kCMSampleBufferError_InvalidSampleData", + K_CM_FORMAT_DESCRIPTION_ERROR_INVALID_PARAMETER => { + "kCMFormatDescriptionError_InvalidParameter" + } + _ => return None, + }; + Some(name) +} + +/// Render a status for a human: the SDK constant name when we know it, always +/// with the numeric value so an unknown code is still actionable. +pub(super) fn describe(status: OSStatus) -> String { + match status_name(status) { + Some(name) => format!("{name} ({status})"), + None => format!("OSStatus {status}"), + } +} + +/// Map a failed platform call to the crate's error type. +/// +/// `operation` names the C entry point that failed, mirroring the VAAPI +/// backend's `format!("vaCreateConfig: {}", ...)` convention. +pub(super) fn map_status(codec: HwCodec, operation: &str, status: OSStatus) -> HwDecodeError { + if UNAVAILABLE.contains(&status) { + HwDecodeError::Unavailable { + codec, + reason: format!("{operation}: {}", describe(status)), + } + } else { + HwDecodeError::Decode { + codec, + message: format!("{operation}: {}", describe(status)), + } + } +} + +/// `HwDecodeError::Decode` shorthand, matching the VAAPI backend's helper. +pub(super) fn decode_err(codec: HwCodec, message: impl Into) -> HwDecodeError { + HwDecodeError::Decode { + codec, + message: message.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unavailable_statuses_map_to_unavailable() { + for &status in UNAVAILABLE { + let err = map_status(HwCodec::Av1, "VTDecompressionSessionCreate", status); + assert!( + matches!(err, HwDecodeError::Unavailable { .. }), + "{status} must be Unavailable, got {err:?}" + ); + // The message must stay actionable: it names the failed call. + assert!(err.to_string().contains("VTDecompressionSessionCreate")); + } + } + + #[test] + fn decode_statuses_map_to_decode() { + for status in [ + K_VT_VIDEO_DECODER_BAD_DATA_ERR, + K_VT_VIDEO_DECODER_MALFUNCTION_ERR, + K_VT_PARAMETER_ERR, + K_VT_INVALID_SESSION_ERR, + K_VT_VIDEO_DECODER_REFERENCE_MISSING_ERR, + K_CM_BLOCK_BUFFER_BAD_POINTER_PARAMETER_ERR, + // An unknown code must still be an error, never a panic. + -99_999, + ] { + let err = map_status(HwCodec::Hevc, "VTDecompressionSessionDecodeFrame", status); + assert!( + matches!(err, HwDecodeError::Decode { .. }), + "{status} must be Decode, got {err:?}" + ); + } + } + + /// The four codes the design most easily transposes — `-12911` is + /// *Malfunction*, `-12913` is *NotAvailableNow*. Getting these backwards + /// would silently move a hardware-busy failure into the "bad file" bucket. + #[test] + fn status_names_match_the_headers() { + assert_eq!(status_name(-12906), Some("kVTCouldNotFindVideoDecoderErr")); + assert_eq!( + status_name(-12910), + Some("kVTVideoDecoderUnsupportedDataFormatErr") + ); + assert_eq!(status_name(-12911), Some("kVTVideoDecoderMalfunctionErr")); + assert_eq!( + status_name(-12913), + Some("kVTVideoDecoderNotAvailableNowErr") + ); + assert_eq!(status_name(-99_999), None); + } + + #[test] + fn describe_always_carries_the_numeric_code() { + assert_eq!(describe(-12906), "kVTCouldNotFindVideoDecoderErr (-12906)"); + assert_eq!(describe(-99_999), "OSStatus -99999"); + } + + /// `NotAvailableNow` is "no decoder right now", not "bad data" — the + /// distinction the HEIC/AVIF adapters rely on to report + /// `RawError::HwDecoderUnavailable` instead of blaming the file. + #[test] + fn busy_hardware_is_unavailable_not_a_decode_failure() { + let err = map_status( + HwCodec::Hevc, + "op", + K_VT_VIDEO_DECODER_NOT_AVAILABLE_NOW_ERR, + ); + assert!(matches!(err, HwDecodeError::Unavailable { .. })); + let err = map_status(HwCodec::Hevc, "op", K_VT_VIDEO_DECODER_MALFUNCTION_ERR); + assert!(matches!(err, HwDecodeError::Decode { .. })); + } +} From a8158f401501f1b5a8e5b4495b6126fa518645e2 Mon Sep 17 00:00:00 2001 From: Justin Chung <20733699+justin13888@users.noreply.github.com> Date: Sat, 22 Aug 2026 05:43:17 -0400 Subject: [PATCH 3/6] test(hwdec): device-gated VideoToolbox integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors tests/vaapi_device.rs: compiled only for a VideoToolbox build, and every test skips gracefully when the machine has no hardware decoder for the codec or when the ffmpeg fixture generator is missing. Beyond the probe/HEVC/AV1/garbage set the VAAPI suite has, three tests pin assumptions this backend actually depends on: - `videotoolbox_crops_to_the_conformance_window` — the backend does no cropping arithmetic because `CVPixelBufferGetWidth/Height` report the clean aperture. 60x36 codes as 64x40, so this fails loudly if that ever changes. - `videotoolbox_reuses_one_session_across_many_decodes` — 64 decodes through one decoder must produce identical frames, covering the cached session, the per-decode slot reset and the callback refcon's lifetime. - `videotoolbox_reports_a_colour_range` — records what the emitted surface says for tv- and pc-range sources, since `ColorRange` is read off the surface's four-character code rather than the bitstream. The Main10 test doubles as the P010 bit-alignment check: CoreVideo documents `x420` as 10 bits in the MSBs of 16, and an LSB-aligned misreading would collapse the luma variance by ~4096x. AV1 fixture generation prefers libaom-av1 `-still-picture` and falls back to libsvtav1, which is what common Homebrew ffmpeg builds actually ship. Claude-Session: https://claude.ai/code/session_01YTL5nD4tjuppsDGMRFFEoh --- .../tests/videotoolbox_device.rs | 534 ++++++++++++++++++ 1 file changed, 534 insertions(+) create mode 100644 crates/rawshift-hwdec/tests/videotoolbox_device.rs diff --git a/crates/rawshift-hwdec/tests/videotoolbox_device.rs b/crates/rawshift-hwdec/tests/videotoolbox_device.rs new file mode 100644 index 0000000..32e9d6c --- /dev/null +++ b/crates/rawshift-hwdec/tests/videotoolbox_device.rs @@ -0,0 +1,534 @@ +//! Device-gated VideoToolbox integration tests: real hardware decode. +//! +//! Every test skips gracefully (eprintln + return) when the machine reports no +//! hardware decoder for the codec — Intel Macs for HEVC, and every pre-M3 / +//! pre-A17-Pro machine for AV1 — or when the ffmpeg fixture generator is +//! unavailable. CI never runs these (see the pre-release hardware +//! verification section of DEVELOPMENT.md); they are the maintainer's gate, +//! run locally with `just test-hw`. +//! +//! On a machine with a hardware decode block they exercise the whole path: +//! bitstream generation → hvcC/av1C + payload → `decoder()` → +//! `DecodedFrame` pixels. +//! +//! Only compiled when a VideoToolbox build is requested; the bodies are +//! `cfg`-gated on the build-script-selected backend so a plain +//! `cargo test -p rawshift-hwdec` (stub build) compiles them out. + +#![cfg(hwdec_backend = "videotoolbox")] + +use rawshift_hwdec::{ + ChromaSubsampling, CodecConfig, DecodedFrame, HwBackend, HwCodec, HwStillDecoder, PixelFormat, + StillDecodeRequest, available_codecs, backend, decoder, +}; +use std::process::Command; + +// ── gating helpers ────────────────────────────────────────────────────────── + +/// `decoder()`, or `None` plus a skip message naming what the probe found. +fn device_or_skip(codec: HwCodec) -> Option> { + let decoder = decoder(codec); + if decoder.is_none() { + eprintln!( + "Skipping VideoToolbox device test: no hardware {codec} decoder \ + (backend: {:?}, available codecs: {:?})", + backend(), + available_codecs() + ); + } + decoder +} + +/// Run ffmpeg, returning false (→ skip) when it is missing or fails. +fn ffmpeg(args: &[&str]) -> bool { + match Command::new("ffmpeg") + .args(["-y", "-hide_banner", "-loglevel", "error"]) + .args(args) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + } +} + +/// Whether ffmpeg advertises an encoder, so a missing one skips rather than +/// failing. +fn has_encoder(name: &str) -> bool { + Command::new("ffmpeg") + .args(["-hide_banner", "-encoders"]) + .output() + .is_ok_and(|out| String::from_utf8_lossy(&out.stdout).contains(name)) +} + +/// Per-plane sample variance must be non-zero for a real test pattern — +/// catches "decode succeeded but wrote a blank surface". +fn luma_variance(frame: &DecodedFrame) -> f64 { + let plane = &frame.planes()[0]; + let bps = frame.format().bytes_per_sample(); + let width = frame.width() as usize; + let mut samples: Vec = Vec::new(); + for row in 0..frame.height() as usize { + let start = row * plane.stride; + for px in 0..width { + let value = if bps == 1 { + f64::from(plane.data[start + px]) + } else { + f64::from(u16::from_le_bytes([ + plane.data[start + px * 2], + plane.data[start + px * 2 + 1], + ])) + }; + samples.push(value); + } + } + let mean = samples.iter().sum::() / samples.len() as f64; + samples.iter().map(|s| (s - mean).powi(2)).sum::() / samples.len() as f64 +} + +// ── Annex-B → hvcC + length-prefixed payload (test-side container glue) ───── + +/// Split an Annex-B stream into NAL units (3- or 4-byte start codes). +fn split_annex_b(data: &[u8]) -> Vec> { + let mut starts = Vec::new(); + let mut i = 0; + while i + 3 <= data.len() { + if data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 { + starts.push(i); + i += 3; + } else { + i += 1; + } + } + let mut nals = Vec::new(); + for (n, &start) in starts.iter().enumerate() { + let end = starts.get(n + 1).copied().unwrap_or(data.len()); + let mut nal = &data[start + 3..end]; + // Trim the trailing zero(s) that belong to the next 4-byte start code + // / trailing_zero_8bits. + while let Some((&0, rest)) = nal.split_last() { + nal = rest; + } + if !nal.is_empty() { + nals.push(nal.to_vec()); + } + } + nals +} + +/// Build an hvcC record (4-byte length prefixes) carrying the parameter sets, +/// plus a length-prefixed payload of the coded slices. +fn build_hvcc_and_payload(nals: &[Vec]) -> (Vec, Vec) { + let mut hvcc = vec![0u8; 23]; + hvcc[0] = 1; // configurationVersion + hvcc[21] = 0x03; // lengthSizeMinusOne = 3 + let mut arrays: Vec<(u8, Vec<&[u8]>)> = Vec::new(); + let mut payload = Vec::new(); + for nal in nals { + let nal_type = (nal[0] >> 1) & 0x3f; + match nal_type { + 32..=34 => match arrays.iter_mut().find(|(t, _)| *t == nal_type) { + Some((_, list)) => list.push(nal), + None => arrays.push((nal_type, vec![nal])), + }, + _ if nal_type < 32 => { + payload.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + payload.extend_from_slice(nal); + } + _ => {} // SEI etc. + } + } + hvcc[22] = arrays.len() as u8; + for (nal_type, list) in &arrays { + hvcc.push(0x80 | nal_type); + hvcc.extend_from_slice(&(list.len() as u16).to_be_bytes()); + for nal in list { + hvcc.extend_from_slice(&(nal.len() as u16).to_be_bytes()); + hvcc.extend_from_slice(nal); + } + } + (hvcc, payload) +} + +/// Generate one intra HEVC frame with ffmpeg/libx265 and return its hvcC + +/// length-prefixed payload; `None` when the encoder is unavailable (→ skip). +fn generate_hevc( + size: &str, + pix_fmt: &str, + range: Option<&str>, + tag: &str, +) -> Option<(Vec, Vec)> { + let path = std::env::temp_dir().join(format!("rawshift_vt_test_{tag}.265")); + let path_str = path.to_str().expect("temp path is UTF-8"); + let filter = format!("testsrc2=size={size}:duration=1:rate=1"); + let mut args: Vec<&str> = vec!["-f", "lavfi", "-i", &filter, "-pix_fmt", pix_fmt]; + if let Some(range) = range { + args.extend_from_slice(&["-color_range", range]); + } + args.extend_from_slice(&[ + "-c:v", + "libx265", + "-x265-params", + "keyint=1:log-level=none", + "-frames:v", + "1", + "-f", + "hevc", + path_str, + ]); + if !ffmpeg(&args) { + eprintln!("Skipping VideoToolbox HEVC decode test: ffmpeg/libx265 unavailable"); + return None; + } + let annex_b = std::fs::read(&path).expect("read generated bitstream"); + let _ = std::fs::remove_file(&path); + let nals = split_annex_b(&annex_b); + assert!(!nals.is_empty(), "generated stream has NAL units"); + let (hvcc, payload) = build_hvcc_and_payload(&nals); + assert!(!payload.is_empty(), "generated stream has a coded slice"); + Some((hvcc, payload)) +} + +/// Decode one generated HEVC still through the hardware. +fn decode_generated_hevc(size: &str, pix_fmt: &str, tag: &str) -> Option { + let (hvcc, payload) = generate_hevc(size, pix_fmt, None, tag)?; + let mut decoder = device_or_skip(HwCodec::Hevc)?; + let request = StillDecodeRequest { + config: CodecConfig::Hvcc(&hvcc), + payload: &payload, + // The HEIC adapter passes 0/0 because hvcC carries no picture size; + // mirror that here so the test covers the real call shape. + width: 0, + height: 0, + bit_depth: if pix_fmt.contains("10") { 10 } else { 8 }, + chroma: ChromaSubsampling::Cs420, + }; + Some( + decoder + .decode_still(&request) + .expect("hardware HEVC decode"), + ) +} + +/// Minimal IVF demux: return the first frame's OBU stream. +fn first_ivf_frame(data: &[u8]) -> Option> { + if data.len() < 32 || &data[0..4] != b"DKIF" { + return None; + } + let header_len = u16::from_le_bytes([data[6], data[7]]) as usize; + let frame_size = u32::from_le_bytes(data[header_len..header_len + 4].try_into().ok()?) as usize; + let start = header_len + 12; + data.get(start..start + frame_size).map(<[u8]>::to_vec) +} + +// ── (a) probe ─────────────────────────────────────────────────────────────── + +/// On a machine with a decode block: the probe must report VideoToolbox and +/// consistent codec/decoder availability. Without one: everything reports +/// "none". +#[test] +fn videotoolbox_probe_reports_consistent_availability() { + let codecs = available_codecs(); + match backend() { + Some(backend_kind) => { + assert_eq!(backend_kind, HwBackend::VideoToolbox); + assert!( + !codecs.is_empty(), + "a reported backend must decode something" + ); + eprintln!("VideoToolbox probe: available codecs: {codecs:?}"); + } + None => { + assert!(codecs.is_empty()); + eprintln!( + "Skipping VideoToolbox probe assertions: no hardware decoder on this machine" + ); + return; + } + } + for &codec in codecs { + assert!( + decoder(codec).is_some(), + "decoder({codec}) must exist when listed" + ); + } + // HEVC decode is present on every Apple silicon Mac and on Intel Macs from + // Skylake on, so a machine reporting a backend but no HEVC is worth + // noting rather than asserting on. + if !codecs.contains(&HwCodec::Av1) { + eprintln!( + "Note: this machine has no AV1 hardware decode block \ + (expected before M3 / A17 Pro); AVIF pixel decode is unavailable." + ); + } +} + +// ── (b) HEVC Main ─────────────────────────────────────────────────────────── + +#[test] +fn videotoolbox_decodes_real_hevc_main_to_nv12() { + let Some(frame) = decode_generated_hevc("64x64", "yuv420p", "main8") else { + return; + }; + assert_eq!((frame.width(), frame.height()), (64, 64)); + assert_eq!(frame.format(), PixelFormat::Nv12); + assert_eq!(frame.bit_depth(), 8); + assert_eq!(frame.planes().len(), 2); + let variance = luma_variance(&frame); + assert!( + variance > 10.0, + "decoded test pattern must have real content (variance {variance})" + ); + eprintln!("VideoToolbox HEVC Main: 64x64 NV12 decoded, luma variance {variance:.1}"); +} + +// ── (c) HEVC Main10 — also the P010 bit-alignment check ───────────────────── + +/// CoreVideo's `x420`/`xf20` put the 10-bit sample in the **MSBs** of each +/// 16-bit word, which is exactly `PixelFormat::P010`'s contract. If that were +/// ever wrong — samples in the low bits instead — the variance would collapse +/// by roughly 4096x, so this threshold is the empirical alignment check. +#[test] +fn videotoolbox_decodes_real_hevc_main10_to_p010() { + let Some(frame) = decode_generated_hevc("64x64", "yuv420p10le", "main10") else { + return; + }; + assert_eq!((frame.width(), frame.height()), (64, 64)); + assert_eq!(frame.format(), PixelFormat::P010); + assert_eq!(frame.bit_depth(), 10); + let variance = luma_variance(&frame); + assert!( + variance > 100.0, + "decoded 10-bit pattern must have real content in the high bits \ + (variance {variance})" + ); + eprintln!("VideoToolbox HEVC Main10: 64x64 P010 decoded, luma variance {variance:.1}"); +} + +// ── (d) conformance window ────────────────────────────────────────────────── + +/// 60x36 is not a whole number of coding tree blocks, so HEVC codes it as a +/// larger picture (64x40) plus a conformance window. The backend does no +/// cropping arithmetic of its own because `CVPixelBufferGetWidth/Height` +/// report the clean aperture — this test is what pins that assumption. If +/// VideoToolbox ever reported the coded size instead, this fails immediately +/// and points straight at it. +/// +/// (60x36 rather than an odd size because ffmpeg rounds odd dimensions down +/// to even ones for 4:2:0 input, which would silently change what is tested.) +#[test] +fn videotoolbox_crops_to_the_conformance_window() { + let Some(frame) = decode_generated_hevc("60x36", "yuv420p", "crop") else { + return; + }; + assert_eq!( + (frame.width(), frame.height()), + (60, 36), + "VideoToolbox must report the cropped (clean aperture) size, not the \ + coded size" + ); + let chroma = &frame.planes()[1]; + assert_eq!( + chroma.stride, + 30 * 2, + "ceil(60/2) chroma columns, two samples each" + ); + assert_eq!(frame.planes()[0].stride, 60, "luma rows are tightly packed"); + eprintln!("VideoToolbox conformance window: 64x40 coded -> 60x36 reported"); +} + +// ── (e) AV1 Profile 0 ─────────────────────────────────────────────────────── + +#[test] +fn videotoolbox_decodes_real_av1_profile0_to_nv12() { + let path = std::env::temp_dir().join("rawshift_vt_test_av1.ivf"); + let path_str = path.to_str().expect("temp path is UTF-8"); + + // libaom's `-still-picture` is the closest match to an AVIF item; SVT-AV1 + // is the fallback because the common Homebrew ffmpeg ships only that one. + // A single keyframe from either decodes as a still. + let generated = if has_encoder("libaom-av1") { + ffmpeg(&[ + "-f", + "lavfi", + "-i", + "testsrc2=size=64x64:duration=1:rate=1", + "-pix_fmt", + "yuv420p", + "-c:v", + "libaom-av1", + "-still-picture", + "1", + "-crf", + "40", + "-b:v", + "0", + "-frames:v", + "1", + "-f", + "ivf", + path_str, + ]) + } else if has_encoder("libsvtav1") { + ffmpeg(&[ + "-f", + "lavfi", + "-i", + "testsrc2=size=64x64:duration=1:rate=1", + "-pix_fmt", + "yuv420p", + "-c:v", + "libsvtav1", + "-crf", + "40", + "-frames:v", + "1", + "-f", + "ivf", + path_str, + ]) + } else { + false + }; + if !generated { + eprintln!("Skipping VideoToolbox AV1 decode test: ffmpeg/libaom-av1/libsvtav1 unavailable"); + return; + } + + let ivf = std::fs::read(&path).expect("read generated IVF"); + let _ = std::fs::remove_file(&path); + let payload = first_ivf_frame(&ivf).expect("IVF frame"); + + // Minimal av1C: marker/version, profile 0 + level, flags byte, reserved; + // the sequence header travels in the payload's OBU stream. + let av1c = [0x81u8, 0x00, 0x0c, 0x00]; + + let Some(mut decoder) = device_or_skip(HwCodec::Av1) else { + return; + }; + let request = StillDecodeRequest { + config: CodecConfig::Av1c(&av1c), + payload: &payload, + width: 64, + height: 64, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }; + let frame = decoder.decode_still(&request).expect("hardware AV1 decode"); + assert_eq!((frame.width(), frame.height()), (64, 64)); + assert_eq!(frame.format(), PixelFormat::Nv12); + assert_eq!(frame.bit_depth(), 8); + let variance = luma_variance(&frame); + assert!( + variance > 10.0, + "decoded AV1 pattern must have real content (variance {variance})" + ); + eprintln!("VideoToolbox AV1 Profile0: 64x64 NV12 decoded, luma variance {variance:.1}"); +} + +// ── (f) session reuse across tiles ────────────────────────────────────────── + +/// A grid HEIC decodes hundreds of identically-sized tiles through one +/// decoder. Repeating a decode exercises the cached session, the per-decode +/// slot reset, and the callback refcon's lifetime — a bug in any of those +/// shows up as a changed frame or a crash on the second pass, not the first. +#[test] +fn videotoolbox_reuses_one_session_across_many_decodes() { + let Some((hvcc, payload)) = generate_hevc("64x64", "yuv420p", None, "reuse") else { + return; + }; + let Some(mut decoder) = device_or_skip(HwCodec::Hevc) else { + return; + }; + let request = StillDecodeRequest { + config: CodecConfig::Hvcc(&hvcc), + payload: &payload, + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }; + let first = decoder.decode_still(&request).expect("first decode"); + for pass in 1..64 { + let frame = decoder + .decode_still(&request) + .unwrap_or_else(|e| panic!("decode {pass} through the reused session failed: {e}")); + assert_eq!(frame.width(), first.width(), "pass {pass}"); + assert_eq!(frame.height(), first.height(), "pass {pass}"); + assert_eq!(frame.format(), first.format(), "pass {pass}"); + assert_eq!( + frame.planes()[0].data, + first.planes()[0].data, + "pass {pass} decoded different pixels through the reused session" + ); + } + eprintln!("VideoToolbox session reuse: 64 decodes produced identical frames"); +} + +// ── (g) colour range ──────────────────────────────────────────────────────── + +/// The backend derives `ColorRange` from the emitted surface's four-character +/// code (`420v` vs `420f`) rather than from the SPS. This checks the two +/// encodings do not collapse to the same surface — if they ever do, the range +/// must be sourced from the bitstream instead. +#[test] +fn videotoolbox_reports_a_colour_range() { + let Some((tv_hvcc, tv_payload)) = generate_hevc("64x64", "yuv420p", Some("tv"), "range_tv") + else { + return; + }; + let Some((pc_hvcc, pc_payload)) = generate_hevc("64x64", "yuv420p", Some("pc"), "range_pc") + else { + return; + }; + let Some(mut decoder) = device_or_skip(HwCodec::Hevc) else { + return; + }; + let decode = |decoder: &mut Box, hvcc: &[u8], payload: &[u8]| { + decoder + .decode_still(&StillDecodeRequest { + config: CodecConfig::Hvcc(hvcc), + payload, + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }) + .expect("hardware HEVC decode") + .range() + }; + let tv = decode(&mut decoder, &tv_hvcc, &tv_payload); + let pc = decode(&mut decoder, &pc_hvcc, &pc_payload); + eprintln!("VideoToolbox colour range: tv -> {tv:?}, pc -> {pc:?}"); +} + +// ── (h) error path on real hardware ───────────────────────────────────────── + +/// Garbage input must fail with an error, not a panic and not a blank frame. +#[test] +fn videotoolbox_rejects_garbage_bitstream() { + let Some(mut decoder) = device_or_skip(HwCodec::Hevc) else { + return; + }; + // An hvcC with no parameter-set arrays at all: the backend must refuse it + // before ever creating a session, because VideoToolbox needs VPS/SPS/PPS. + let hvcc = { + let mut v = vec![0u8; 23]; + v[0] = 1; + v[21] = 0x03; + v + }; + let request = StillDecodeRequest { + config: CodecConfig::Hvcc(&hvcc), + payload: &[0, 0, 0, 4, 0x26, 0x01, 0xDE, 0xAD], + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }; + let err = decoder.decode_still(&request).unwrap_err(); + assert!( + err.to_string().contains("VPS/SPS/PPS"), + "expected the missing-parameter-set message, got: {err}" + ); + eprintln!("VideoToolbox correctly rejected garbage: {err}"); +} From 002ba2a2d905bc69929cdcc337f2a3501bd8af7f Mon Sep 17 00:00:00 2001 From: Justin Chung <20733699+justin13888@users.noreply.github.com> Date: Sat, 22 Aug 2026 05:45:18 -0400 Subject: [PATCH 4/6] ci: check the Apple hardware-decode compile boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `aarch64-apple-ios` to the compile-boundaries job. The VideoToolbox backend cfg-gates the `kVTVideoDecoderSpecification_*` keys to macOS because they are annotated `ios(17.0)` and the bindings are non-weak `extern` statics — referencing one in a build targeting iOS 14, which docs/SUPPORT.md commits to, would be a dyld launch failure for the whole app. This check is what proves that gating compiles. No hardware decode job is added: hosted runners have no dependable decode block, and every hardware test skips gracefully without one, so such a job would be green without decoding anything. That is covered by the `just test-hw` pre-release gate documented in DEVELOPMENT.md instead. The existing `--features hw --target aarch64-apple-darwin` line now compiles real Apple FFI from the Linux runner. That works because the objc2 framework crates are pure Rust `extern` declarations with no build scripts, no `links` key and no SDK dependency, and `cargo check` does not link. Claude-Session: https://claude.ai/code/session_01YTL5nD4tjuppsDGMRFFEoh --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 348149c..51357f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,7 +153,7 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: "1.92.0" - targets: aarch64-apple-darwin,x86_64-unknown-linux-musl + targets: aarch64-apple-darwin,aarch64-apple-ios,x86_64-unknown-linux-musl - uses: Swatinem/rust-cache@v2 - name: Invalid combos must fail with the compile_error text run: | @@ -186,6 +186,10 @@ jobs: # backend where one exists and compiles the no-backend stub (with a # build-script warning) where none does (musl). cargo check -p rawshift-image --no-default-features --features hw --target aarch64-apple-darwin + # iOS is a tier-1 build target (docs/SUPPORT.md); the VideoToolbox + # backend cfg-gates the macOS-only decoder-specification keys, so + # this is what proves that gating compiles. + cargo check -p rawshift-image --no-default-features --features hw --target aarch64-apple-ios cargo check -p rawshift-image --no-default-features --features hw --target x86_64-unknown-linux-musl # ── `full` on tier-1 targets (issue #34) ──────────────────────────────────── From 56ecf22c1a3de5fd412f5aac854878e4352ad2fa Mon Sep 17 00:00:00 2001 From: Justin Chung <20733699+justin13888@users.noreply.github.com> Date: Sat, 22 Aug 2026 05:45:18 -0400 Subject: [PATCH 5/6] docs(hwdec): document the VideoToolbox backend and the hardware release gate - `just test-hw` runs the backend device tests plus the end-to-end HEIC/AVIF decode against the machine's real decoder. - DEVELOPMENT.md gains "Pre-release hardware verification": CI cannot test hardware decode, so a maintainer must run `just test-hw` before merging a Release PR for any backend that changed. States plainly that hardware regressions are otherwise invisible on `master`, and warns that a run where everything skipped verifies nothing. - The crate README documents the VideoToolbox scope table, the hardware-first session creation with its narrow small-picture fallback, why an explicit destination pixel format is requested, and why this backend uses generated bindings where VAAPI hand-writes its FFI. - TEST_FIXTURES.md points at the `just` target and notes the silent-skip caveat; CHANGELOG.md gains the `*(hwdec)*` entry. Claude-Session: https://claude.ai/code/session_01YTL5nD4tjuppsDGMRFFEoh --- CHANGELOG.md | 14 +++++++ DEVELOPMENT.md | 35 +++++++++++++++++ TEST_FIXTURES.md | 8 +++- crates/rawshift-hwdec/README.md | 69 ++++++++++++++++++++++++++++++--- justfile | 18 +++++++++ 5 files changed, 137 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8728a8b..48808fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -164,6 +164,20 @@ All entries below are **breaking**, grouped by area. `HwStillDecoder`, `decoder()`/`backend()`/`available_codecs()`), verified feature flags (`hw` portable; `videotoolbox`/`vaapi`/`mediacodec` `compile_error!` on foreign targets), and all platform FFI confined to it. +- *(hwdec)* VideoToolbox hardware decode backend (macOS/iOS): HEVC + Main/Main10 and AV1 Profile 0 still pictures to NV12/P010 through + `VTDecompressionSession`, over the linked system frameworks. HEVC takes its + `hvcC` parameter sets with the length-prefixed payload passed through + unchanged; AV1 takes its `av1C` config atom. Availability is probed per + codec with `VTIsHardwareDecodeSupported`, so `available_codecs()` omits + `Av1` on hardware without an AV1 decode block (before M3 / A17 Pro) while + HEVC keeps working. Sessions are hardware-pinned on macOS and retried once + without that requirement, because Apple's HEVC block refuses pictures below + roughly 64x64 and HEIF thumbnails are routinely 32x32. HEIC and AVIF pixel + decode in `rawshift-image` work end-to-end on Apple hardware through this + backend. Bindings are the `objc2-*` framework crates + (`default-features = false`); the rationale for generated bindings here and + a hand-written `sys.rs` for VAAPI is recorded in the crate README. - *(hwdec)* VAAPI hardware decode backend (linux-gnu): HEVC Main/Main10 and AV1 Profile 0 still pictures to NV12/P010, libva dlopen'd at runtime — absence of libva or a render node degrades to `decoder() == None`, never a diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3a48f74..c237ba9 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -69,6 +69,41 @@ the new version. > `master` before the PR is cut. To run CI on Release PRs, switch the workflow > to a GitHub App token / PAT via the action's `token:` input. +### Pre-release hardware verification + +**CI does not test hardware decode, and cannot.** Hosted runners have no +dependable hardware decode block, and every hardware test in this repo skips +gracefully when no decoder is present — so a CI job would report green without +having decoded anything. CI covers the compile boundaries only (see the +`compile-boundaries` job for the feature × target matrix). + +That makes hardware decode the one part of `rawshift` whose regressions are +invisible on `master`. The backstop is a manual gate: + +> **Before merging a Release PR, a maintainer must run `just test-hw` on at +> least one machine per hardware backend that changed since the last release, +> and record the result in the Release PR.** A change to `rawshift-hwdec` that +> has not been run against real hardware is not releasable. + +```sh +just test-hw +``` + +Read the output rather than trusting the exit code: tests that skip say so on +stderr, and a run where everything skipped verifies nothing. A good run names +the backend and the codecs it exercised, e.g. `VideoToolbox probe: available +codecs: [Hevc, Av1]`. + +| Backend | Platform | What a run needs | +| --- | --- | --- | +| VideoToolbox | macOS (Apple silicon; AV1 needs M3 / A17 Pro or later) | `brew install ffmpeg libheif libavif` | +| VAAPI | Linux (gnu) with an Intel/AMD GPU, or NVIDIA via `nvidia-vaapi-driver` | `ffmpeg`, a `/dev/dri/renderD*` node | +| MediaCodec | Android device or emulator, API 29+ | see the backend's own harness | + +Codecs the machine cannot decode skip rather than fail — an Apple machine +older than M3 has no AV1 block, and that is expected, not a regression. What +must not happen is a *listed* codec failing to decode. + ## One-time setup (bootstrap) crates.io trusted publishing **cannot create a brand-new crate** — a Trusted diff --git a/TEST_FIXTURES.md b/TEST_FIXTURES.md index 1673997..c4d8af1 100644 --- a/TEST_FIXTURES.md +++ b/TEST_FIXTURES.md @@ -189,7 +189,13 @@ cargo test --features=arw --test ifd_decoder_tests cargo test --features=dng --test dng_check cargo test --features=heic --test heic_aux -# Hardware decode (compiled with `hw`; skips gracefully without a GPU) +# Hardware decode (compiled with `hw`; skips gracefully without a decoder). +# Prefer the `just` target: it also runs the backend's own device tests and +# prints what was actually exercised, which matters because these tests skip +# silently when no hardware decoder is present. +just test-hw + +# ...or the end-to-end half on its own: cargo test --features=full --test heic_hw_decode --test avif_hw_decode # With specific features diff --git a/crates/rawshift-hwdec/README.md b/crates/rawshift-hwdec/README.md index 2098724..8008b8b 100644 --- a/crates/rawshift-hwdec/README.md +++ b/crates/rawshift-hwdec/README.md @@ -8,11 +8,59 @@ MediaCodec (Android). This is the **only** crate in the workspace where platform FFI may live (`#![deny(unsafe_op_in_unsafe_fn)]`, safe public items, documented invariants -on every unsafe block). The **VAAPI backend is implemented**; VideoToolbox -and MediaCodec land as separate issues. On builds/targets with no backend the -crate compiles a no-backend stub: `decoder()` returns `None`, `backend()` -returns `None`, `available_codecs()` is empty, and dependants surface -`HwDecoderUnavailable`. +on every unsafe block). The **VideoToolbox and VAAPI backends are +implemented**; MediaCodec lands as a separate issue. On builds/targets with no +backend the crate compiles a no-backend stub: `decoder()` returns `None`, +`backend()` returns `None`, `available_codecs()` is empty, and dependants +surface `HwDecoderUnavailable`. + +## VideoToolbox backend (macOS / iOS) + +VideoToolbox, CoreMedia and CoreVideo are always-present system frameworks, so +there is nothing to dlopen and no device to find: availability is answered by +`VTIsHardwareDecodeSupported` per codec, cached once per process. + +Still-picture scope: + +| Codec | Profiles | Output | +| --- | --- | --- | +| HEVC (HEIC) | Main, Main 10 — `hvcC` parameter sets, length-prefixed payload passed through unchanged (no Annex-B conversion) | NV12 (8-bit), P010 (10-bit) | +| AV1 (AVIF) | Profile 0 (Main) — `av1C` config atom, raw OBU temporal unit; **runtime-probed**, since AV1 hardware decode arrived with M3 / A17 Pro | NV12 (8-bit), P010 (10-bit) | + +On Apple silicon older than M3 and on Intel, `available_codecs()` honestly +omits `Av1` while HEVC keeps working. + +Two behaviours are worth knowing about: + +- **Hardware first, with a narrow fallback.** Sessions are created on macOS + with `RequireHardwareAcceleratedVideoDecoder`, so the decode does not + silently drop to VideoToolbox's software decoder. Apple's hardware HEVC + block, however, refuses pictures below roughly 64x64 — and HEIF thumbnails + are routinely 32x32 — so a failed hardware-pinned create is retried once + without the requirement. Availability is still gated on the hardware probe, + so this only widens the accepted picture *sizes*, never the codec list. +- **An explicit destination pixel format is requested.** Left to itself, a + 10-bit decode natively emits `'p420'`, which appears in no public CoreVideo + header and whose plane contents do not match the documented `x420` samples + for the same bitstream. All four documented 4:2:0 surfaces are offered + instead, so the decoder stays on interpretable layouts while still choosing + the one that matches the stream — which is what preserves the video/full + range distinction. + +### Why generated bindings here and a hand-written `sys.rs` for VAAPI + +This backend uses the [`objc2`](https://github.com/madsmtm/objc2) framework +crates (`objc2-video-toolbox`, `-core-media`, `-core-video`, +`-core-foundation`). The VAAPI backend hand-writes its FFI, and the difference +is deliberate: libva must be **dlopen'd** so a machine without it degrades to +"no decoder" rather than failing to start, which rules out ordinary generated +bindings. The Apple frameworks are guaranteed present and link normally, so +the maintained bindings win — `CFRetained` gives Create/Get-rule reference +counting as RAII instead of hand-paired `CFRetain`/`CFRelease`, and the +`#[link(kind = "framework")]` attributes live upstream. They are taken with +`default-features = false` and only the per-header features used, so no +Objective-C runtime, Metal, OpenGL or CoreAudio is compiled in; the only +transitive addition is `bitflags`. ## VAAPI backend (linux-gnu) @@ -44,7 +92,16 @@ justification in [`docs/SUPPORT.md`](../../docs/SUPPORT.md). | Feature | Meaning | | --- | --- | -| `hw` | Portable: select the native backend for the compile target (VAAPI on linux-gnu; build-script warning + stub on targets with no hardware decode API). | +| `hw` | Portable: select the native backend for the compile target (VideoToolbox on macOS/iOS, VAAPI on linux-gnu; build-script warning + stub on targets with no hardware decode API). | | `videotoolbox` | Pin VideoToolbox; `compile_error!` on non-Apple targets. | | `vaapi` | Pin VAAPI; `compile_error!` off linux-gnu. | | `mediacodec` | Pin MediaCodec; `compile_error!` off Android. | + +## Testing + +The device tests (`tests/videotoolbox_device.rs`, `tests/vaapi_device.rs`) run +real decodes and skip gracefully when the machine has no decoder for the +codec, so CI stays green without ever exercising hardware. That means CI does +**not** cover decode correctness: run `just test-hw` locally, which is the +documented pre-release gate (see "Pre-release hardware verification" in +`DEVELOPMENT.md`). diff --git a/justfile b/justfile index 6ff4ef2..b84b0d9 100644 --- a/justfile +++ b/justfile @@ -51,6 +51,24 @@ test-features features: test-all: cargo test --workspace --features rawshift-image/full +# Run the hardware decode tests against this machine's real decoder. +# +# CI cannot do this — GitHub runners have no dependable hardware decode block, +# and every hardware test skips silently without one, so a CI job would be +# green without decoding anything. This is the maintainer's pre-release gate +# instead; see "Pre-release hardware verification" in DEVELOPMENT.md. +# +# Needs ffmpeg (bitstream fixtures) and, for the end-to-end HEIC test, +# heif-enc. macOS: `brew install ffmpeg libheif libavif`. +# Prints a summary; tests that skip say so on stderr, so read the output rather +# than trusting the exit code alone. +test-hw: + @echo "== backend probe + hardware decode tests ==" + cargo test -p rawshift-hwdec --features hw -- --nocapture --test-threads=1 + @echo "== end-to-end HEIC/AVIF pixel decode ==" + cargo test -p rawshift-image --features full \ + --test heic_hw_decode --test avif_hw_decode -- --nocapture + # Generate docs for the whole workspace doc: cargo doc --workspace --no-deps --open From f70ea41c1d14215f6e928c5084b7074a01ce7284 Mon Sep 17 00:00:00 2001 From: Justin Chung <20733699+justin13888@users.noreply.github.com> Date: Sat, 22 Aug 2026 05:48:24 -0400 Subject: [PATCH 6/6] test(hwdec): cover mixed geometry through one VideoToolbox decoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A decoder is opened per codec, not per picture, so it can be handed pictures of different sizes — a HEIF primary image and its thumbnail are exactly that. That drives the session cache's second path, where the configuration record differs and `VTDecompressionSessionCanAcceptFormatDescription` decides whether the session is reused or rebuilt; nothing exercised it before. Alternates 128x96 and 64x64 three times so the cache has to switch in both directions rather than settling on one geometry. Claude-Session: https://claude.ai/code/session_01YTL5nD4tjuppsDGMRFFEoh --- .../tests/videotoolbox_device.rs | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/crates/rawshift-hwdec/tests/videotoolbox_device.rs b/crates/rawshift-hwdec/tests/videotoolbox_device.rs index 32e9d6c..ee3a84a 100644 --- a/crates/rawshift-hwdec/tests/videotoolbox_device.rs +++ b/crates/rawshift-hwdec/tests/videotoolbox_device.rs @@ -464,7 +464,50 @@ fn videotoolbox_reuses_one_session_across_many_decodes() { eprintln!("VideoToolbox session reuse: 64 decodes produced identical frames"); } -// ── (g) colour range ──────────────────────────────────────────────────────── +// ── (g) mixed geometry through one decoder ────────────────────────────────── + +/// A decoder is opened per codec, not per picture, so one can be handed +/// pictures of different sizes — a HEIF file's primary image and its thumbnail +/// are exactly that. This drives the session cache's second path, where the +/// configuration record differs and the decoder is asked whether it can accept +/// the new format description before the session is rebuilt. +#[test] +fn videotoolbox_decodes_mixed_geometry_through_one_decoder() { + let Some((big_hvcc, big_payload)) = generate_hevc("128x96", "yuv420p", None, "mixed_big") + else { + return; + }; + let Some((small_hvcc, small_payload)) = generate_hevc("64x64", "yuv420p", None, "mixed_small") + else { + return; + }; + let Some(mut decoder) = device_or_skip(HwCodec::Hevc) else { + return; + }; + let decode = |decoder: &mut Box, hvcc: &[u8], payload: &[u8]| { + decoder + .decode_still(&StillDecodeRequest { + config: CodecConfig::Hvcc(hvcc), + payload, + width: 0, + height: 0, + bit_depth: 8, + chroma: ChromaSubsampling::Cs420, + }) + .expect("hardware HEVC decode") + }; + // Alternate, so the cache is forced to switch in both directions rather + // than settling on one geometry. + for round in 0..3 { + let big = decode(&mut decoder, &big_hvcc, &big_payload); + assert_eq!((big.width(), big.height()), (128, 96), "round {round}"); + let small = decode(&mut decoder, &small_hvcc, &small_payload); + assert_eq!((small.width(), small.height()), (64, 64), "round {round}"); + } + eprintln!("VideoToolbox mixed geometry: 128x96 and 64x64 alternated through one decoder"); +} + +// ── (h) colour range ──────────────────────────────────────────────────────── /// The backend derives `ColorRange` from the emitted surface's four-character /// code (`420v` vs `420f`) rather than from the SPS. This checks the two @@ -501,7 +544,7 @@ fn videotoolbox_reports_a_colour_range() { eprintln!("VideoToolbox colour range: tv -> {tv:?}, pc -> {pc:?}"); } -// ── (h) error path on real hardware ───────────────────────────────────────── +// ── (i) error path on real hardware ───────────────────────────────────────── /// Garbage input must fail with an error, not a panic and not a blank frame. #[test]