fix(ci): make the Atheris fuzz job capable of failing + repair its dead targets (LAB-1140)

.github/workflows/ci.yml

@@ -196,7 +196,7 @@ jobs:
     - name: Scan Python dependencies for CVEs
       run: |
         # No suppressions: every prior CVE is resolved at source on the py3.10+
-        # resolution. urllib3>=2.7.0 and pip>=26.1.2 are pinned via
+        # resolution. urllib3>=2.7.0 and pip>=26.2 are floored via
         # [tool.uv] constraint-dependencies; pygments/pyarrow advisories cleared
         # by their py3.10+ fix versions. Keep this list IDENTICAL to
         # security-fast.yml's pip-audit so the two cannot drift.

.github/workflows/security-deep.yml

@@ -147,31 +147,58 @@ jobs:
       run: |
         uv sync --group dev --group fuzz --python 3.11
 
+    # Exit-code contract — no `|| true`, every non-zero exit fails the job.
+    # (scripts/fuzz-python.sh carries the same contract for local runs; keep
+    # the budgets and flags in the two files in sync.)
+    #   - Pass is libFuzzer exiting 0 when -max_total_time=600 expires cleanly.
+    #   - A crash, OOM, or uncaught Python exception exits non-zero and fails
+    #     the job. atheris.Setup(sys.argv, ...) forwards flags to libFuzzer, so
+    #     -artifact_prefix drops the reproducer in tests/fuzzing/artifacts/.
+    #     No corpus dir is passed, so that directory only ever holds crash
+    #     artifacts — a file there is always a finding, never corpus growth.
+    #   - A hung input is caught by libFuzzer's own per-input watchdog
+    #     (-timeout=60), which writes a timeout-* reproducer and exits
+    #     non-zero. `timeout -k 30s 15m` is the backstop for hangs in native
+    #     code where that watchdog can't fire: exit 124 is a deliberate
+    #     FAILURE, not budget exhaustion — the fuzz budget is 600 s and budget
+    #     exhaustion exits 0 above, so a target still alive at 15 min is hung
+    #     (and libFuzzer traps SIGTERM, hence the -k hard kill).
+    #   - Zero matched targets fails the job: a glob that stops matching must
+    #     not "pass" having fuzzed nothing (LAB-1136 post-mortem pattern).
     - name: Run Atheris fuzz targets (10 min each)
       run: |
-        for fuzz_target in tests/fuzzing/fuzz_*.py; do
-          if [ -f "$fuzz_target" ]; then
-            echo "Fuzzing $fuzz_target..."
-            timeout 10m uv run --python 3.11 python "$fuzz_target" -max_total_time=600 || true
-          fi
+        mkdir -p tests/fuzzing/artifacts
+        shopt -s nullglob
+        targets=(tests/fuzzing/fuzz_*.py)
+        if [ "${#targets[@]}" -eq 0 ]; then
+          echo "::error::no Atheris targets matched tests/fuzzing/fuzz_*.py — refusing to pass having fuzzed nothing"
+          exit 1
+        fi
+        for fuzz_target in "${targets[@]}"; do
+          echo "Fuzzing $fuzz_target..."
+          timeout -k 30s 15m uv run --python 3.11 python "$fuzz_target" \
+            -max_total_time=600 -timeout=60 -artifact_prefix=tests/fuzzing/artifacts/
         done
 
-    - name: Upload crash corpus
-      if: always()
+    # No "Report fuzzing results" step: now that `|| true` is gone, a crash
+    # fails its own fuzz step, so a trailing check could only ever run in the
+    # no-crash case and print ✅ — a named green step incapable of failing,
+    # the manufactured-evidence pattern the Extended Fuzzing job above already
+    # removed. (The old gate also inspected tests/fuzzing/corpus/, a directory
+    # nothing ever wrote to.) The fuzz steps' exit codes are the signal; this
+    # upload preserves the reproducer, which would otherwise die with the
+    # ephemeral runner.
+    - name: Upload crash artifacts
+      if: failure()
       uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
       with:
-        name: atheris-crashes-${{ github.run_id }}
-        path: tests/fuzzing/corpus/
-
-    - name: Report fuzzing results
-      if: always()
-      run: |
-        if [ -d tests/fuzzing/corpus/ ] && find tests/fuzzing/corpus/ -mindepth 1 ! -name '.gitignore' -print -quit | grep -q .; then
-          echo "⚠️  Crashes discovered during Atheris fuzzing!"
-          ls -lh tests/fuzzing/corpus/
-          exit 1
-        fi
-        echo "✅ No crashes discovered during Atheris fuzzing"
+        name: atheris-crash-artifacts
+        path: tests/fuzzing/artifacts/
+        # Public repo: a crash reproducer is a ready-made PoC for anyone while
+        # it is retrievable. 7 days covers triage of a nightly red without
+        # leaving a month-long public exploit window.
+        retention-days: 7
+        if-no-files-found: warn
 
   miri-full:
     name: Miri Full Suite

.github/workflows/security-fast.yml

@@ -91,7 +91,7 @@ jobs:
     - name: Run pip-audit
       run: |
         # No suppressions: every prior CVE is resolved at source on the py3.10+
-        # resolution. urllib3>=2.7.0 and pip>=26.1.2 are pinned via
+        # resolution. urllib3>=2.7.0 and pip>=26.2 are floored via
         # [tool.uv] constraint-dependencies; pygments/pyarrow advisories cleared
         # by their py3.10+ fix versions. Keep this list IDENTICAL to ci.yml's
         # post-merge pip-audit so the two cannot drift.

.gitignore

@@ -73,6 +73,9 @@ target/
 benchmark_results/
 *.benchmark
 
+# Atheris/libFuzzer crash reproducers (see -artifact_prefix in scripts/fuzz-python.sh)
+tests/fuzzing/artifacts/
+
 # Temporary files
 *.log
 logs/

pyproject.toml

@@ -247,10 +247,12 @@ constraint-dependencies = [
     "urllib3>=2.7.0",
     "fonttools>=4.60.2",
     "werkzeug>=3.1.4",
-    # pip is a dev-only transitive dep (pip-audit -> pip-api -> pip). 26.1.2 fixes
+    # pip is a dev-only transitive dep (pip-audit -> pip-api -> pip). 26.1.2 fixed
     # PYSEC-2026-196 (entry-point path traversal), GHSA-58qw-9mgm-455v (tar/zip
-    # confusion) and GHSA-jp4c-xjxw-mgf9 (self-update import ordering).
-    "pip>=26.1.2",
+    # confusion) and GHSA-jp4c-xjxw-mgf9 (self-update import ordering); 26.2 adds
+    # PYSEC-2026-3721 (doubly-encoded index URLs escaping the target directory —
+    # arbitrary write on install from a malicious index).
+    "pip>=26.2",
     # h2 is a transitive dep (httpx[http2] -> h2). 4.4.1 fixes
     # GHSA-6hr6-w5qg-qmwg (duplicate Host headers forwarded on HTTP/2 ->
     # HTTP/1.1 downgrade — request smuggling primitive).

scripts/fuzz-python.sh

@@ -6,15 +6,34 @@ set -euo pipefail
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_common.sh"
 
-if command -v python &>/dev/null && python -c "import atheris" 2>/dev/null; then
-	echo "${BLUE}Running Atheris fuzzing...${RESET}"
-	for fuzz_target in tests/fuzzing/fuzz_*.py; do
-		if [ -f "$fuzz_target" ]; then
-			echo "${YELLOW}Fuzzing $fuzz_target...${RESET}"
-			timeout 10m uv run python "$fuzz_target" -max_total_time=600 || true
-		fi
-	done
-else
-	echo "${YELLOW}⚠️  Atheris not available (macOS limitation - libFuzzer not in Apple Clang)${RESET}"
-	echo "${YELLOW}   Atheris fuzzing will run in CI on Linux${RESET}"
+# macOS is the ONLY soft skip (Apple Clang ships no libFuzzer, so atheris
+# cannot work there). Everywhere else the targets run unconditionally: if
+# atheris is missing, the target's own import fails loudly and reds the run —
+# probing for it first and skipping (the old behavior) was a silent green on
+# Linux, the exact lie this contract exists to kill (LAB-1140).
+if [ "$(uname -s)" = "Darwin" ]; then
+	echo "${YELLOW}⚠️  Skipping Atheris fuzzing (macOS limitation - libFuzzer not in Apple Clang)${RESET}"
+	echo "${YELLOW}   Atheris fuzzing runs in CI on Linux${RESET}"
+	exit 0
 fi
+
+echo "${BLUE}Running Atheris fuzzing...${RESET}"
+# Same exit-code contract as the atheris-fuzzing job in security-deep.yml
+# (LAB-1140) — keep budgets/flags in sync with it: no `|| true`; a crash,
+# import error, or hang fails the run (libFuzzer's -timeout=60 per-input
+# watchdog writes a timeout-* reproducer; `timeout -k 30s 15m` is the
+# backstop for native hangs, since libFuzzer traps SIGTERM; budget exhaustion
+# exits 0 well before 15 min). Reproducers land in tests/fuzzing/artifacts/
+# (gitignored), which only ever holds crash artifacts — no corpus dir is
+# passed.
+mkdir -p tests/fuzzing/artifacts
+shopt -s nullglob
+targets=(tests/fuzzing/fuzz_*.py)
+if [ "${#targets[@]}" -eq 0 ]; then
+	echo "${YELLOW}no Atheris targets matched tests/fuzzing/fuzz_*.py — refusing to pass having fuzzed nothing${RESET}" >&2
+	exit 1
+fi
+for fuzz_target in "${targets[@]}"; do
+	echo "${YELLOW}Fuzzing $fuzz_target...${RESET}"
+	timeout -k 30s 15m uv run python "$fuzz_target" -max_total_time=600 -timeout=60 -artifact_prefix=tests/fuzzing/artifacts/
+done

tests/fuzzing/fuzz_byte_storage.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Atheris fuzz target for ByteStorage (Python → Rust FFI boundary).
+
+This guards the FFI binding contract: roundtrip fidelity, and that hostile
+envelopes surface as clean ValueError — never an interpreter crash or a
+pyo3 PanicException. Coverage-guided exploration of the Rust decode surface
+itself lives in `cargo fuzz run byte_storage_decompress` (rust/fuzz/); the
+native .so is invisible to Atheris' bytecode instrumentation, so this
+target's mutation is unguided by design.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import importlib
+import sys
+
+import atheris
+
+# Pre-import third-party deps so instrument_imports() below skips them
+# (already in sys.modules = not instrumented) — we fuzz cachekit's code;
+# third-party coverage is not the goal, and atheris-instrumented third-party
+# bytecode is a proven startup-crash class: instrumented pydantic segfaults
+# CPython 3.11 in _decorators.merge_seqs during pydantic_settings'
+# CLI-provider model construction (pulled in transitively via
+# cachekit.hiredis_compat). That SIGSEGV killed every nightly target during
+# startup, before a single fuzz iteration (LAB-1140/LAB-2528). Optional deps
+# use suppress: absent is fine, instrumented is the trap.
+for _mod in (
+    "pydantic",
+    "pydantic_settings",
+    "numpy",
+    "pandas",
+    "pyarrow",
+    "redis",
+    "msgpack",
+    "xxhash",
+    "prometheus_client",
+):
+    with contextlib.suppress(ImportError):
+        importlib.import_module(_mod)
+
+with atheris.instrument_imports():
+    from cachekit._rust_serializer import ByteStorage
+
+_STORAGE = ByteStorage("msgpack")
+
+
+def TestOneInput(data: bytes) -> None:
+    """Fuzz the ByteStorage store/retrieve FFI roundtrip and hostile-envelope decode."""
+    fdp = atheris.FuzzedDataProvider(data)
+
+    if fdp.ConsumeBool():
+        # Roundtrip: store must retrieve byte-identically.
+        payload = bytes(fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, 4096)))
+        envelope = _STORAGE.store(payload, "msgpack")
+        retrieved, fmt = _STORAGE.retrieve(envelope)
+        # Explicit raise, not assert: -O / PYTHONOPTIMIZE strips assert, which
+        # would leave this target reporting no crashes while verifying nothing.
+        if bytes(retrieved) != payload:
+            raise AssertionError("ByteStorage roundtrip failed")
+        if fmt != "msgpack":
+            raise AssertionError(f"format tag corrupted: {fmt}")
+    else:
+        # Attacker-controlled envelope (cache content is untrusted): must
+        # raise cleanly, never crash the interpreter.
+        try:
+            _STORAGE.retrieve(bytes(fdp.ConsumeBytes(fdp.remaining_bytes())))
+        except ValueError:
+            pass
+
+
+if __name__ == "__main__":
+    atheris.Setup(sys.argv, TestOneInput)
+    atheris.Fuzz()

tests/fuzzing/fuzz_decorator_stack.py

@@ -1,51 +1,70 @@
 #!/usr/bin/env python3
-"""Atheris fuzz target for decorator stack (full integration)."""
+"""Atheris fuzz target for the cache decorator stack (L1-only integration)."""
 
 from __future__ import annotations
 
+import contextlib
+import importlib
 import sys
 
 import atheris
 
+# Pre-import third-party deps so instrument_imports() below skips them
+# (already in sys.modules = not instrumented) — we fuzz cachekit's code;
+# third-party coverage is not the goal, and atheris-instrumented third-party
+# bytecode is a proven startup-crash class: instrumented pydantic segfaults
+# CPython 3.11 in _decorators.merge_seqs during pydantic_settings'
+# CLI-provider model construction (pulled in transitively via
+# cachekit.hiredis_compat). That SIGSEGV killed every nightly target during
+# startup, before a single fuzz iteration (LAB-1140/LAB-2528). Optional deps
+# use suppress: absent is fine, instrumented is the trap.
+for _mod in (
+    "pydantic",
+    "pydantic_settings",
+    "numpy",
+    "pandas",
+    "pyarrow",
+    "redis",
+    "msgpack",
+    "xxhash",
+    "prometheus_client",
+):
+    with contextlib.suppress(ImportError):
+        importlib.import_module(_mod)
+
 with atheris.instrument_imports():
-    from cachekit.decorators.main import redis_cache
-    from cachekit.serializers.raw import RawSerializer
+    from cachekit import cache
+    from cachekit.config import DecoratorConfig
+    from cachekit.config.nested import L1CacheConfig
+
+
+# L1-only (backend=None): no network, deterministic — exercises the
+# decorator / key-generation / ObjectCache (L1) stack on every call. In this
+# mode values are stored as raw Python objects (no serializer runs).
+#
+# max_size_mb=8: L1's byte accounting counts only getsizeof(value); the
+# ~450 B of real per-entry overhead (key string + entry bookkeeping) is
+# uncounted, so the default 100 MB budget reaches ~1.5 GB real RSS over a
+# 600 s run — inside libFuzzer's default -rss_limit_mb=2048 OOM kill. 8 MB
+# accounted keeps real RSS comfortably bounded.
+@cache(config=DecoratorConfig(backend=None, l1=L1CacheConfig(max_size_mb=8, swr_enabled=False)))
+def _cached_identity(value: bytes) -> bytes:
+    return value
 
 
 def TestOneInput(data: bytes) -> None:
-    """Fuzz the complete cache decorator stack."""
+    """Fuzz the decorator stack: both calls must roundtrip byte-identically."""
     fdp = atheris.FuzzedDataProvider(data)
+    payload = bytes(fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, 1024)))
 
-    try:
-        # Generate test function names to avoid collision
-        func_id = fdp.ConsumeIntInRange(0, 1000000)
-
-        # Create a simple cached function with RawSerializer
-        @redis_cache(
-            redis_url="redis://localhost:6379",
-            serializer=RawSerializer(),
-            default_ttl=3600,
-        )
-        def cached_func(value: bytes) -> bytes:
-            """Simple cached function that returns input."""
-            return value
-
-        # Test with random payload
-        payload = fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, 1024))
-
-        # Attempt to call the function
-        # May fail if Redis is unavailable, which is expected
-        try:
-            result = cached_func(payload)
-            # If it works, verify roundtrip
-            assert result == payload, "Decorator roundtrip failed"
-        except (ConnectionError, TimeoutError, OSError):
-            # Expected when Redis is unavailable
-            pass
-
-    except (ValueError, OverflowError, RuntimeError, AttributeError, TypeError):
-        # Expected exceptions for malformed input or missing Redis
-        pass
+    # First call stores (or hits L1); the repeat exercises the hit path when
+    # the entry survived eviction. Both must return the payload unchanged.
+    # Explicit raise, not assert: -O / PYTHONOPTIMIZE strips assert, which would
+    # leave this target reporting no crashes while verifying nothing.
+    if _cached_identity(payload) != payload:
+        raise AssertionError("Decorator roundtrip failed (first call)")
+    if _cached_identity(payload) != payload:
+        raise AssertionError("Decorator roundtrip failed (repeat call)")
 
 
 if __name__ == "__main__":