fix(apa-48): ship the parse runtime in the worker image; harness faults are not bad documents

.dockerignore

@@ -0,0 +1,44 @@
+# Docker build context hygiene for the repo-root worker build.
+#
+# The worker image builds from the REPO ROOT (see apps/api/Dockerfile.worker):
+# the Go module is apps/api, but the parse runtime needs the repo-root shim
+# (tools/) and the pinned Python dependency source (pyproject.toml, uv.lock).
+#
+# Everything below is excluded so the build context stays small, secret-free
+# and reproducible. The negative-pattern intent is deny-by-default: only what
+# the worker image actually needs may enter the context.
+
+# VCS + local state
+.git
+.gitignore
+.github
+
+# Secrets and environments. .env must never reach a build layer.
+.env
+.env.*
+.venv
+venv
+
+# Tooling caches
+.mypy_cache
+.ruff_cache
+.pytest_cache
+**/__pycache__
+**/*.pyc
+
+# Test + evaluation material. The container smoke MOUNTS the corpus
+# read-only at run time; fixtures are never baked into the production image.
+fixtures
+tests
+eval
+mocks
+docs
+workflows
+infra
+
+# Docs and non-build root files
+prd.md
+README.md
+AGENTS.md
+skills-lock.json
+Makefile

.github/workflows/ci.yml

@@ -6,9 +6,13 @@
 #   emulators are unreachable (see requirePool in
 #   apps/api/internal/repository/postgres/postgres_test.go and the Makefile
 #   header: "integration-gated tests skip gracefully ... exit 0").
-# - The LiteParse live smoke below skips when the `liteparse` Python package
-#   is absent (see liveShim in
-#   apps/api/internal/parser/liteparse/live_test.go).
+#
+# The LiteParse live smoke is the exception and no longer self-skips here:
+# it runs AFTER `uv sync`, with CLAIMOPS_REQUIRE_LITEPARSE=1, so an absent
+# or broken `liteparse` FAILS the job. Before APA-48 it ran before the
+# Python setup and skipped on every run, which is how a worker image with
+# no interpreter shipped green (see liveShim in
+# apps/api/internal/parser/liteparse/live_test.go).
 #
 # Full parser benchmark (#31: Python + liteparse + 45 PDFs, ~1-2 min) is
 # intentionally NOT run here. The benchmark-smoke step runs only the gated
@@ -63,13 +67,6 @@ jobs:
         # package list down to something that excludes them.
         run: go test ./...
 
-      - name: benchmark smoke (live shim only, skips without liteparse)
-        working-directory: apps/api
-        # Gated smoke from #31: real shim + mapping + Validate path against
-        # CASE-001, or a clean SKIP when liteparse is not installed. This is
-        # the smoke gate, not the fidelity benchmark.
-        run: go test ./internal/parser/liteparse/ -run TestLiveShimSmoke -count=1 -v
-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -84,6 +81,26 @@ jobs:
         # `dev` dependency-group in pyproject.toml.
         run: uv sync --group dev
 
+      # APA-48: this gate used to run BEFORE `Set up Python` / `uv sync`, so
+      # liteparse was never importable and TestLiveShimSmoke SKIPPED on every
+      # run — a permanently green pipeline while every ingested document
+      # terminal-FAILED at parse in production. It now runs after the Python
+      # environment exists, and CLAIMOPS_REQUIRE_LITEPARSE=1 makes a broken
+      # parser environment a hard failure instead of a skip.
+      #
+      # Kept out of `go test ./...` on purpose: a skip-then-succeed interaction
+      # there would hide the gate behind an unrelated package failure. As its
+      # own step, a missing runtime fails THIS job loudly.
+      - name: benchmark smoke (live shim, fails if parser env is broken)
+        working-directory: apps/api
+        # Gated smoke from #31: real shim + mapping + Validate path against
+        # CASE-001, asserting pages, tables, bounding boxes, block IDs, block
+        # types, parser metadata and reading-order evidence. This is the
+        # smoke gate, not the fidelity benchmark.
+        env:
+          CLAIMOPS_REQUIRE_LITEPARSE: '1'
+        run: go test ./internal/parser/liteparse/ -run TestLiveShimSmoke -count=1 -v
+
       - name: Python unit tests
         run: uv run pytest tests/unit -q
 

apps/api/Dockerfile.worker

@@ -1,23 +1,74 @@
 # ClaimOps Worker — production image (Cloud Run contract).
 # Serves POST /events/document-ingested for Pub/Sub push delivery.
-# Build from the module root: docker build -f Dockerfile.worker .
-# Alpine-only, squeezed: static binary on minimal Alpine, non-root,
-# ca-certificates for TLS. No shell tools in the final layer.
+#
+# Build from the REPO ROOT (context changed in APA-48):
+#   docker build -f apps/api/Dockerfile.worker -t claimops-worker .
+#
+# The context is the repo root, not the module root, because the deployed
+# worker needs two things that live outside apps/api:
+#   1. tools/parsers/liteparse/shim.py — the parse shim, which
+#      internal/parser/liteparse resolves as DefaultShimPath relative to
+#      WORKDIR, i.e. /app/tools/parsers/liteparse/shim.py, and
+#   2. pyproject.toml + uv.lock — the pinned LiteParse dependency source.
+# Everything the Go stage needs is still scoped to apps/api/, so the Go
+# build is byte-for-byte the same as before.
+#
+# APA-48: the image previously shipped the static Go binary on bare Alpine
+# with no interpreter and no tools/, so internal/parser/liteparse
+# client.go exec of "python3 tools/parsers/liteparse/shim.py" failed at
+# cmd.Start() and EVERY ingested document died terminal-FAILED at parse.
+# The parse runtime is now part of the artifact, and the container smoke
+# (tools/parsers/liteparse/container_smoke.sh) gates it inside this image.
+
+# ---------- Stage 1: static Go binary ----------
 FROM golang:1.27-alpine AS build
 RUN apk add --no-cache git ca-certificates
 WORKDIR /src
-COPY go.mod go.sum ./
+COPY apps/api/go.mod apps/api/go.sum ./
 RUN go mod download
-COPY . .
+COPY apps/api/ ./
 RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker \
  && ls -la /out/worker
 
+# ---------- Stage 2: pinned LiteParse runtime ----------
+# Isolated so no pip cache, no build tooling, and no other project
+# dependency lands in the final image. Only the one wheel the parse shim
+# imports is installed: the shim needs liteparse and nothing else, and the
+# worker image stays "minimum" as the header requires.
+FROM alpine:3.21 AS pybuild
+RUN apk add --no-cache python3 py3-pip
+COPY pyproject.toml uv.lock /src/
+# Resolve the wheel straight out of uv.lock instead of re-typing a URL or
+# version here, so uv.lock stays the single source of truth and cannot
+# drift from the image. musllinux is the Alpine (musl) build; cp310-abi3
+# is satisfied by Alpine's Python 3.12.
+RUN python3 - <<'PY'
+import pathlib, tomllib
+lock = tomllib.loads(pathlib.Path("/src/uv.lock").read_text())
+wheels = [w for pkg in lock["package"] if pkg["name"] == "liteparse" for w in pkg["wheels"]]
+match = [w for w in wheels if "musllinux_1_2_x86_64" in w["url"]]
+if len(match) != 1:
+    raise SystemExit(f"uv.lock: expected exactly 1 musllinux liteparse wheel, found {len(match)}")
+w = match[0]
+pathlib.Path("/tmp/req.txt").write_text(f"{w['url']}#sha256={w['hash'].split(':', 1)[1]}\n")
+print("locked:", w["url"].rsplit('/', 1)[-1], w["hash"])
+PY
+RUN pip install --no-cache-dir --break-system-packages --target=/opt/lp -r /tmp/req.txt \
+ && PYTHONPATH=/opt/lp python3 -c "import liteparse; print('build-stage liteparse import OK')"
+
+# ---------- Stage 3: the deployed artifact ----------
 FROM alpine:3.21
-RUN apk add --no-cache ca-certificates \
+RUN apk add --no-cache ca-certificates python3 \
  && adduser -D -H -u 10001 apprunner \
  && rm -rf /var/cache/apk/*
 WORKDIR /app
 COPY --from=build /out/worker /app/worker
+# Python site-packages only — the stdlib comes from the apk python3 package
+# already installed above, so nothing from pybuild but the wheel is copied.
+COPY --from=pybuild /opt/lp/ /usr/lib/python3.12/site-packages/
+# The shim at the exact path DefaultShimPath resolves to from WORKDIR /app.
+# Copied from repo source so the image can never run a stale shim.
+COPY tools/parsers/liteparse/shim.py /app/tools/parsers/liteparse/shim.py
 ENV PORT=8080 APP_ENV=prod PUSH_AUTH_MODE=oidc
 EXPOSE 8080
 USER apprunner

apps/api/internal/parser/liteparse/client.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"os"
 	"os/exec"
 	"time"
 )
@@ -54,6 +55,16 @@ func NewExecRunner(pythonBin, shimPath string, timeout time.Duration) *ExecRunne
 
 // Run implements Runner.
 func (r *ExecRunner) Run(ctx context.Context, pdfPath string) ([]byte, error) {
+	// Pre-flight the shim. A shim that is not in the image is a misbuilt
+	// artifact, not a bad document: without this the interpreter starts,
+	// exits 2 with "can't open file" on stderr, and mapExitError files it
+	// as a document fault. Checking first keeps the runtime class explicit
+	// and avoids paying for a subprocess that cannot succeed.
+	if _, err := os.Stat(r.shimPath); err != nil {
+		return nil, fmt.Errorf("%w: liteparse shim not readable at %q: %v",
+			ErrRuntimeUnavailable, r.shimPath, err)
+	}
+
 	runCtx := ctx
 	cancel := context.CancelFunc(func() {})
 	if _, hasDeadline := ctx.Deadline(); !hasDeadline {
@@ -70,7 +81,7 @@ func (r *ExecRunner) Run(ctx context.Context, pdfPath string) ([]byte, error) {
 	cmd.Stderr = &stderr
 
 	if err := cmd.Start(); err != nil {
-		return nil, wrapExec("start", err, nil, ctx)
+		return nil, wrapSpawn(err, ctx)
 	}
 	// Bound stdout: read at most MaxStdoutBytes+1 to detect overflow.
 	limited := io.LimitReader(stdout, MaxStdoutBytes+1)

apps/api/internal/parser/liteparse/errors.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os/exec"
 	"strings"
 
 	"claimops-api/internal/parser"
@@ -14,6 +15,26 @@ import (
 // failure, not a caller media error).
 var errShimOverflow = errors.New("liteparse: shim output overflow")
 
+// ErrRuntimeUnavailable marks a LiteParse EXECUTION-ENVIRONMENT fault: the
+// interpreter could not be resolved, the shim is not present at
+// DefaultShimPath, or the vendor library failed to import inside the
+// subprocess. The shim reports the in-process cases with envelope code
+// "harness" (shim.py:99, 109, 148).
+//
+// APA-48: this is deliberately NOT parser.ErrParseFailure. ErrParseFailure
+// is the DOCUMENT-fault class — the worker failTerminal()s it, durably
+// recording the customer's file as unparseable. A misbuilt image or an
+// uninstalled dependency is our own fault, says nothing about the
+// document, and may heal on redelivery, so the worker routes this to the
+// existing OutcomeTransient instead. Before the fix these conditions were
+// terminal-FAILED: a worker image shipping without an interpreter burned
+// every ingested document.
+//
+// The sentinel lives here, not in internal/parser, because that package is
+// a sealed contract that must never import adapters (parser.go:304) and
+// enumerates its error set; no other adapter can raise this condition.
+var ErrRuntimeUnavailable = errors.New("liteparse: parser runtime unavailable")
+
 // unsupportedMedia classifies a non-vendor-handled input. Returned BEFORE
 // spawning the shim so unsupported media never pays subprocess cost.
 func unsupportedMedia(mediaType string) error {
@@ -55,6 +76,21 @@ func wrapExec(stage string, err error, stderr []byte, ctx context.Context) error
 	return fmt.Errorf("%w: liteparse shim %s: %v", parser.ErrParseFailure, stage, err)
 }
 
+// wrapSpawn maps a failure to LAUNCH the shim subprocess. An unresolvable
+// interpreter (exec.ErrNotFound — the exact failure of an image built
+// without python3) is an environment fault, not a document fault, so it
+// gets ErrRuntimeUnavailable instead of ErrParseFailure. Cancellation
+// still takes precedence.
+func wrapSpawn(err error, ctx context.Context) error {
+	if ctx.Err() != nil {
+		return wrapCtx(ctx.Err())
+	}
+	if errors.Is(err, exec.ErrNotFound) {
+		return fmt.Errorf("%w: %v", ErrRuntimeUnavailable, err)
+	}
+	return wrapExec("start", err, nil, ctx)
+}
+
 // mapExitError maps a non-zero shim exit (or signal kill) to a typed
 // error. ctx cancellation (including the timeout kill) maps to the ctx
 // error; anything else is a shim crash -> ErrParseFailure with a stderr
@@ -83,9 +119,15 @@ func mapEnvelopeError(code, message string) error {
 		// gate let through). Same taxonomy as the Go pre-spawn gate.
 		return fmt.Errorf("%w: liteparse shim: %s",
 			parser.ErrUnsupportedMediaType, truncate(message, 500))
-	case "parse_failure", "timeout", "harness":
+	case "parse_failure", "timeout":
 		return fmt.Errorf("%w: liteparse shim %s: %s",
 			parser.ErrParseFailure, code, truncate(message, 500))
+	case "harness":
+		// Runtime/environment fault reported from inside the shim (cannot
+		// read input, vendor import failed, payload serialization failed).
+		// Never a document fault — see ErrRuntimeUnavailable.
+		return fmt.Errorf("%w: liteparse shim %s: %s",
+			ErrRuntimeUnavailable, code, truncate(message, 500))
 	default:
 		return fmt.Errorf("%w: liteparse shim unknown error code %q: %s",
 			parser.ErrParseFailure, code, truncate(message, 500))

apps/api/internal/parser/liteparse/errors_test.go

@@ -12,7 +12,14 @@ func TestMapEnvelopeErrorTaxonomy(t *testing.T) {
 	if err := mapEnvelopeError("unsupported_media", "x"); !errors.Is(err, parser.ErrUnsupportedMediaType) {
 		t.Fatalf("unsupported_media = %v", err)
 	}
-	for _, code := range []string{"parse_failure", "timeout", "harness", "bogus"} {
+	// APA-48: "harness" moved OUT of this list. It reports a runtime/
+	// environment fault (shim cannot read input, vendor import failed,
+	// serialization failed), not a defect in the document, so it now maps to
+	// ErrRuntimeUnavailable; the worker routes that to the existing
+	// OutcomeTransient instead of failTerminal, which used to durably record
+	// a good customer's file as unparseable whenever our own image was broken.
+	// See runtime_errors_test.go for the dedicated coverage.
+	for _, code := range []string{"parse_failure", "timeout", "bogus"} {
 		if err := mapEnvelopeError(code, "x"); !errors.Is(err, parser.ErrParseFailure) {
 			t.Fatalf("%s = %v, want ErrParseFailure", code, err)
 		}