diff --git a/AGENTS.md b/AGENTS.md index fda08b23c..761969ae6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,40 @@ pre-commit run --all-files Fix any failures before committing. +## Test Coverage + +Always prefer Python coverage — a doctest example in a docstring, or a pytest +case. The user-facing Python surface is the first line of defense and the +primary focus, so behavior should be pinned where users actually meet it. + +**CI does not run Rust tests.** No workflow invokes `cargo test`; the only +Rust checks are `cargo fmt --check` and +`cargo clippy --no-deps --all-targets`. `--all-targets` compiles +`#[cfg(test)]` code, so a Rust test cannot rot into a non-compiling state, but +it is never executed and a behavioral regression will not fail the build. A +Rust test added today is dead weight. + +Adding a `cargo test` job is not a one-line change: `crates/core/Cargo.toml` +enables `pyo3/extension-module` unconditionally, so the test binary fails to +link against `Py_*` symbols on Linux. The feature would have to be gated first. + +Write a Rust test only when the behavior is genuinely unreachable from Python, +and wire up CI in the same change so it actually runs. Before concluding it is +unreachable, check the suites that already exist: + +- `python/tests/` — the main suite. Run `pytest python/`, **not** + `pytest python/tests/`: `--doctest-modules` is on by default and the + narrower path skips the doctests in `python/datafusion/`. +- `examples/datafusion-ffi-example/python/tests/` and + `examples/datafusion-ffi-query-planner-example/python/tests/` — integration + coverage across a real FFI boundary, for anything involving extension + codecs, table providers, query planners, or capsule export. These need the + example crates built (`maturin build`, then install the wheel). +- `examples/tpch/` — end-to-end query coverage. + +Prefer asserting observable behavior over internal accessors. A test that +checks a getter can pass while the path a user actually takes is broken. + ## Python Function Docstrings Every Python function must include a docstring with usage examples. diff --git a/crates/core/src/codec.rs b/crates/core/src/codec.rs index 26853e69f..c3f78f044 100644 --- a/crates/core/src/codec.rs +++ b/crates/core/src/codec.rs @@ -29,16 +29,16 @@ //! //! [`PythonLogicalCodec`] is the [`LogicalExtensionCodec`] that //! datafusion-python parks on every `SessionContext`. It wraps a -//! user-supplied (or default) inner codec and adds Python-aware -//! in-band encoding on top: when the encoder sees a Python-defined -//! UDF, the codec cloudpickles the callable + signature into the -//! `fun_definition` proto field; when the decoder sees a payload it -//! produced, it reconstructs the UDF from the bytes alone — no -//! pre-registration on the receiver. UDFs the codec does not -//! recognise are delegated to `inner`, which is typically -//! `DefaultLogicalExtensionCodec` but may be a downstream-supplied -//! FFI codec installed via -//! `SessionContext.with_logical_extension_codec(...)`. +//! chain of composable codecs and adds Python-aware in-band encoding +//! on top: when the encoder sees a Python-defined UDF, the codec +//! cloudpickles the callable + signature into the `fun_definition` +//! proto field; when the decoder sees a payload it produced, it +//! reconstructs the UDF from the bytes alone — no pre-registration on +//! the receiver. Everything the codec does not recognise is delegated +//! to the chain: each downstream FFI codec installed via +//! `SessionContext.with_logical_extension_codec(...)` is consulted in +//! most-recently-installed-first order, with +//! `DefaultLogicalExtensionCodec` as the terminal fallback. //! //! [`PythonPhysicalCodec`] is the symmetric wrapper around //! [`PhysicalExtensionCodec`]. Logical and physical layers each have @@ -58,7 +58,7 @@ //! actionable error instead of an opaque `marshal` failure on load //! (cloudpickle payloads are not portable across Python minor //! versions). Dispatch precedence on decode: **family match + -//! supported version + matching Python version → `inner` codec → +//! supported version + matching Python version → codec chain → //! caller's `FunctionRegistry` fallback.** //! //! ## Wire-format family registry @@ -81,10 +81,11 @@ //! for an older shape. //! //! Downstream FFI codecs should pick non-colliding family prefixes -//! (use a `DF` namespace plus a crate-specific suffix). The codec -//! implementations in this module currently delegate every method to -//! `inner`; the encoder/decoder hooks for each kind are added as the -//! corresponding Python-side type becomes serializable. +//! (use a `DF` namespace plus a crate-specific suffix) and return an +//! error for payloads and objects they do not own — that error is the +//! chain's "not mine" signal, letting the next codec take a turn. A +//! codec that answers `Ok` for objects outside its family shadows +//! every codec installed before it. use std::sync::Arc; @@ -167,7 +168,7 @@ fn write_wire_header(buf: &mut Vec, family: &[u8], py_version: (u8, u8)) { /// Inspect the framing on `buf`. /// /// * `Ok(None)` — `buf` does not carry `family`. The caller should -/// delegate to its `inner` codec. +/// delegate to its codec chain. /// * `Ok(Some(payload))` — `buf` carries `family` at a version this /// build accepts and a Python `(major, minor)` matching /// `expected_py`; `payload` is the cloudpickle blob. @@ -223,32 +224,129 @@ fn strip_wire_header<'a>( Ok(Some(&buf[py_minor_idx + 1..])) } +/// Run `f` against each codec in `chain`, returning the first `Ok`. +/// +/// A codec signals "not mine" by returning an error, so the chain +/// keeps trying until a codec succeeds. When every codec fails and the +/// chain has more than one entry, the errors are aggregated into a +/// single message — returning only the last error would surface the +/// terminal `Default*ExtensionCodec` "not provided" message and mask +/// the more specific diagnostic from an installed codec (e.g. a +/// corrupt-token error from the codec that owns the payload family). +fn chain_try(chain: &[Arc], what: &str, f: impl Fn(&C) -> Result) -> Result { + let mut errors: Vec = Vec::new(); + for codec in chain { + match f(codec) { + Ok(value) => return Ok(value), + Err(err) => errors.push(err), + } + } + Err(aggregate_chain_errors(what, errors)) +} + +/// Collapse per-codec failures into one error. A single failure is +/// returned as-is so the one-codec (default-only) chain behaves +/// exactly like the pre-chain implementation. +fn aggregate_chain_errors( + what: &str, + mut errors: Vec, +) -> datafusion::error::DataFusionError { + match errors.len() { + 0 => datafusion::error::DataFusionError::Internal(format!( + "Empty extension codec chain while handling {what}" + )), + 1 => errors.swap_remove(0), + _ => { + let joined = errors + .iter() + .map(|err| err.to_string()) + .collect::>() + .join("; "); + datafusion::error::DataFusionError::Execution(format!( + "None of the {} composed extension codecs handled {what}: {joined}", + errors.len() + )) + } + } +} + +/// Encode variant of [`chain_try`] for methods that write into a +/// caller-provided buffer. +/// +/// Each codec encodes into a scratch buffer so a failed attempt cannot +/// leave partial bytes behind. `Ok` with bytes written commits those +/// bytes and ends the chain. `Ok` with an empty buffer is treated as +/// "no opinion" — the standard `Default*ExtensionCodec` behavior of +/// encoding a UDF by name writes nothing — so later codecs still get a +/// chance to emit a richer payload. If no codec writes bytes but at +/// least one returned `Ok`, the overall result is `Ok` with nothing +/// written (encode by name). +fn chain_encode( + chain: &[Arc], + buf: &mut Vec, + what: &str, + f: impl Fn(&C, &mut Vec) -> Result<()>, +) -> Result<()> { + let mut saw_empty_ok = false; + let mut errors: Vec = Vec::new(); + for codec in chain { + let mut scratch = Vec::new(); + match f(codec, &mut scratch) { + Ok(()) if !scratch.is_empty() => { + buf.extend_from_slice(&scratch); + return Ok(()); + } + Ok(()) => saw_empty_ok = true, + Err(err) => errors.push(err), + } + } + if saw_empty_ok { + return Ok(()); + } + Err(aggregate_chain_errors(what, errors)) +} + /// `LogicalExtensionCodec` parked on every `SessionContext`. Holds /// the Python-aware encoding hooks for logical-layer types /// (`LogicalPlan`, `Expr`) and delegates everything it does not -/// handle to the composable `inner` codec — typically -/// `DefaultLogicalExtensionCodec`, or a downstream FFI codec -/// installed via `SessionContext.with_logical_extension_codec(...)`. +/// handle to a chain of composable codecs. The chain starts as just +/// `DefaultLogicalExtensionCodec`; each downstream FFI codec installed +/// via `SessionContext.with_logical_extension_codec(...)` is prepended, +/// so the most recently installed codec is consulted first and the +/// default codec always runs last. +/// +/// Chain dispatch relies on each codec recognizing its own payloads +/// (distinct family prefixes — see the module docs) and returning an +/// error for everything else so the next codec gets a chance. /// /// Sitting at the top of the session's logical codec stack means /// every serializer that reads `session.logical_codec()` automatically /// picks up Python-aware encoding for free. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PythonLogicalCodec { - inner: Arc, + chain: Vec>, python_udf_inlining: bool, } impl PythonLogicalCodec { pub fn new(inner: Arc) -> Self { Self { - inner, + chain: vec![inner], python_udf_inlining: true, } } - pub fn inner(&self) -> &Arc { - &self.inner + /// Return a copy of this codec with `codec` prepended to the + /// chain, preserving the Python-UDF-inlining setting. The new + /// codec is consulted before every previously installed codec. + pub fn with_additional_codec(&self, codec: Arc) -> Self { + let mut chain = Vec::with_capacity(self.chain.len() + 1); + chain.push(codec); + chain.extend(self.chain.iter().map(Arc::clone)); + Self { + chain, + python_udf_inlining: self.python_udf_inlining, + } } /// Toggle inline encoding of Python UDFs. See @@ -289,11 +387,18 @@ impl LogicalExtensionCodec for PythonLogicalCodec { inputs: &[LogicalPlan], ctx: &TaskContext, ) -> Result { - self.inner.try_decode(buf, inputs, ctx) + chain_try(&self.chain, "an extension logical plan node", |codec| { + codec.try_decode(buf, inputs, ctx) + }) } fn try_encode(&self, node: &Extension, buf: &mut Vec) -> Result<()> { - self.inner.try_encode(node, buf) + chain_encode( + &self.chain, + buf, + "an extension logical plan node", + |codec, buf| codec.try_encode(node, buf), + ) } fn try_decode_table_provider( @@ -303,8 +408,9 @@ impl LogicalExtensionCodec for PythonLogicalCodec { schema: SchemaRef, ctx: &TaskContext, ) -> Result> { - self.inner - .try_decode_table_provider(buf, table_ref, schema, ctx) + chain_try(&self.chain, "a table provider", |codec| { + codec.try_decode_table_provider(buf, table_ref, Arc::clone(&schema), ctx) + }) } fn try_encode_table_provider( @@ -313,7 +419,9 @@ impl LogicalExtensionCodec for PythonLogicalCodec { node: Arc, buf: &mut Vec, ) -> Result<()> { - self.inner.try_encode_table_provider(table_ref, node, buf) + chain_encode(&self.chain, buf, "a table provider", |codec, buf| { + codec.try_encode_table_provider(table_ref, Arc::clone(&node), buf) + }) } fn try_decode_file_format( @@ -321,7 +429,9 @@ impl LogicalExtensionCodec for PythonLogicalCodec { buf: &[u8], ctx: &TaskContext, ) -> Result> { - self.inner.try_decode_file_format(buf, ctx) + chain_try(&self.chain, "a file format", |codec| { + codec.try_decode_file_format(buf, ctx) + }) } fn try_encode_file_format( @@ -329,14 +439,18 @@ impl LogicalExtensionCodec for PythonLogicalCodec { buf: &mut Vec, node: Arc, ) -> Result<()> { - self.inner.try_encode_file_format(buf, node) + chain_encode(&self.chain, buf, "a file format", |codec, buf| { + codec.try_encode_file_format(buf, Arc::clone(&node)) + }) } fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec) -> Result<()> { if self.python_udf_inlining && try_encode_python_scalar_udf(node, buf)? { return Ok(()); } - self.inner.try_encode_udf(node, buf) + chain_encode(&self.chain, buf, "a scalar UDF", |codec, buf| { + codec.try_encode_udf(node, buf) + }) } fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result> { @@ -347,14 +461,18 @@ impl LogicalExtensionCodec for PythonLogicalCodec { } else { refuse_if_inline(buf, PY_SCALAR_UDF_FAMILY, "scalar UDF", name)?; } - self.inner.try_decode_udf(name, buf) + chain_try(&self.chain, "a scalar UDF", |codec| { + codec.try_decode_udf(name, buf) + }) } fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec) -> Result<()> { if self.python_udf_inlining && try_encode_python_udaf(node, buf)? { return Ok(()); } - self.inner.try_encode_udaf(node, buf) + chain_encode(&self.chain, buf, "an aggregate UDF", |codec, buf| { + codec.try_encode_udaf(node, buf) + }) } fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result> { @@ -365,14 +483,18 @@ impl LogicalExtensionCodec for PythonLogicalCodec { } else { refuse_if_inline(buf, PY_AGG_UDF_FAMILY, "aggregate UDF", name)?; } - self.inner.try_decode_udaf(name, buf) + chain_try(&self.chain, "an aggregate UDF", |codec| { + codec.try_decode_udaf(name, buf) + }) } fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec) -> Result<()> { if self.python_udf_inlining && try_encode_python_udwf(node, buf)? { return Ok(()); } - self.inner.try_encode_udwf(node, buf) + chain_encode(&self.chain, buf, "a window UDF", |codec, buf| { + codec.try_encode_udwf(node, buf) + }) } fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result> { @@ -383,13 +505,15 @@ impl LogicalExtensionCodec for PythonLogicalCodec { } else { refuse_if_inline(buf, PY_WINDOW_UDF_FAMILY, "window UDF", name)?; } - self.inner.try_decode_udwf(name, buf) + chain_try(&self.chain, "a window UDF", |codec| { + codec.try_decode_udwf(name, buf) + }) } } /// Strict-mode gate: if `buf` is a well-framed inline payload for /// `family`, return the strict-refusal error; otherwise return -/// `Ok(())` so the caller can delegate to its `inner` codec. +/// `Ok(())` so the caller can delegate to its codec chain. /// /// Routing through [`read_framed_payload`] (rather than a bare /// `starts_with` probe) means malformed inline bytes — wrong @@ -434,7 +558,8 @@ fn refuse_inline_payload(kind: &str, name: &str) -> datafusion::error::DataFusio /// `PhysicalExtensionCodec` mirror of [`PythonLogicalCodec`] parked /// on the same `SessionContext`. Carries the Python-aware encoding /// hooks for physical-layer types (`ExecutionPlan`, `PhysicalExpr`) -/// and delegates the rest to `inner`. +/// and delegates the rest to the composable codec chain (see +/// [`PythonLogicalCodec`] for chain ordering and dispatch rules). /// /// The `PhysicalExtensionCodec` trait has its own `try_encode_udf` /// / `try_decode_udf` pair distinct from the logical one, so a @@ -443,22 +568,31 @@ fn refuse_inline_payload(kind: &str, name: &str) -> datafusion::error::DataFusio /// would round-trip at the logical level but break at the physical /// level. Both layers reuse the shared payload framing /// ([`PY_SCALAR_UDF_FAMILY`] et al.) so the wire format is identical. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PythonPhysicalCodec { - inner: Arc, + chain: Vec>, python_udf_inlining: bool, } impl PythonPhysicalCodec { pub fn new(inner: Arc) -> Self { Self { - inner, + chain: vec![inner], python_udf_inlining: true, } } - pub fn inner(&self) -> &Arc { - &self.inner + /// Return a copy of this codec with `codec` prepended to the + /// chain, preserving the Python-UDF-inlining setting. The new + /// codec is consulted before every previously installed codec. + pub fn with_additional_codec(&self, codec: Arc) -> Self { + let mut chain = Vec::with_capacity(self.chain.len() + 1); + chain.push(codec); + chain.extend(self.chain.iter().map(Arc::clone)); + Self { + chain, + python_udf_inlining: self.python_udf_inlining, + } } /// Toggle inline encoding of Python UDFs on this physical codec. @@ -489,7 +623,9 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { ctx: &TaskContext, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - self.inner.try_decode(buf, inputs, ctx, proto_converter) + chain_try(&self.chain, "an execution plan", |codec| { + codec.try_decode(buf, inputs, ctx, proto_converter) + }) } fn try_encode( @@ -498,14 +634,18 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { buf: &mut Vec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result<()> { - self.inner.try_encode(node, buf, proto_converter) + chain_encode(&self.chain, buf, "an execution plan", |codec, buf| { + codec.try_encode(Arc::clone(&node), buf, proto_converter) + }) } fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec) -> Result<()> { if self.python_udf_inlining && try_encode_python_scalar_udf(node, buf)? { return Ok(()); } - self.inner.try_encode_udf(node, buf) + chain_encode(&self.chain, buf, "a scalar UDF", |codec, buf| { + codec.try_encode_udf(node, buf) + }) } fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result> { @@ -516,7 +656,9 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { } else { refuse_if_inline(buf, PY_SCALAR_UDF_FAMILY, "scalar UDF", name)?; } - self.inner.try_decode_udf(name, buf) + chain_try(&self.chain, "a scalar UDF", |codec| { + codec.try_decode_udf(name, buf) + }) } fn try_encode_expr( @@ -525,7 +667,9 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { buf: &mut Vec, ctx: &PhysicalExprEncodeCtx<'_>, ) -> Result<()> { - self.inner.try_encode_expr(node, buf, ctx) + chain_encode(&self.chain, buf, "a physical expression", |codec, buf| { + codec.try_encode_expr(node, buf, ctx) + }) } fn try_decode_expr( @@ -534,14 +678,18 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { inputs: &[Arc], ctx: &PhysicalExprDecodeCtx<'_>, ) -> Result> { - self.inner.try_decode_expr(buf, inputs, ctx) + chain_try(&self.chain, "a physical expression", |codec| { + codec.try_decode_expr(buf, inputs, ctx) + }) } fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec) -> Result<()> { if self.python_udf_inlining && try_encode_python_udaf(node, buf)? { return Ok(()); } - self.inner.try_encode_udaf(node, buf) + chain_encode(&self.chain, buf, "an aggregate UDF", |codec, buf| { + codec.try_encode_udaf(node, buf) + }) } fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result> { @@ -552,14 +700,18 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { } else { refuse_if_inline(buf, PY_AGG_UDF_FAMILY, "aggregate UDF", name)?; } - self.inner.try_decode_udaf(name, buf) + chain_try(&self.chain, "an aggregate UDF", |codec| { + codec.try_decode_udaf(name, buf) + }) } fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec) -> Result<()> { if self.python_udf_inlining && try_encode_python_udwf(node, buf)? { return Ok(()); } - self.inner.try_encode_udwf(node, buf) + chain_encode(&self.chain, buf, "a window UDF", |codec, buf| { + codec.try_encode_udwf(node, buf) + }) } fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result> { @@ -570,7 +722,9 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { } else { refuse_if_inline(buf, PY_WINDOW_UDF_FAMILY, "window UDF", name)?; } - self.inner.try_decode_udwf(name, buf) + chain_try(&self.chain, "a window UDF", |codec| { + codec.try_decode_udwf(name, buf) + }) } } @@ -587,7 +741,7 @@ impl PhysicalExtensionCodec for PythonPhysicalCodec { /// `Ok(true)` when the payload (`DFPYUDF` family prefix, version byte, /// cloudpickled tuple) was written and the caller should skip its /// inner codec. Returns `Ok(false)` for any non-Python UDF, signalling -/// the caller to delegate to its `inner`. +/// the caller to delegate to its codec chain. pub(crate) fn try_encode_python_scalar_udf(node: &ScalarUDF, buf: &mut Vec) -> Result { let Some(py_udf) = node.inner().downcast_ref::() else { return Ok(false); @@ -602,7 +756,7 @@ pub(crate) fn try_encode_python_scalar_udf(node: &ScalarUDF, buf: &mut Vec) /// Decode an inline Python scalar UDF payload. Returns `Ok(None)` /// when `buf` does not carry the `DFPYUDF` family prefix, signalling -/// the caller to delegate to its `inner` codec (and eventually the +/// the caller to delegate to its codec chain (and eventually the /// `FunctionRegistry`). pub(crate) fn try_decode_python_scalar_udf(buf: &[u8]) -> Result>> { if !buf.starts_with(PY_SCALAR_UDF_FAMILY) { @@ -1028,137 +1182,3 @@ fn decode_python_udaf(py: Python<'_>, payload: &[u8]) -> PyResult PyDataFusionResult { let inner_ffi = ffi_logical_codec_from_pycapsule(codec)?; let inner: Arc = (&inner_ffi).into(); - let logical_codec = Arc::new(PythonLogicalCodec::new(inner)); + // Prepend rather than replace: previously installed codecs stay + // active, with the most recently installed one consulted first. + let logical_codec = Arc::new(self.logical_codec.with_additional_codec(inner)); let physical_codec = Arc::clone(&self.physical_codec); let ctx = self @@ -1497,7 +1499,9 @@ impl PySessionContext { codec: Bound<'py, PyAny>, ) -> PyDataFusionResult { let inner = physical_codec_from_pycapsule(&codec)?; - let physical_codec = Arc::new(PythonPhysicalCodec::new(inner)); + // Prepend rather than replace: previously installed codecs stay + // active, with the most recently installed one consulted first. + let physical_codec = Arc::new(self.physical_codec.with_additional_codec(inner)); let logical_codec = Arc::clone(&self.logical_codec); let ctx = self @@ -1511,11 +1515,15 @@ impl PySessionContext { pub fn with_python_udf_inlining(&self, enabled: bool) -> Self { let logical_codec = Arc::new( - PythonLogicalCodec::new(Arc::clone(self.logical_codec.inner())) + self.logical_codec + .as_ref() + .clone() .with_python_udf_inlining(enabled), ); let physical_codec = Arc::new( - PythonPhysicalCodec::new(Arc::clone(self.physical_codec.inner())) + self.physical_codec + .as_ref() + .clone() .with_python_udf_inlining(enabled), ); let ctx = self diff --git a/docs/source/contributor-guide/ffi.md b/docs/source/contributor-guide/ffi.md index d13eda4d8..3b5b8b91b 100644 --- a/docs/source/contributor-guide/ffi.md +++ b/docs/source/contributor-guide/ffi.md @@ -248,15 +248,65 @@ foreign planner. This lets the planner decode provider-owned objects and lets process-local tokens to demonstrate ownership; production codecs should serialize durable metadata instead. -The current Python API has one external logical codec and one external physical codec. -Installing another codec replaces the prior codec rather than composing a registry. -The example therefore has one external codec owner, and the planner uses built-in -physical nodes. Install the provider codecs before the planner where possible. +### Composable codecs + +Extension codecs compose. Each call to `with_logical_extension_codec` or +`with_physical_extension_codec` adds the codec to the front of the session's codec +chain rather than replacing prior codecs. During encoding and decoding, the most +recently installed codec is consulted first, falling through codec by codec to +DataFusion's default codec. A codec signals "not mine" by returning an error, which +sends the chain on to the next codec. Two conventions keep this dispatch sound: + +- Frame your payloads with a distinct byte prefix (pick a `DF` namespace plus a + crate-specific suffix) and only decode payloads carrying your prefix. +- Return an error for objects and payloads you do not own. A codec that answers + success for objects outside its family shadows every codec installed before it. + +Because dispatch keys off payload prefixes rather than install position, codec +registration order between independent libraries does not matter. The current FFI logical codec supports providers and UDFs but not arbitrary custom `LogicalPlan::Extension` nodes. See both example READMEs for the supported flow and local build commands. +### One planner per session, with explicit fallback + +Unlike codecs, a `SessionState` holds exactly one query planner — installing another +replaces it. Planner layering is therefore explicit: a planner that wants to handle +only some queries should accept a fallback planner and delegate the rest to it. The +current planner can be exported for that purpose with +`ctx.__datafusion_query_planner__()`. + +One ordering rule applies: a planner capsule captures the session's codecs at export +time and cannot be rebound afterward. Codec changes made after installing a single +planner are rebound automatically, but a planner wrapped inside another planner as a +fallback is opaque and keeps the codecs it was exported with. **Install all extension +codecs before exporting or chaining planners.** + +Putting it together for a session using two extension libraries that each provide +tables, functions, and a query planner: + +```python +ctx = SessionContext(config) + +# 1. Codecs from both libraries. Order between libraries does not matter. +ctx = ctx.with_logical_extension_codec(lib_a.codec()) +ctx = ctx.with_logical_extension_codec(lib_b.codec()) +ctx = ctx.with_physical_extension_codec(lib_a.physical_codec()) +ctx = ctx.with_physical_extension_codec(lib_b.physical_codec()) + +# 2. Planners, innermost fallback first. Library A's planner falls back to +# DataFusion's default planner; library B's planner falls back to A's. +ctx = ctx.with_query_planner(lib_a.Planner()) +ctx = ctx.with_query_planner( + lib_b.Planner(fallback=ctx.__datafusion_query_planner__()) +) + +# 3. Tables and functions — any time before the first query. +ctx.register_table("t", lib_a.TableProvider()) +ctx.register_udf(udf(lib_b.SomeUDF())) +``` + ## Alternative Approach Suppose you needed to expose some other features of DataFusion and you could not wait diff --git a/examples/datafusion-ffi-example/README.md b/examples/datafusion-ffi-example/README.md index c897c067a..6b15cbb18 100644 --- a/examples/datafusion-ffi-example/README.md +++ b/examples/datafusion-ffi-example/README.md @@ -35,7 +35,9 @@ Separate shared libraries guarantee distinct DataFusion library markers. This ca The example codecs do not inspect the callback `TaskContext`. A production codec that depends on session configuration or registered functions must ensure its exported FFI codec is bound to, and retains, the appropriate host `TaskContextProvider`. -The current Python API installs one external logical codec and one external physical codec. It does not yet compose codecs from several independent plugin owners. This example therefore makes the provider library the sole external codec owner; the planner uses built-in physical nodes and receives the provider codecs from the host. +Extension codecs compose: each `with_logical_extension_codec` / `with_physical_extension_codec` call prepends the codec to the session's codec chain, with the most recently installed codec consulted first and DataFusion's default codec as the terminal fallback. A codec signals "not mine" by returning an error, so several independent plugin libraries can install codecs on the same session as long as each only answers for payloads it owns (frame them with a distinct byte prefix). In this example the provider library is the only codec owner; the planner uses built-in physical nodes and receives the provider codecs from the host. + +`MyLogicalExtensionCodec` takes an optional token argument (`MyLogicalExtensionCodec("TOKENAAA")`) that overrides the byte prefix it stamps on encoded table providers. It exists so the tests can install two instances that own disjoint slices of the wire format, which is what makes chain ordering and fall-through observable from Python. Real plugin libraries should hard-code a prefix unique to the library rather than accept one from the caller. Register both provider codecs before installing the planner: diff --git a/examples/datafusion-ffi-example/python/tests/_test_logical_extension_codec.py b/examples/datafusion-ffi-example/python/tests/_test_logical_extension_codec.py index cd0c5a61a..065ebf185 100644 --- a/examples/datafusion-ffi-example/python/tests/_test_logical_extension_codec.py +++ b/examples/datafusion-ffi-example/python/tests/_test_logical_extension_codec.py @@ -17,8 +17,36 @@ from __future__ import annotations -from datafusion import LogicalPlan, SessionContext -from datafusion_ffi_example import MyLogicalExtensionCodec +import pyarrow as pa +import pytest +from datafusion import Expr, LogicalPlan, SessionContext, col, udf +from datafusion_ffi_example import MyLogicalExtensionCodec, MyTableProvider + + +def _double_udf(): + return udf( + lambda arr: pa.array([(v.as_py() or 0) * 2 for v in arr]), + [pa.int64()], + pa.int64(), + volatility="immutable", + name="double", + ) + + +def _encode_provider_plan(token: str) -> tuple[bytes, MyLogicalExtensionCodec]: + """Serialize a plan over this library's table provider using a codec + that stamps `token` on the encoded provider. + + Returns the blob and the codec, so callers can assert on its call + counters. The token is chosen per test so a second codec installed + later is provably unable to claim these bytes. + """ + codec = MyLogicalExtensionCodec(token) + ctx = SessionContext().with_logical_extension_codec(codec) + ctx.register_table("numbers", MyTableProvider(1, 4, 1)) + blob = ctx.sql('SELECT "A" FROM numbers').logical_plan().to_bytes(ctx) + assert token.encode() in blob + return blob, codec def _setup_session_with_codec() -> tuple[SessionContext, MyLogicalExtensionCodec]: @@ -80,3 +108,130 @@ def test_ffi_logical_codec_roundtrip(): restored = LogicalPlan.from_bytes(ctx, blob) df_round_trip = ctx.create_dataframe_from_logical_plan(restored) assert df.collect() == df_round_trip.collect() + + +def test_ffi_logical_codec_composes_with_later_install(): + """Codecs compose: installing a second codec prepends it to the + session's codec chain instead of replacing the first. The second + codec here (a default-backed codec exported from a fresh session) + cannot encode this library's table provider, so encoding falls + through to the user codec installed first. Under replace semantics + this test fails with `LogicalExtensionCodec is not provided`.""" + ctx, codec = _setup_session_with_codec() + ctx = ctx.with_logical_extension_codec( + SessionContext().__datafusion_logical_extension_codec__() + ) + + ctx.register_table("numbers", MyTableProvider(1, 4, 1)) + df = ctx.sql('SELECT "A" FROM numbers') + plan = df.logical_plan() + + before = codec.table_provider_encode_calls() + blob = plan.to_bytes(ctx) + assert codec.table_provider_encode_calls() > before + + restored = LogicalPlan.from_bytes(ctx, blob) + df_round_trip = ctx.create_dataframe_from_logical_plan(restored) + assert df.collect() == df_round_trip.collect() + + +def test_most_recently_installed_codec_encodes_first(): + """Encoding walks the chain front to back, and the front is the most + recently installed codec. Both codecs here can encode the provider, + so the winner is decided purely by install order. + + Both orders are exercised in one test on purpose. Asserting a single + order would also pass under replace semantics, where the second + install simply discards the first codec; swapping the order and + getting the other token proves the losing codec was still installed + and merely lost the race. + """ + for winner, loser in (("TOKENAAA", "TOKENBBB"), ("TOKENBBB", "TOKENAAA")): + loser_codec = MyLogicalExtensionCodec(loser) + winner_codec = MyLogicalExtensionCodec(winner) + ctx = SessionContext().with_logical_extension_codec(loser_codec) + ctx = ctx.with_logical_extension_codec(winner_codec) + + ctx.register_table("numbers", MyTableProvider(1, 4, 1)) + blob = ctx.sql('SELECT "A" FROM numbers').logical_plan().to_bytes(ctx) + + assert winner.encode() in blob + assert loser.encode() not in blob + assert winner_codec.table_provider_encode_calls() == 1 + assert loser_codec.table_provider_encode_calls() == 0 + + +def test_decode_falls_through_to_earlier_installed_codec(): + """A codec that does not own the payload signals "not mine" by + erroring, and the chain keeps walking. The bytes here are stamped + with the first codec's token, so the more recently installed second + codec must decline and let the first one decode.""" + blob, first = _encode_provider_plan("TOKENAAA") + + second = MyLogicalExtensionCodec("TOKENBBB") + ctx = SessionContext().with_logical_extension_codec(first) + ctx = ctx.with_logical_extension_codec(second) + + restored = LogicalPlan.from_bytes(ctx, blob) + assert ctx.create_dataframe_from_logical_plan(restored).collect() + + assert first.table_provider_decode_calls() == 1 + assert second.table_provider_decode_calls() == 0 + + +def test_decode_failure_aggregates_every_codec_error(): + """When no codec in the chain claims the payload, the error names + the number of codecs tried and carries each one's message, so an + operator can see which library was expected to own the bytes.""" + blob, _owner = _encode_provider_plan("TOKENBBB") + + # Neither installed codec owns TOKENBBB, so the chain is exhausted: + # two example codecs plus DataFusion's default codec. + ctx = SessionContext().with_logical_extension_codec( + MyLogicalExtensionCodec("TOKENCCC") + ) + ctx = ctx.with_logical_extension_codec(MyLogicalExtensionCodec("TOKENDDD")) + + with pytest.raises(Exception, match="None of the 3 composed extension codecs"): + LogicalPlan.from_bytes(ctx, blob) + + +def test_single_codec_chain_error_is_returned_verbatim(): + """A session with no extra codec has a one-entry chain, so a decode + failure surfaces DataFusion's own error rather than the aggregated + wrapper. Keeps error messages unchanged for the common case where + nobody composed anything.""" + blob, _owner = _encode_provider_plan("TOKENEEE") + + # DataFusion's own wording for "no codec claimed this", surfaced + # unwrapped because the chain has a single entry. + with pytest.raises( + Exception, match="LogicalExtensionCodec is not provided" + ) as excinfo: + LogicalPlan.from_bytes(SessionContext(), blob) + + assert "composed extension codecs" not in str(excinfo.value) + + +def test_udf_inlining_setting_survives_codec_install(): + """Installing an extension codec must not silently re-enable inline + Python UDF encoding on a session that opted out. Regression guard in + both directions: the encoder still emits the by-name form, and the + decoder still refuses an inline payload. + + The codec installed here delegates UDF encoding to DataFusion's + default codec. A codec exported from another `SessionContext` would + not work as a probe: that export is itself a Python-aware codec with + inlining enabled, so the strict outer codec would delegate to it and + the inline payload would reappear. + """ + strict = SessionContext().with_python_udf_inlining(enabled=False) + extended = strict.with_logical_extension_codec(MyLogicalExtensionCodec("TOKENFFF")) + + e = _double_udf()(col("a")) + assert b"DFPYUDF" not in e.to_bytes(extended) + + inline_blob = e.to_bytes(SessionContext()) + assert b"DFPYUDF" in inline_blob + with pytest.raises(Exception, match="inlining is disabled"): + Expr.from_bytes(inline_blob, ctx=extended) diff --git a/examples/datafusion-ffi-example/python/tests/_test_physical_extension_codec.py b/examples/datafusion-ffi-example/python/tests/_test_physical_extension_codec.py index 28eaaf2d9..82116bef7 100644 --- a/examples/datafusion-ffi-example/python/tests/_test_physical_extension_codec.py +++ b/examples/datafusion-ffi-example/python/tests/_test_physical_extension_codec.py @@ -76,3 +76,26 @@ def test_ffi_physical_codec_roundtrip(): restored = ExecutionPlan.from_bytes(ctx, blob) assert str(original) == str(restored) + + +def test_ffi_physical_codec_composes_with_later_install(): + """Codecs compose: a second install prepends to the chain instead + of replacing the first codec. The second codec here (default-backed + export from a fresh session) encodes UDFs by name without writing + bytes, which the chain treats as "no opinion" — so the user codec + installed first is still consulted. Under replace semantics its + counter stays at zero.""" + ctx, codec = _setup_session_with_codec() + ctx = ctx.with_physical_extension_codec( + SessionContext().__datafusion_physical_extension_codec__() + ) + + df = ctx.sql("SELECT abs(a) AS x FROM t") + original = df.execution_plan() + + before = codec.encode_udf_calls() + blob = original.to_bytes(ctx) + assert codec.encode_udf_calls() > before + + restored = ExecutionPlan.from_bytes(ctx, blob) + assert str(original) == str(restored) diff --git a/examples/datafusion-ffi-example/src/logical_extension_codec.rs b/examples/datafusion-ffi-example/src/logical_extension_codec.rs index 0474a8d39..4ea2e1e57 100644 --- a/examples/datafusion-ffi-example/src/logical_extension_codec.rs +++ b/examples/datafusion-ffi-example/src/logical_extension_codec.rs @@ -63,6 +63,9 @@ pub(crate) struct CallCounters { struct CountingLogicalExtensionCodec { inner: DefaultLogicalExtensionCodec, counters: Arc, + // Byte prefix identifying providers this codec owns. Distinct tokens let a + // test install several instances and observe which one the chain picks. + token: Arc<[u8]>, // The FFI task-context handle is weak. Retain its provider for as long as // this codec can be called, even if Python drops the exporter object. _ctx_provider: Arc, @@ -99,7 +102,7 @@ impl LogicalExtensionCodec for CountingLogicalExtensionCodec { schema: SchemaRef, ctx: &TaskContext, ) -> Result> { - if let Some(id) = token_id(buf, TABLE_PROVIDER_TOKEN) { + if let Some(id) = token_id(buf, &self.token) { self.counters .decode_table_provider .fetch_add(1, Ordering::SeqCst); @@ -132,7 +135,7 @@ impl LogicalExtensionCodec for CountingLogicalExtensionCodec { .lock() .map_err(|err| DataFusionError::Internal(err.to_string()))? .insert(id, node); - buf.extend_from_slice(TABLE_PROVIDER_TOKEN); + buf.extend_from_slice(&self.token); buf.extend_from_slice(&id.to_le_bytes()); return Ok(()); } @@ -160,15 +163,26 @@ impl LogicalExtensionCodec for CountingLogicalExtensionCodec { pub(crate) struct MyLogicalExtensionCodec { counters: Arc, ctx_provider: Arc, + token: Arc<[u8]>, } #[pymethods] impl MyLogicalExtensionCodec { + /// `token` overrides the byte prefix stamped on encoded table + /// providers. Two instances built with different tokens each own a + /// disjoint slice of the wire format, which is what lets a test + /// install both and tell from the decoded bytes which one the + /// session's codec chain consulted. #[new] - fn new() -> Self { + #[pyo3(signature = (token = None))] + fn new(token: Option<&str>) -> Self { Self { counters: Arc::new(CallCounters::default()), ctx_provider: Arc::new(SessionContext::new()), + token: token.map_or_else( + || Arc::from(TABLE_PROVIDER_TOKEN), + |token| Arc::from(token.as_bytes()), + ), } } @@ -195,6 +209,7 @@ impl MyLogicalExtensionCodec { let inner: Arc = Arc::new(CountingLogicalExtensionCodec { inner: DefaultLogicalExtensionCodec {}, counters: Arc::clone(&self.counters), + token: Arc::clone(&self.token), _ctx_provider: Arc::clone(&self.ctx_provider), }); diff --git a/examples/datafusion-ffi-query-planner-example/README.md b/examples/datafusion-ffi-query-planner-example/README.md index 53e2bddc5..7e1ec899a 100644 --- a/examples/datafusion-ffi-query-planner-example/README.md +++ b/examples/datafusion-ffi-query-planner-example/README.md @@ -55,6 +55,6 @@ ctx = ctx.with_query_planner(MyQueryPlanner()) `PlannerConfig` is transferred through the foreign session. `MyQueryPlanner` reads `ffi_query_planner.max_rows`, creates the plan with `DefaultPhysicalPlanner`, and adds a built-in `GlobalLimitExec`. The test changes the setting with `SET` and verifies the new row limit. -The provider's codec pair is attached to the planner when the derived context is created and is also used to decode the returned physical plan in `datafusion-python`. The API currently supports one external codec owner rather than a registry of independently composed codecs, so this planner deliberately uses only built-in physical nodes. Install the codecs before the planner where possible; derived contexts rebind codecs after planner installation, but planner-last order is easier to audit. +The provider's codec pair is attached to the planner when the derived context is created and is also used to decode the returned physical plan in `datafusion-python`. Extension codecs compose: each `with_logical_extension_codec` / `with_physical_extension_codec` call prepends to the session's codec chain, so several libraries can install codecs on the same session. This planner owns no serializable types of its own and deliberately uses only built-in physical nodes. Install codecs before the planner; derived contexts rebind codecs after a planner is installed directly, but a planner exported as a fallback for another planner keeps the codecs captured at export time. The pinned FFI logical codec cannot encode arbitrary custom `LogicalPlan::Extension` nodes. The example therefore demonstrates table-provider, UDF, and physical-plan interoperability without claiming custom logical extension support. diff --git a/examples/datafusion-ffi-query-planner-example/python/tests/_test_three_library_query_planner.py b/examples/datafusion-ffi-query-planner-example/python/tests/_test_three_library_query_planner.py index 8279879d1..0991c5c9e 100644 --- a/examples/datafusion-ffi-query-planner-example/python/tests/_test_three_library_query_planner.py +++ b/examples/datafusion-ffi-query-planner-example/python/tests/_test_three_library_query_planner.py @@ -118,3 +118,26 @@ def test_query_planner_rejects_invalid_config(max_rows: str): with pytest.raises(Exception, match=r"max_rows|Invalid value"): ctx.sql(f"SET ffi_query_planner.max_rows = '{max_rows}'").collect() + + +def test_composed_codecs_with_query_planner(): + """A second pair of codecs installed on top of the provider codecs + composes with them instead of replacing them. The extra codecs + (default-backed exports from a fresh session) decline everything, + so planner-driven encode/decode falls through to the provider + codecs and the query still succeeds end to end.""" + ctx, logical_codec, physical_codec = configured_context(max_rows=2) + other = SessionContext() + ctx = ctx.with_logical_extension_codec( + other.__datafusion_logical_extension_codec__() + ) + ctx = ctx.with_physical_extension_codec( + other.__datafusion_physical_extension_codec__() + ) + ctx = ctx.with_query_planner(MyQueryPlanner()) + + batches = ctx.sql('SELECT "A" FROM numbers ORDER BY "A"').collect() + assert batches[0].column(0).to_pylist() == [0, 1] + assert logical_codec.table_provider_encode_calls() > 0 + assert logical_codec.table_provider_decode_calls() > 0 + assert physical_codec.execution_plan_decode_calls() > 0 diff --git a/python/datafusion/context.py b/python/datafusion/context.py index c7b73c5da..b4214fdd5 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -1779,6 +1779,15 @@ def with_query_planner( its logical and physical extension codec settings. Codec changes made on a derived context are rebound to the planner before planning. + A session holds exactly one query planner; installing another replaces + it. To layer planners, construct the new planner with the current + planner as its fallback (export it via + :py:meth:`__datafusion_query_planner__`) before installing. A planner + exported this way captures the codecs installed at export time and + cannot be rebound afterward, so install all extension codecs before + chaining planners. See the FFI extensions guide for the full + multi-library registration recipe. + Args: planner: Object exposing ``__datafusion_query_planner__`` or a raw ``datafusion_query_planner`` PyCapsule. @@ -2229,11 +2238,19 @@ def __datafusion_query_planner__(self) -> Any: def with_logical_extension_codec( self, codec: LogicalExtensionCodecExportable | _PyCapsule ) -> SessionContext: - """Create a new session context with specified codec. + """Create a new session context with an additional logical codec. Only FFI codecs are supported. Pass any object implementing ``__datafusion_logical_extension_codec__`` (see :py:class:`~datafusion.user_defined.LogicalExtensionCodecExportable`). + + Codecs compose: each call adds the codec to the front of the + session's codec chain rather than replacing prior codecs. During + encoding and decoding, the most recently installed codec is + consulted first, falling through codec by codec to DataFusion's + default codec. Codecs signal "not mine" by returning an error, so + extension codecs should only answer for payloads they own — + typically identified by a distinct byte prefix. """ new_internal = self.ctx.with_logical_extension_codec(codec) new = SessionContext.__new__(SessionContext) @@ -2247,11 +2264,16 @@ def __datafusion_physical_extension_codec__(self) -> Any: def with_physical_extension_codec( self, codec: PhysicalExtensionCodecExportable | _PyCapsule ) -> SessionContext: - """Create a new session context with the specified physical codec. + """Create a new session context with an additional physical codec. Only FFI codecs are supported. Pass any object implementing ``__datafusion_physical_extension_codec__`` (see :py:class:`~datafusion.user_defined.PhysicalExtensionCodecExportable`). + + Codecs compose the same way as in + :py:meth:`with_logical_extension_codec`: each call prepends to the + session's codec chain, and the most recently installed codec is + consulted first. """ new_internal = self.ctx.with_physical_extension_codec(codec) new = SessionContext.__new__(SessionContext) diff --git a/python/tests/test_pickle_expr.py b/python/tests/test_pickle_expr.py index 588caa21a..56e3c151f 100644 --- a/python/tests/test_pickle_expr.py +++ b/python/tests/test_pickle_expr.py @@ -323,6 +323,49 @@ def test_cross_version_error_message(self): ): Expr.from_bytes(bytes(tampered)) + def test_unsupported_wire_version_error_message(self): + """A payload stamped with a wire-format version newer than this + build supports names both versions and points at the fix, rather + than failing deep inside cloudpickle with an opaque tuple-unpack + error. + + Patches the version byte at offset 7 of the frame described in + :meth:`test_cross_version_error_message`. The patch is + length-preserving, so the enclosing protobuf stays parseable and + the bytes reach the codec. + """ + e = _double_udf()(col("a")) + blob = e.to_bytes() + + idx = blob.find(b"DFPYUDF") + assert idx >= 0, "DFPYUDF frame not found in payload" + + tampered = bytearray(blob) + tampered[idx + 7] = 2 # WIRE_VERSION_CURRENT is 1 + + with pytest.raises(Exception, match="wire-format version v2") as excinfo: + Expr.from_bytes(bytes(tampered)) + assert "Align datafusion-python versions" in str(excinfo.value) + + def test_cross_major_version_error_message(self): + """Same diagnostic as the minor-version mismatch, driven from the + major byte at offset 8. Guards against a check that compares only + the minor component.""" + import sys + + e = _double_udf()(col("a")) + blob = e.to_bytes() + + idx = blob.find(b"DFPYUDF") + assert idx >= 0, "DFPYUDF frame not found in payload" + + tampered = bytearray(blob) + tampered[idx + 8] = (sys.version_info.major + 1) % 256 + + with pytest.raises(Exception, match="not portable") as excinfo: + Expr.from_bytes(bytes(tampered)) + assert f"Python {sys.version_info.major + 1}." in str(excinfo.value) + class TestPythonUdfInliningToggle: """`SessionContext.with_python_udf_inlining(enabled=False)` opts out of