From d66b9e3d15a19b5b323b3df76b1ddca6b8c8086f Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Wed, 9 Sep 2026 14:13:38 -0600 Subject: [PATCH 01/11] feat: DataFrame builders and conversions (issue #6, Phases 2-3) Adds the calculations construction and read-back layer the Phase 1 clients accept, per plan D6/D7. data_frame.py (no optional dependencies): - sampling_clock() / timestamp_list() / timestamp_count() relocated here from sample_status_client now that calculations frames are a second caller, and re-exported from it so existing imports keep working. - Typed scalar column builders, the legacy data_column() escape hatch (a None entry becomes an unset oneof -- the only way to express a gap on a shared axis), and the provenance helpers. - data_frame() routes columns by type and enforces the server's SHAPE rules client-side, so an error names the offending column instead of bouncing the whole save. Size caps stay server-side: they are deployment policy. - Array/image/struct/serialized builders are deliberately #17's; hand-built columns of those kinds pass through data_frame() today. data_frame_conversions.py: - Pure Python: data_frame_timestamps() (integer nanoseconds throughout), column_values() (standalone, so the bucket query #16 can reuse it; array columns reshape to one list per sample), data_frame_columns(), column_metadata_dict(). - Behind [analysis]: data_frame_to_pandas() (UTC index from int64 nanos), data_frame_from_pandas() (dtype -> typed column, NaN fail-loud since a dense typed column cannot express a gap), and the calculations bridges. The pandas direction always emits a TimestampList rather than inferring a SamplingClock: a clock is only correct if the spacing is exactly uniform at nanosecond precision, and guessing would move timestamps. Two things surfaced while building this: - The data_frame() *function* is deliberately NOT re-exported from dp_python_lib.client. Binding that name shadows the data_frame *module*, so both import forms hand back the function -- breaking the documented `from dp_python_lib.client import data_frame as dfb` usage. - An array column with no dims must be treated as underivable, not as an empty product of 1, which would silently make every element its own sample. Fixed in both the write and read paths. 87 new unit tests: 657 total with the [analysis] extra, and 601 passing with 56 skipping cleanly without it (verified in a venv that has no pandas). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- CLAUDE.md | 6 +- plan/tickets/6/plan.md | 8 + src/dp_python_lib/client/__init__.py | 33 + src/dp_python_lib/client/data_frame.py | 624 ++++++++++++++++++ .../client/data_frame_conversions.py | 457 +++++++++++++ .../client/sample_status_client.py | 106 +-- tests/unit/test_data_frame.py | 356 ++++++++++ tests/unit/test_data_frame_conversions.py | 429 ++++++++++++ 8 files changed, 1930 insertions(+), 89 deletions(-) create mode 100644 src/dp_python_lib/client/data_frame.py create mode 100644 src/dp_python_lib/client/data_frame_conversions.py create mode 100644 tests/unit/test_data_frame.py create mode 100644 tests/unit/test_data_frame_conversions.py diff --git a/CLAUDE.md b/CLAUDE.md index 68d6d64..cc93563 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,11 +151,13 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/annotation_client.py` - Annotation service facade; groups feature-scoped clients sharing the one `DpAnnotationService` channel (`.pv_metadata`, `.machine_config`, `.sample_status`, `.datasets`, `.annotations`, `.export` — every implemented `DpAnnotationService` feature area) - `src/dp_python_lib/client/pv_metadata_client.py` - PV metadata client (`save_pv_metadata()`, `get_pv_metadata()`, `query_pv_metadata()`, `iter_pv_metadata()`, `delete_pv_metadata()`) plus the `PvMetadataQuery` (`Q`) criterion helpers - `src/dp_python_lib/client/machine_config_client.py` - Machine configuration client covering both configurations (`save_configuration()`, `get_configuration()`, `query_configurations()`, `iter_configurations()`, `delete_configuration()`) and their temporal activations (`save_configuration_activation()`, `get_configuration_activation()`, `query_configuration_activations()`, `iter_configuration_activations()`, `delete_configuration_activation()`, `get_active_configurations()`). Includes the `ConfigurationQuery` (`C`) and `ConfigurationActivationQuery` (`CA`) criterion helpers and the `to_timestamp()` helper (tz-aware datetime / epoch seconds / `common.Timestamp`). Get/delete activation take a composite key (`client_activation_id` XOR `configuration_name`+`start_time`). Activation `end_time` is optional — omit it for an open-ended activation ("still in effect"); the field is then genuinely absent on the wire -- `src/dp_python_lib/client/sample_status_client.py` - Sample status client (`save_sample_statuses()`, `query_sample_statuses()`, `iter_sample_statuses()`, `iter_sample_statuses_stream()`, `delete_sample_statuses()`) plus the `sampling_clock()` / `timestamp_list()` axis builders and the `SampleStatusColumn` / `SampleStatusFrame` construction classes. A status's identity key is `(pvName, timestamp, domain, layer)`; `delete_sample_statuses()` requires either `pv_names` or an explicit `all_pvs=True` opt-in for the destructive wildcard +- `src/dp_python_lib/client/sample_status_client.py` - Sample status client (`save_sample_statuses()`, `query_sample_statuses()`, `iter_sample_statuses()`, `iter_sample_statuses_stream()`, `delete_sample_statuses()`) plus the `SampleStatusColumn` / `SampleStatusFrame` construction classes. The `sampling_clock()` / `timestamp_list()` axis builders now live in `data_frame.py` (issue #6 Phase 2, once calculations frames became a second caller) and are re-exported here, so existing imports are unaffected. A status's identity key is `(pvName, timestamp, domain, layer)`; `delete_sample_statuses()` requires either `pv_names` or an explicit `all_pvs=True` opt-in for the destructive wildcard - `src/dp_python_lib/client/sample_status_conversions.py` - Per-sample expansion of query results (no optional extras required): `expand_data_timestamps()` (SamplingClock positions computed in **integer nanoseconds**, never float seconds — the exact-match contract depends on it), `bucket_to_rows()` / `buckets_to_rows()` / `iter_rows()` yielding `SampleStatusRow` objects with absent confidence/reason surfaced as `None` rather than fabricated `0.0`/`""` - `src/dp_python_lib/client/dataset_client.py` - DataSet client (`save_dataset()`, `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, plus the `get_datasets(ids)` batch fetch that avoids the annotation-listing N+1) with the `DataSetQuery` (`DS`) criterion helpers and the `data_block()` builder. `data_block()` is the only place `begin < end` is checked — the server does not - `src/dp_python_lib/client/annotations_client.py` - Annotations client (`save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, `delete_annotation()`, `get_calculations()`) with the `AnnotationQuery` (`AQ`) criterion helpers and the `calculations()` builder, which takes a `dict[str, DataFrame]` so frame-name uniqueness is true by construction. Note `AnnotationsClient` (feature client) vs `AnnotationClient` (facade) - `src/dp_python_lib/client/export_client.py` - Export client (`export_data()`) with the `ExportFormat` str enum and the `calculations_spec()` builder +- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Array/image/struct/serialized builders are #17's; hand-built ones pass through +- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one list per sample), `data_frame_columns()`, `column_metadata_dict()`. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap), and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock` - `src/dp_python_lib/client/query_client.py` - v2 time-series query client (sample-oriented) exposed as `client.query`. Low-level wrappers `query_samples()` (unary, one resumable page) and `iter_query_samples()` (transparent paging), plus `iter_query_samples_stream()` (server-streaming, fire-and-consume, lazy). Queries are described by a kind-neutral `QueryParams` built from the `PvQuery` (`PV`) and `ConfigQuery` (`CFG`) criterion helpers; shares a `_build_query_spec()` seam so a future bucket request builder reuses it. Results wrap the raw `ColumnTable` (`.column_table`, `.next_page_token`); `.to_dataframe()`/`.to_numpy()` delegate to `query_conversions` (Phase 2, optional `[analysis]` extra) - `src/dp_python_lib/client/query_conversions.py` - Pythonic conversions for query results (optional `[analysis]` extra: pandas/numpy/openpyxl, imported lazily). `data_value_to_python()` (oneof extractor: scalars→native, timestamp→epoch-nanos, array→list, structure→dict, image→`Image` wrapper, fail-loud on unhandled arm), `column_table_to_dataframe()` (UTC datetime index + one column per DataColumn; dense-alignment and duplicate-column-name fail-loud; ColumnMetadata in `df.attrs`), `column_table_to_numpy()` (dict of 1-D arrays; complex arms stay 1-D object arrays rather than collapsing to 2-D), `dataframe_to_excel()` (thin `to_excel()` wrapper: row-limit guard, tz-drop, complex-cell stringification), and `query_samples_to_dataframe()`/`stream_query_samples_to_dataframes()` whole-query conveniences (unary concats by column name; streaming yields per-page frames lazily) - `src/dp_python_lib/client/service_api_client_base.py` - Base class for the service clients: owns the channel and the one-per-client gRPC stub, and provides `_dispatch()`, the shared three-tier sender that all 18 unary `_send_*` methods delegate to @@ -172,6 +174,8 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `tests/unit/test_export_client.py` - Unit tests for ExportClient (`ExportFormat` mapping and unreachable `UNSPECIFIED`, `calculations_spec()`, the zero-source rejection, three-tier error handling) - `tests/unit/test_annotation_client.py` - Unit tests pinning the `AnnotationClient` facade wiring (every feature client present, one shared channel, one stub apiece) - `tests/integration/test_datasets_annotations_integration.py` - Live-server round trip for datasets/annotations/calculations; ingests its own samples first, because `saveDataSet` requires archived PVs +- `tests/unit/test_data_frame.py` - Unit tests for the data_frame builders (axis relocation, each typed column, `data_column()` bool-before-int and unset-oneof handling, provenance helpers, and every `data_frame()` shape rule incl. array dims and serialized-column name-only checks) +- `tests/unit/test_data_frame_conversions.py` - Unit tests for data_frame_conversions (nanosecond-exact expansion, per-column conversion incl. array reshaping, duplicate-name fail-loud, and — skipping cleanly without `[analysis]` — the pandas round trip, dtype mapping, and NaN fail-loud) - `tests/unit/test_query_client.py` - Unit tests for QueryClient (request building, three-tier error handling, unary paging, streaming, `PvQuery`/`ConfigQuery` helpers, `QueryParams` validation) - `tests/unit/test_query_conversions.py` - Unit tests for query_conversions (each DataValue arm, dense-alignment and duplicate-column-name fail-loud, int-gap float-upcast, timestamp columns, 1-D object arrays for complex arms, metadata in attrs, concat-by-name, Excel row-limit/stringification/native-bytes; DataFrame/NumPy/Excel tests skip cleanly when the `[analysis]` extra is absent) - `pyproject.toml` - Project metadata and dependencies diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index 7e48e79..4602525 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -36,6 +36,14 @@ dp-service `fddf692` (annotation on `localhost:50053`, ingestion on `:50051`): 18 tests, 12 subtests. Writing it surfaced one further server behavior the triage had not found — `saveDataSet` requires its PVs to exist in the archive — now recorded in section 3 and in `CLAUDE.md`. +- **Phases 2 and 3 implemented 2026-09-09.** `data_frame.py` (axis builders relocated from `sample_status_client` + and re-exported, typed scalar column builders, the `data_column()` escape hatch, provenance helpers, and + `data_frame()` assembly with the server's shape rules checked client-side) and `data_frame_conversions.py` (the + pure-Python read side plus the pandas bridges behind `[analysis]`). 87 new unit tests; 657 total with the extra + installed, 601 passing and 56 skipping cleanly without it (verified in a venv that has no pandas). One naming + collision surfaced and was resolved: the `data_frame()` *function* is deliberately not re-exported from + `dp_python_lib.client`, because binding that name would shadow the `data_frame` *module* and break the + `from dp_python_lib.client import data_frame as dfb` form this plan's own reference snippet uses. ## Overview diff --git a/src/dp_python_lib/client/__init__.py b/src/dp_python_lib/client/__init__.py index 8c139c8..71eb7c0 100644 --- a/src/dp_python_lib/client/__init__.py +++ b/src/dp_python_lib/client/__init__.py @@ -10,6 +10,26 @@ SaveAnnotationRequestParams, calculations, ) + +# NOTE: the data_frame() *function* is deliberately NOT re-exported here. Binding that name in the package +# namespace would shadow the data_frame *module*, so both `from dp_python_lib.client import data_frame` and +# `import dp_python_lib.client.data_frame` would hand back the function instead of the module -- breaking the +# documented `from dp_python_lib.client import data_frame as dfb` usage. Reach it as dfb.data_frame(...). +from dp_python_lib.client.data_frame import ( + bool_column, + calculations_source, + column_metadata, + data_column, + double_column, + enum_column, + float_column, + int32_column, + int64_column, + provenance, + pv_source, + string_column, + timestamp_count, +) from dp_python_lib.client.dataset_client import ( DataSetClient, DataSetQuery, @@ -142,10 +162,23 @@ "SavePvMetadataRequestParams", "SaveSampleStatusesApiResult", "SaveSampleStatusesRequestParams", + "bool_column", "calculations", + "calculations_source", "calculations_spec", + "column_metadata", "data_block", + "data_column", + "double_column", + "enum_column", + "float_column", + "int32_column", + "int64_column", + "provenance", + "pv_source", "sampling_clock", + "string_column", + "timestamp_count", "timestamp_list", "to_timestamp", ] diff --git a/src/dp_python_lib/client/data_frame.py b/src/dp_python_lib/client/data_frame.py new file mode 100644 index 0000000..1418027 --- /dev/null +++ b/src/dp_python_lib/client/data_frame.py @@ -0,0 +1,624 @@ +""" +Builders for common.DataFrame -- the shared time-series payload shape (issue #6, Phase 2). + +A DataFrame is one time axis plus a set of columns sampled on it. It is the payload of an annotation's +Calculations, and it is also ingestion's `ingestionDataFrame`: the same message, so this module is the substrate +issue #17 extends rather than a parallel one to fork. + +Design decisions (see plan/tickets/6/plan.md, D6): + - The time axis builders sampling_clock() / timestamp_list() live here now that a second caller exists; they are + re-exported from sample_status_client so existing imports keep working. + - The axis carries its own sample count and every column is validated against it. That restates a number the + caller already has (sampling_clock(t0, period, count=len(values))), which is deliberate: deriving the count + from the columns would fork the axis API by caller, since sample_status_client genuinely knows its count + independently of any column. SampleStatusFrame set the same precedent -- explicit axis, per-column validation. + - Validation mirrors the server's SHAPE rules only (non-blank names, non-empty values, count match, column-name + uniqueness across types), so an error names the offending frame or column instead of bouncing the whole batch. + The server's resource caps -- 256-char strings, 10M-element arrays, 50 MB images, 1 MB structs -- stay + server-side: they are deployment policy, and duplicating numbers that can change is how clients drift. + - Array, image, struct, and serialized column builders are deliberately absent. Their ergonomics (dims, image + descriptors, schema ids) are #17's to design once with ingestion data in hand. Meanwhile they are reachable by + building the proto directly and passing it to data_frame(), which accepts pre-built column messages alongside + the ones its builders return. +""" + +from typing import Any + +from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp +from dp_python_lib.grpc import common_pb2 + +# Each typed column message, paired with the DataFrame field it belongs in. data_frame() routes by exact type, so +# a pre-built column of any supported kind lands in the right repeated field without the caller naming it. +_COLUMN_FIELD_BY_TYPE = { + common_pb2.DoubleColumn: "doubleColumns", + common_pb2.FloatColumn: "floatColumns", + common_pb2.Int64Column: "int64Columns", + common_pb2.Int32Column: "int32Columns", + common_pb2.BoolColumn: "boolColumns", + common_pb2.StringColumn: "stringColumns", + common_pb2.EnumColumn: "enumColumns", + common_pb2.ImageColumn: "imageColumns", + common_pb2.StructColumn: "structColumns", + common_pb2.DoubleArrayColumn: "doubleArrayColumns", + common_pb2.FloatArrayColumn: "floatArrayColumns", + common_pb2.Int32ArrayColumn: "int32ArrayColumns", + common_pb2.Int64ArrayColumn: "int64ArrayColumns", + common_pb2.BoolArrayColumn: "boolArrayColumns", + common_pb2.DataColumn: "dataColumns", + common_pb2.SerializedDataColumn: "serializedDataColumns", +} + +# Columns whose per-sample count is len(values). The array columns are excluded: their values are flat, so the +# sample count is len(values) / prod(dims). Serialized columns carry no countable values at all. +_SCALAR_COLUMN_TYPES = ( + common_pb2.DoubleColumn, + common_pb2.FloatColumn, + common_pb2.Int64Column, + common_pb2.Int32Column, + common_pb2.BoolColumn, + common_pb2.StringColumn, + common_pb2.EnumColumn, + common_pb2.StructColumn, +) + +_ARRAY_COLUMN_TYPES = ( + common_pb2.DoubleArrayColumn, + common_pb2.FloatArrayColumn, + common_pb2.Int32ArrayColumn, + common_pb2.Int64ArrayColumn, + common_pb2.BoolArrayColumn, +) + + +# ---------------------------------------------------------------------- +# time axis +# ---------------------------------------------------------------------- + + +def sampling_clock( + start_time: TimestampInput, + period_nanos: int, + count: int, +) -> common_pb2.DataTimestamps: + """ + Builds a DataTimestamps with a SamplingClock time axis, the compact form for regularly-sampled data. + + When labeling existing archived data (sample status), the clock must match that data's clock exactly -- + matching is by exact timestamp at nanosecond precision, so an off-by-one-nanosecond period misses every sample + after the first. When describing derived values (calculations), the clock defines the axis, and count must + equal the length of every column on it. + + Note the server rejects a calculations axis whose startTime is epoch zero, so build fixtures from a real time + rather than datetime(1970, 1, 1). + + :param start_time: Time of the first sample (tz-aware datetime, epoch seconds, or common.Timestamp). + :param period_nanos: Period between samples, in nanoseconds. Must be > 0. + :param count: Number of samples in the interval. Must be >= 1. + :return: A DataTimestamps carrying a SamplingClock. + :raises ValueError: if period_nanos is not positive or count is less than 1. + """ + if period_nanos <= 0: + raise ValueError(f"sampling_clock() requires period_nanos > 0, got {period_nanos}") + if count < 1: + raise ValueError(f"sampling_clock() requires count >= 1, got {count}") + + timestamps = common_pb2.DataTimestamps() + timestamps.samplingClock.startTime.CopyFrom(to_timestamp(start_time)) + timestamps.samplingClock.periodNanos = period_nanos + timestamps.samplingClock.count = count + return timestamps + + +def timestamp_list(values: list[TimestampInput]) -> common_pb2.DataTimestamps: + """ + Builds a DataTimestamps with an explicit TimestampList time axis, the form for irregularly-spaced samples -- + and, for sample status, for sparse labeling that names only the samples being labeled. + + Timestamps must be strictly increasing, which is validated here rather than deferred to the server. When + labeling existing data, supply timestamps taken from query results (or exact SamplingClock arithmetic); a + recomputed or rounded timestamp will silently fail to match its sample. + + :param values: The timestamps of the samples (tz-aware datetimes, epoch seconds, or common.Timestamp objects). + :return: A DataTimestamps carrying a TimestampList. + :raises ValueError: if values is empty or the timestamps are not strictly increasing. + """ + if not values: + raise ValueError("timestamp_list() requires a non-empty values list") + + converted = [to_timestamp(value) for value in values] + + previous = converted[0] + for index, current in enumerate(converted[1:], start=1): + if (current.epochSeconds, current.nanoseconds) <= (previous.epochSeconds, previous.nanoseconds): + raise ValueError( + f"timestamp_list() requires strictly increasing timestamps; entry {index} " + f"({current.epochSeconds}.{current.nanoseconds:09d}) does not follow entry {index - 1} " + f"({previous.epochSeconds}.{previous.nanoseconds:09d})" + ) + previous = current + + timestamps = common_pb2.DataTimestamps() + timestamps.timestampList.timestamps.extend(converted) + return timestamps + + +def timestamp_count(timestamps: common_pb2.DataTimestamps) -> int: + """ + Returns the number of timestamps a DataTimestamps describes, for validating parallel-array lengths. + + An empty axis is rejected here rather than allowed to surface later as a confusing column-length mismatch. + The axis builders already make this unreachable -- sampling_clock() requires count >= 1 and timestamp_list() + requires a non-empty list -- but a hand-built DataTimestamps can still carry a zero-count SamplingClock or an + empty TimestampList, and both report their oneof arm as set. Rejecting them keeps this in step with + expand_data_timestamps(), which applies the same rule on the read path. + + :param timestamps: The time axis to measure. + :return: The number of timestamps on the axis. + :raises ValueError: if neither axis form is set, or if the axis describes no timestamps. + """ + axis = timestamps.WhichOneof("value") + if axis == "samplingClock": + count = timestamps.samplingClock.count + if count < 1: + raise ValueError(f"DataTimestamps samplingClock requires count >= 1, got {count}") + return count + if axis == "timestampList": + count = len(timestamps.timestampList.timestamps) + if count < 1: + raise ValueError("DataTimestamps timestampList requires at least one timestamp, got an empty list") + return count + raise ValueError("DataTimestamps must specify either a samplingClock or a timestampList") + + +# ---------------------------------------------------------------------- +# provenance +# ---------------------------------------------------------------------- + + +def _time_range(bounds: tuple[TimestampInput, TimestampInput]) -> common_pb2.TimeRange: + """ + Builds a common.TimeRange from a (begin, end) pair. + + :param bounds: The (begin, end) pair, each a tz-aware datetime, epoch seconds, or common.Timestamp. + :return: The equivalent common.TimeRange. + :raises ValueError: if begin is not strictly before end. + """ + begin, end = (to_timestamp(bound) for bound in bounds) + if (begin.epochSeconds, begin.nanoseconds) >= (end.epochSeconds, end.nanoseconds): + raise ValueError( + f"time_range requires begin strictly before end; got begin " + f"{begin.epochSeconds}.{begin.nanoseconds:09d} and end {end.epochSeconds}.{end.nanoseconds:09d}" + ) + time_range = common_pb2.TimeRange() + time_range.beginTime.CopyFrom(begin) + time_range.endTime.CopyFrom(end) + return time_range + + +def pv_source( + pv_name: str, + time_range: tuple[TimestampInput, TimestampInput] | None = None, +) -> common_pb2.ColumnProvenance.ColumnSource: + """ + Builds a ColumnSource naming an archived PV a calculated column was derived from. + + :param pv_name: Name of the source PV. + :param time_range: Optional (begin, end) pair narrowing which part of the PV's history was used. + :return: A ColumnProvenance.ColumnSource with its pvName origin set. + :raises ValueError: if pv_name is empty, or if the time range's begin is not before its end. + """ + if not pv_name: + raise ValueError("pv_source() requires a non-empty pv_name") + + source = common_pb2.ColumnProvenance.ColumnSource() + source.pvName = pv_name + if time_range is not None: + source.timeRange.CopyFrom(_time_range(time_range)) + return source + + +def calculations_source( + calculations_id: str, + frame_name: str, + column_name: str, + time_range: tuple[TimestampInput, TimestampInput] | None = None, +) -> common_pb2.ColumnProvenance.ColumnSource: + """ + Builds a ColumnSource naming another calculations column this one was derived from. + + These are soft references: deleting the annotation that owns the referenced calculations leaves this link + dangling, which readers are expected to tolerate rather than resolve. + + :param calculations_id: Id of the calculations object holding the source column. + :param frame_name: Name of the frame within those calculations. + :param column_name: Name of the column within that frame. + :param time_range: Optional (begin, end) pair narrowing which part of the source column was used. + :return: A ColumnProvenance.ColumnSource with its calculationsColumn origin set. + :raises ValueError: if any of the three identifiers is empty, or if the time range is reversed. + """ + if not calculations_id: + raise ValueError("calculations_source() requires a non-empty calculations_id") + if not frame_name: + raise ValueError("calculations_source() requires a non-empty frame_name") + if not column_name: + raise ValueError("calculations_source() requires a non-empty column_name") + + source = common_pb2.ColumnProvenance.ColumnSource() + source.calculationsColumn.calculationsId = calculations_id + source.calculationsColumn.frameName = frame_name + source.calculationsColumn.columnName = column_name + if time_range is not None: + source.timeRange.CopyFrom(_time_range(time_range)) + return source + + +def provenance( + source: str | None = None, + process: str | None = None, + derived_from: list[common_pb2.ColumnProvenance.ColumnSource] | None = None, +) -> common_pb2.ColumnProvenance: + """ + Builds a ColumnProvenance recording where a column's values came from and how they were produced. + + :param source: Free-text name of the producing system or person. + :param process: Free-text description of the computation (e.g. "1 Hz RMS"). + :param derived_from: The inputs this column was computed from (see pv_source() / calculations_source()). + :return: A common.ColumnProvenance. + """ + result = common_pb2.ColumnProvenance() + if source: + result.source = source + if process: + result.process = process + if derived_from: + result.derivedFrom.extend(derived_from) + return result + + +def column_metadata( + tags: list[str] | None = None, + attributes: dict[str, str] | None = None, + provenance: common_pb2.ColumnProvenance | None = None, +) -> common_pb2.ColumnMetadata: + """ + Builds a ColumnMetadata: per-column tags, attributes, and provenance. + + :param tags: Tags (keywords) describing the column. + :param attributes: Map of key/value attributes describing the column. + :param provenance: Where the column's values came from (see provenance()). + :return: A common.ColumnMetadata. + """ + metadata = common_pb2.ColumnMetadata() + if provenance is not None: + metadata.provenance.CopyFrom(provenance) + if tags: + metadata.tags[:] = tags + if attributes: + for name, value in attributes.items(): + attribute = metadata.attributes.add() + attribute.name = name + attribute.value = value + return metadata + + +# ---------------------------------------------------------------------- +# typed scalar columns +# ---------------------------------------------------------------------- + + +def _build_scalar_column( + column_class: Any, + builder_name: str, + name: str, + values: list, + metadata: common_pb2.ColumnMetadata | None, +) -> Any: + """ + Builds one typed scalar column, applying the shape rules every column type shares. + + :param column_class: The typed column message class to construct. + :param builder_name: The public builder's name, used in error messages. + :param name: The column's name. + :param values: The column's values, one per sample. + :param metadata: Optional per-column metadata. + :return: The constructed typed column message. + :raises ValueError: if name is empty or values is empty. + """ + if not name: + raise ValueError(f"{builder_name}() requires a non-empty name") + if not values: + raise ValueError(f"{builder_name}() requires a non-empty values list for column '{name}'") + + column = column_class() + column.name = name + column.values[:] = values + if metadata is not None: + column.metadata.CopyFrom(metadata) + return column + + +def double_column( + name: str, values: list[float], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.DoubleColumn: + """ + Builds a DoubleColumn (float64) with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One float per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.DoubleColumn. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.DoubleColumn, "double_column", name, values, metadata) + + +def float_column( + name: str, values: list[float], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.FloatColumn: + """ + Builds a FloatColumn (float32) with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One float per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.FloatColumn. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.FloatColumn, "float_column", name, values, metadata) + + +def int64_column( + name: str, values: list[int], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.Int64Column: + """ + Builds an Int64Column with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One int per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.Int64Column. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.Int64Column, "int64_column", name, values, metadata) + + +def int32_column( + name: str, values: list[int], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.Int32Column: + """ + Builds an Int32Column with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One int per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.Int32Column. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.Int32Column, "int32_column", name, values, metadata) + + +def bool_column( + name: str, values: list[bool], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.BoolColumn: + """ + Builds a BoolColumn with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One bool per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.BoolColumn. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.BoolColumn, "bool_column", name, values, metadata) + + +def string_column( + name: str, values: list[str], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.StringColumn: + """ + Builds a StringColumn with one value per sample. + + The server caps individual string values (256 characters at the time of writing); that limit is deployment + policy and is deliberately not duplicated here. + + :param name: The column's name, unique within its frame. + :param values: One string per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.StringColumn. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.StringColumn, "string_column", name, values, metadata) + + +def enum_column( + name: str, values: list[int], enum_id: str, metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.EnumColumn: + """ + Builds an EnumColumn: integer codes plus the id of the enumeration that gives them meaning. + + :param name: The column's name, unique within its frame. + :param values: One integer code per sample on the frame's time axis. + :param enum_id: Id of the enumeration defining what the codes mean. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.EnumColumn. + :raises ValueError: if name, values, or enum_id is empty. + """ + if not enum_id: + raise ValueError("enum_column() requires a non-empty enum_id") + column = _build_scalar_column(common_pb2.EnumColumn, "enum_column", name, values, metadata) + column.enumId = enum_id + return column + + +def data_column( + name: str, values: list[Any], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.DataColumn: + """ + Builds a legacy DataColumn of per-sample DataValues -- the escape hatch for a column that needs gaps. + + The typed columns above are dense: every sample has a value, because the repeated field carries no notion of a + missing entry. A DataColumn's values are DataValue messages, whose oneof can be left unset, so a None entry + here becomes a genuinely absent value. That makes this the only way to express a gap on a shared time axis + short of giving the sparse column its own frame. + + Values are mapped to DataValue arms by Python type: bool -> booleanValue (checked before int, since bool is a + subclass of int), int -> longValue, float -> doubleValue, str -> stringValue, bytes -> byteArrayValue, and + None -> an unset oneof. Anything else raises: silently coercing an unexpected type would store something the + caller did not mean. Pass a pre-built DataValue to use an arm this mapping does not cover. + + :param name: The column's name, unique within its frame. + :param values: One value per sample, where None means "no value for this sample". + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.DataColumn. + :raises ValueError: if name or values is empty, or if a value's type has no DataValue mapping. + """ + if not name: + raise ValueError("data_column() requires a non-empty name") + if not values: + raise ValueError(f"data_column() requires a non-empty values list for column '{name}'") + + column = common_pb2.DataColumn() + column.name = name + for index, value in enumerate(values): + data_value = column.dataValues.add() + if value is None: + continue + if isinstance(value, common_pb2.DataValue): + data_value.CopyFrom(value) + elif isinstance(value, bool): + # Checked before int: bool is a subclass of int, so the int branch would swallow it. + data_value.booleanValue = value + elif isinstance(value, int): + data_value.longValue = value + elif isinstance(value, float): + data_value.doubleValue = value + elif isinstance(value, str): + data_value.stringValue = value + elif isinstance(value, bytes): + data_value.byteArrayValue = value + else: + raise ValueError( + f"data_column() cannot map value at index {index} of column '{name}' " + f"(type {type(value).__name__}) to a DataValue; pass a pre-built common.DataValue instead" + ) + + if metadata is not None: + column.metadata.CopyFrom(metadata) + return column + + +# ---------------------------------------------------------------------- +# frame assembly +# ---------------------------------------------------------------------- + + +def _column_sample_count(column: Any) -> int | None: + """ + Returns the number of samples a column carries, or None when the column has no countable per-sample values. + + :param column: A typed column, a legacy DataColumn, or a SerializedDataColumn. + :return: The sample count, or None for a SerializedDataColumn (whose payload is opaque). + """ + if isinstance(column, common_pb2.SerializedDataColumn): + return None + if isinstance(column, common_pb2.DataColumn): + return len(column.dataValues) + if isinstance(column, _ARRAY_COLUMN_TYPES): + # Array values are flat: samples x prod(dims). Absent dims are underivable rather than a product of 1 -- + # an empty product would silently treat each element as its own sample. _check_column() reports that + # instead of dividing by zero or accepting a wrong count. + dims = list(column.dimensions.dims) + if not dims: + return None + product = 1 + for dim in dims: + product *= dim + if product <= 0: + return None + return len(column.values) // product + return len(column.values) + + +def _check_column(column: Any, index: int, expected_count: int, seen_names: set[str]) -> None: + """ + Applies the server's per-column shape rules to one column, with a message naming the column. + + :param column: The column to check. + :param index: The column's position in the caller's list, for messages about an unnamed column. + :param expected_count: The frame's sample count, which every counted column must match. + :param seen_names: Names already used in this frame; mutated to record this column's name. + :raises ValueError: if the column is of an unsupported type, unnamed, empty, duplicate-named, or + count-mismatched. + """ + if type(column) not in _COLUMN_FIELD_BY_TYPE: + raise ValueError( + f"data_frame() received an unsupported column type at index {index}: {type(column).__name__}. " + f"Use one of the column builders, or pass a pre-built typed column message." + ) + + name = column.name + if not name: + raise ValueError(f"data_frame() requires a non-empty name for every column; column at index {index} has none") + if name in seen_names: + raise ValueError( + f"data_frame() requires unique column names within a frame; '{name}' appears more than once " + f"(names must be unique across ALL column types, not just within one type)" + ) + seen_names.add(name) + + if isinstance(column, common_pb2.SerializedDataColumn): + # Serialized payloads are opaque; the server checks their names only. + return + + if isinstance(column, _ARRAY_COLUMN_TYPES) and _column_sample_count(column) is None: + raise ValueError( + f"data_frame() cannot determine the sample count of array column '{name}': its dimensions are " + f"missing or zero. Set ArrayDimensions.dims so that values is samples x prod(dims)." + ) + + count = _column_sample_count(column) + if count == 0: + raise ValueError(f"data_frame() requires a non-empty values list for column '{name}'") + if count != expected_count: + raise ValueError( + f"data_frame() column '{name}' has {count} values but the time axis has {expected_count} timestamps; " + f"every column must carry exactly one value per sample" + ) + + +def data_frame( + data_timestamps: common_pb2.DataTimestamps, + columns: list[Any], +) -> common_pb2.DataFrame: + """ + Assembles a common.DataFrame from a time axis and a list of columns, routing each column into the repeated + field for its type. + + Accepts the columns this module builds and pre-built column messages alike, so the kinds without builders yet + (arrays, images, structs, serialized payloads -- issue #17) can be passed straight through. + + Validates the server's shape rules client-side, so a mistake names the offending column instead of bouncing the + whole save: at least one column, a non-blank name and non-empty values for each, a value count matching the + axis (for arrays, values / prod(dims)), and column names unique across ALL types within the frame. The + server's size caps are not duplicated -- see the module docstring. + + :param data_timestamps: The frame's time axis (see sampling_clock() / timestamp_list()). + :param columns: The frame's columns, in any mix of supported types. + :return: A common.DataFrame. + :raises ValueError: if the axis is empty or malformed, if columns is empty, or if any column violates a shape + rule. + """ + if not columns: + raise ValueError("data_frame() requires at least one column") + + expected_count = timestamp_count(data_timestamps) + + seen_names: set[str] = set() + for index, column in enumerate(columns): + _check_column(column, index, expected_count, seen_names) + + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(data_timestamps) + for column in columns: + getattr(frame, _COLUMN_FIELD_BY_TYPE[type(column)]).append(column) + return frame diff --git a/src/dp_python_lib/client/data_frame_conversions.py b/src/dp_python_lib/client/data_frame_conversions.py new file mode 100644 index 0000000..7894d8b --- /dev/null +++ b/src/dp_python_lib/client/data_frame_conversions.py @@ -0,0 +1,457 @@ +""" +Pythonic conversions for common.DataFrame (issue #6, Phase 2 and 3). + +Reads a DataFrame -- an annotation's calculations, and in future a bucket query's typed columns -- back into plain +Python, and, behind the optional [analysis] extra, into pandas. + +The pure-Python half has no third-party dependencies. pandas is imported lazily inside each entry point that needs +it, so importing this module never requires the extra. + +Design decisions (see plan/tickets/6/plan.md, D7): + - Timestamps are computed in INTEGER NANOSECONDS via expand_data_timestamps(), never in float seconds. A float64 + carries 53 bits of mantissa and present-day epoch nanoseconds need ~61, so a float round-trip would silently + move every timestamp. The pandas index is built from those int64 nanoseconds directly for the same reason. + - column_values() is written as a standalone per-column converter because the bucket query (#16) reuses these + same 14 typed column messages; it takes one column and needs to know nothing about the frame. + - Array columns are reshaped into one list per sample using their declared dims, rather than returned flat: a + flat list would silently lose the sample boundaries. + - Dense typed columns cannot express a gap, so the pandas->DataFrame direction rejects NaN/None fail-loud with a + message pointing at the two ways to express one (a separate frame, or data_column()). +""" + +from typing import Any + +from dp_python_lib.client.query_conversions import data_value_to_python +from dp_python_lib.client.sample_status_conversions import expand_data_timestamps +from dp_python_lib.grpc import annotation_pb2, common_pb2 + +# The DataFrame fields holding typed scalar columns, in the proto's declaration order. Each carries `values` +# directly parallel to the time axis. +_SCALAR_COLUMN_FIELDS = ( + "doubleColumns", + "floatColumns", + "int64Columns", + "int32Columns", + "boolColumns", + "stringColumns", + "enumColumns", + "structColumns", +) + +# The DataFrame fields holding array columns, whose `values` are flat (samples x prod(dims)). +_ARRAY_COLUMN_FIELDS = ( + "doubleArrayColumns", + "floatArrayColumns", + "int32ArrayColumns", + "int64ArrayColumns", + "boolArrayColumns", +) + +# ImageColumn stores its per-sample payloads in `images` rather than `values`. +_IMAGE_COLUMN_FIELD = "imageColumns" + + +def _require_pandas(): + """Imports and returns pandas, or raises an actionable error if the optional [analysis] extra is missing.""" + try: + import pandas + except ImportError as e: + raise ImportError( + "pandas is required for DataFrame conversions. Install the optional analysis extra: " + 'pip install "dp-python-lib[analysis]"' + ) from e + return pandas + + +def data_frame_timestamps(frame: common_pb2.DataFrame) -> list[int]: + """ + Expands a frame's time axis into one integer epoch-nanosecond value per sample. + + A SamplingClock is expanded arithmetically in integer nanoseconds, so the result reproduces the producer's + timestamps exactly; see expand_data_timestamps(). + + :param frame: The DataFrame whose axis to expand. + :return: Epoch nanoseconds for each sample, in axis order. + :raises ValueError: if the frame has no time axis, or the axis is empty or malformed. + """ + if not frame.HasField("dataTimestamps"): + raise ValueError("DataFrame has no dataTimestamps; every frame must carry a time axis") + return expand_data_timestamps(frame.dataTimestamps) + + +def _reshape_array_values(column: Any) -> list[list]: + """ + Reshapes an array column's flat values into one list per sample, using its declared dimensions. + + :param column: An array column (DoubleArrayColumn, Int32ArrayColumn, ...). + :return: One list of values per sample; multi-dimensional arrays stay flat WITHIN a sample. + :raises ValueError: if dims are missing/zero, or the value count is not a whole multiple of prod(dims). + """ + # Absent dims are underivable rather than an empty product of 1, which would silently make every element its + # own sample. + dims = list(column.dimensions.dims) + product = 1 + for dim in dims: + product *= dim + if not dims or product <= 0: + raise ValueError( + f"array column '{column.name}' has missing or zero dimensions, so its samples cannot be delimited" + ) + + values = list(column.values) + if len(values) % product != 0: + raise ValueError( + f"array column '{column.name}' has {len(values)} values, which is not a whole multiple of its " + f"per-sample size {product} (from dims {list(column.dimensions.dims)})" + ) + return [values[i : i + product] for i in range(0, len(values), product)] + + +def column_values(column: Any) -> list: + """ + Extracts one Python value per sample from any supported column message. + + Written as a standalone converter because the bucket query (#16) carries the same 14 typed column messages and + can reuse this without going through a DataFrame. + + Mapping: typed scalar columns yield their native values; an EnumColumn yields its integer codes (the enumeration + naming them is `enumId` on the column); an array column yields one list per sample; an ImageColumn yields one + bytes payload per sample; and a legacy DataColumn is converted per value by data_value_to_python(), so an unset + oneof becomes None -- the only representation of a gap in this API. + + :param column: A typed column, a legacy DataColumn, or an ImageColumn. + :return: One value per sample, in axis order. + :raises ValueError: if the column type is unsupported, or an array column's dims do not divide its values. + """ + if isinstance(column, common_pb2.DataColumn): + return [data_value_to_python(value) for value in column.dataValues] + if isinstance(column, common_pb2.ImageColumn): + return list(column.images) + if isinstance( + column, + ( + common_pb2.DoubleArrayColumn, + common_pb2.FloatArrayColumn, + common_pb2.Int32ArrayColumn, + common_pb2.Int64ArrayColumn, + common_pb2.BoolArrayColumn, + ), + ): + return _reshape_array_values(column) + if hasattr(column, "values"): + return list(column.values) + raise ValueError(f"unsupported column type for value extraction: {type(column).__name__}") + + +def iter_frame_columns(frame: common_pb2.DataFrame): + """ + Yields every column in a frame, across all of its repeated fields, in a stable order. + + Serialized columns are skipped: their payloads are opaque encoded blobs that this library does not decode + (deferred, as in query_conversions). A caller needing them can read frame.serializedDataColumns directly. + + :param frame: The DataFrame to walk. + :return: An iterator over the frame's column messages. + """ + for field in (*_SCALAR_COLUMN_FIELDS, _IMAGE_COLUMN_FIELD, *_ARRAY_COLUMN_FIELDS): + yield from getattr(frame, field) + yield from frame.dataColumns + + +def data_frame_columns(frame: common_pb2.DataFrame) -> dict[str, list]: + """ + Converts every column in a frame into a dict of column name -> one Python value per sample. + + Column names are unique across all types within a frame (the server enforces it on save, and data_frame() + checks it on the way out), so keying by name is lossless. A frame that violates it anyway -- hand-built, or + from an older server -- raises rather than silently dropping the earlier column. + + :param frame: The DataFrame to convert. + :return: A dict mapping column name to its per-sample values. + :raises ValueError: if two columns share a name, or a column cannot be converted. + """ + columns: dict[str, list] = {} + for column in iter_frame_columns(frame): + if column.name in columns: + raise ValueError( + f"DataFrame carries more than one column named '{column.name}'; names must be unique across all " + f"column types within a frame" + ) + columns[column.name] = column_values(column) + return columns + + +def column_metadata_dict(column: Any) -> dict[str, Any]: + """ + Summarizes a column's ColumnMetadata as a plain dict, for carrying alongside converted values. + + :param column: A column message that may have a `metadata` field. + :return: A dict with 'tags', 'attributes', and 'provenance' keys; empty containers when unset. + """ + metadata = getattr(column, "metadata", None) + if metadata is None: + return {"tags": [], "attributes": {}, "provenance": None} + + provenance = None + if metadata.HasField("provenance"): + source = metadata.provenance + provenance = { + "source": source.source, + "process": source.process, + "derived_from": [ + { + "pv_name": entry.pvName, + "calculations_column": ( + { + "calculations_id": entry.calculationsColumn.calculationsId, + "frame_name": entry.calculationsColumn.frameName, + "column_name": entry.calculationsColumn.columnName, + } + if entry.WhichOneof("origin") == "calculationsColumn" + else None + ), + } + for entry in source.derivedFrom + ], + } + + return { + "tags": list(metadata.tags), + "attributes": {attribute.name: attribute.value for attribute in metadata.attributes}, + "provenance": provenance, + } + + +def _check_frame_alignment(frame: common_pb2.DataFrame, columns: dict[str, list], n_rows: int) -> None: + """ + Rejects a frame whose columns do not all match its time axis. + + The server validates this on save, so a mismatch here means a hand-built frame or a corrupt record; failing + loudly beats producing a silently truncated or NaN-padded table. + + :param frame: The frame being converted, named in the error message. + :param columns: The already-converted columns. + :param n_rows: The number of timestamps on the axis. + :raises ValueError: if any column's length differs from n_rows. + """ + for name, values in columns.items(): + if len(values) != n_rows: + raise ValueError( + f"DataFrame column '{name}' has {len(values)} values but the time axis has {n_rows} timestamps; " + f"columns must be dense and index-aligned with the axis" + ) + + +def data_frame_to_pandas(frame: common_pb2.DataFrame, exclude_column_metadata: bool = False) -> Any: + """ + Converts a DataFrame into a pandas DataFrame with a UTC DatetimeIndex. + + Requires the optional [analysis] extra. + + The index is built from int64 epoch nanoseconds directly, so a SamplingClock axis stays exact -- routing + through float seconds would move present-day timestamps by hundreds of nanoseconds. Per-column ColumnMetadata + lands in df.attrs["column_metadata"] (a dict keyed by column name), matching query_conversions' convention. + + :param frame: The DataFrame to convert. + :param exclude_column_metadata: When True, skip populating df.attrs["column_metadata"]. + :return: A pandas.DataFrame indexed by UTC timestamp. + :raises ImportError: if the [analysis] extra is not installed. + :raises ValueError: if the frame's axis is missing/empty, or a column is not aligned with it. + """ + pd = _require_pandas() + + epoch_nanos = data_frame_timestamps(frame) + columns = data_frame_columns(frame) + _check_frame_alignment(frame, columns, len(epoch_nanos)) + + index = pd.DatetimeIndex(pd.to_datetime(pd.Series(epoch_nanos, dtype="int64"), unit="ns", utc=True)) + df = pd.DataFrame(columns, index=index) + + if not exclude_column_metadata: + df.attrs["column_metadata"] = { + column.name: column_metadata_dict(column) for column in iter_frame_columns(frame) + } + return df + + +def _timestamps_from_index(index: Any) -> common_pb2.DataTimestamps: + """ + Builds a DataTimestamps from a pandas DatetimeIndex, in integer nanoseconds throughout. + + Always emits a TimestampList rather than trying to detect a regular interval and emit a SamplingClock. A clock + is only correct if the spacing is exactly uniform at nanosecond precision, and inferring that from an index + that merely looks regular would quietly change the timestamps -- the one thing this API cannot tolerate. Build + a clock explicitly with sampling_clock() when that is what the data is. + + :param index: A pandas DatetimeIndex. + :return: A DataTimestamps carrying a TimestampList. + :raises ValueError: if the index is empty, not a DatetimeIndex, or not strictly increasing. + """ + pd = _require_pandas() + + if not isinstance(index, pd.DatetimeIndex): + raise ValueError( + f"DataFrame must be indexed by a pandas DatetimeIndex to become a time axis, got {type(index).__name__}" + ) + if len(index) == 0: + raise ValueError("DataFrame must have at least one row to become a DataFrame with a time axis") + + # tz-naive input is ambiguous: it could be UTC or local. Require the caller to say, as to_timestamp() does. + if index.tz is None: + raise ValueError( + "DataFrame index must be timezone-aware so its instants are unambiguous; localize it first, " + "e.g. df.index = df.index.tz_localize('UTC')" + ) + + epoch_nanos = [int(value) for value in index.view("int64")] if hasattr(index, "view") else None + if epoch_nanos is None: + epoch_nanos = [int(value) for value in index.astype("int64")] + + timestamps = common_pb2.DataTimestamps() + previous = None + for position, nanos in enumerate(epoch_nanos): + if previous is not None and nanos <= previous: + raise ValueError( + f"DataFrame index must be strictly increasing to become a time axis; entry {position} " + f"({nanos} ns) does not follow entry {position - 1} ({previous} ns)" + ) + previous = nanos + timestamp = timestamps.timestampList.timestamps.add() + timestamp.epochSeconds, timestamp.nanoseconds = divmod(nanos, 1_000_000_000) + return timestamps + + +def _column_from_series(name: str, series: Any) -> Any: + """ + Builds the typed column matching a pandas Series' dtype. + + Mapping: float64 -> DoubleColumn, float32 -> FloatColumn, int64 -> Int64Column, int32 -> Int32Column, + bool -> BoolColumn, and object/string -> StringColumn. Anything else raises rather than guessing. + + A missing value anywhere is a fail-loud error: the typed columns are dense repeated fields with no way to mark + an absent entry, so a NaN would have to be either invented as a real value or silently dropped. + + :param name: The column's name. + :param series: The pandas Series holding its values. + :return: The matching typed column message. + :raises ValueError: if the dtype has no mapping, or the series contains a missing value. + """ + from dp_python_lib.client import data_frame as builders + + if series.isna().any(): + missing_positions = [int(position) for position in series.isna().to_numpy().nonzero()[0][:5]] + raise ValueError( + f"column '{name}' contains missing values (first at row position(s) {missing_positions}), which a " + f"dense typed column cannot represent. Either give the sparse column its own frame over only the " + f"timestamps where it has values, or build it with data_frame.data_column(), whose DataValues can be " + f"left unset." + ) + + dtype = series.dtype + dtype_name = str(dtype) + + if dtype_name == "float64": + return builders.double_column(name, [float(v) for v in series]) + if dtype_name == "float32": + return builders.float_column(name, [float(v) for v in series]) + if dtype_name in ("int64", "Int64"): + return builders.int64_column(name, [int(v) for v in series]) + if dtype_name in ("int32", "Int32"): + return builders.int32_column(name, [int(v) for v in series]) + if dtype_name in ("bool", "boolean"): + return builders.bool_column(name, [bool(v) for v in series]) + if dtype_name in ("object", "string", "str") or dtype_name.startswith("string"): + values = list(series) + if not all(isinstance(value, str) for value in values): + offending = next( + (type(value).__name__ for value in values if not isinstance(value, str)), + "unknown", + ) + raise ValueError( + f"column '{name}' has dtype {dtype_name} but holds a non-string value (type {offending}); " + f"object columns are mapped to StringColumn, so convert the values first or build the column " + f"explicitly with a data_frame builder" + ) + return builders.string_column(name, values) + + raise ValueError( + f"column '{name}' has dtype {dtype_name}, which has no typed-column mapping. Supported dtypes are " + f"float64, float32, int64, int32, bool, and object/string; build other kinds explicitly with the " + f"data_frame builders and pass them to data_frame()." + ) + + +def data_frame_from_pandas(df: Any) -> common_pb2.DataFrame: + """ + Converts a pandas DataFrame into a common.DataFrame, mapping each column's dtype to a typed column. + + Requires the optional [analysis] extra. + + The index becomes an explicit TimestampList; see _timestamps_from_index() for why a SamplingClock is never + inferred. Missing values are rejected fail-loud, since a dense typed column cannot express a gap. + + :param df: The pandas DataFrame to convert. Must have a tz-aware, strictly increasing DatetimeIndex. + :return: A common.DataFrame. + :raises ImportError: if the [analysis] extra is not installed. + :raises ValueError: if the index is unusable, the frame has no columns, a dtype has no mapping, or any column + contains a missing value. + """ + from dp_python_lib.client import data_frame as builders + + _require_pandas() + + if len(df.columns) == 0: + raise ValueError("DataFrame must have at least one column") + + data_timestamps = _timestamps_from_index(df.index) + columns = [_column_from_series(str(name), df[name]) for name in df.columns] + return builders.data_frame(data_timestamps, columns) + + +def calculations_from_dataframes(frames: dict[str, Any]) -> annotation_pb2.Calculations: + """ + Converts a mapping of frame name -> pandas DataFrame into a Calculations payload. + + Requires the optional [analysis] extra. + + :param frames: Mapping of frame name to the pandas DataFrame holding that frame's columns. + :return: An annotation.Calculations. + :raises ImportError: if the [analysis] extra is not installed. + :raises ValueError: if frames is empty, a frame name is empty, or any frame cannot be converted. + """ + from dp_python_lib.client.annotations_client import calculations as build_calculations + + _require_pandas() + + if not frames: + raise ValueError("calculations_from_dataframes() requires at least one frame") + + return build_calculations({name: data_frame_from_pandas(df) for name, df in frames.items()}) + + +def calculations_to_dataframes( + calculations: annotation_pb2.Calculations | None, + exclude_column_metadata: bool = False, +) -> dict[str, Any]: + """ + Converts every frame of a Calculations into a pandas DataFrame, keyed by frame name. + + Requires the optional [analysis] extra. + + :param calculations: The Calculations to convert, or None. + :param exclude_column_metadata: When True, skip populating each frame's df.attrs["column_metadata"]. + :return: A dict mapping frame name to its pandas DataFrame; empty when calculations is None or has no frames. + :raises ImportError: if the [analysis] extra is not installed. + :raises ValueError: if two frames share a name, or a frame cannot be converted. + """ + _require_pandas() + + if calculations is None: + return {} + + frames: dict[str, Any] = {} + for entry in calculations.calculationDataFrames: + if entry.name in frames: + raise ValueError(f"Calculations carries more than one frame named '{entry.name}'; frame names are unique") + frames[entry.name] = data_frame_to_pandas(entry.frame, exclude_column_metadata=exclude_column_metadata) + return frames diff --git a/src/dp_python_lib/client/sample_status_client.py b/src/dp_python_lib/client/sample_status_client.py index 25b0d75..436e640 100644 --- a/src/dp_python_lib/client/sample_status_client.py +++ b/src/dp_python_lib/client/sample_status_client.py @@ -3,99 +3,29 @@ import grpc +from dp_python_lib.client.data_frame import sampling_clock, timestamp_list +from dp_python_lib.client.data_frame import timestamp_count as _timestamp_count from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp from dp_python_lib.client.result import ApiResultBase from dp_python_lib.client.service_api_client_base import ServiceApiClientBase from dp_python_lib.grpc import annotation_pb2, annotation_pb2_grpc, common_pb2 - -def sampling_clock( - start_time: TimestampInput, - period_nanos: int, - count: int, -) -> common_pb2.DataTimestamps: - """ - Builds a DataTimestamps with a SamplingClock time axis, the compact form for dense labeling of regularly-sampled - data. The clock must match the archived data's clock exactly -- status-to-sample matching is by exact timestamp - at nanosecond precision, so an off-by-one-nanosecond period misses every sample after the first. - - :param start_time: Time of the first sample (tz-aware datetime, epoch seconds, or common.Timestamp). - :param period_nanos: Period between samples, in nanoseconds. Must be > 0. - :param count: Number of samples in the interval. Must be >= 1. - :return: A DataTimestamps carrying a SamplingClock. - :raises ValueError: if period_nanos is not positive or count is less than 1. - """ - if period_nanos <= 0: - raise ValueError(f"sampling_clock() requires period_nanos > 0, got {period_nanos}") - if count < 1: - raise ValueError(f"sampling_clock() requires count >= 1, got {count}") - - timestamps = common_pb2.DataTimestamps() - timestamps.samplingClock.startTime.CopyFrom(to_timestamp(start_time)) - timestamps.samplingClock.periodNanos = period_nanos - timestamps.samplingClock.count = count - return timestamps - - -def timestamp_list(values: list[TimestampInput]) -> common_pb2.DataTimestamps: - """ - Builds a DataTimestamps with an explicit TimestampList time axis, the form for sparse labeling -- naming only the - samples being labeled. The unlabeled samples carry no assertion; there is no need to mark the rest "good". - - Timestamps must be strictly increasing, which is validated here rather than deferred to the server. Supply - timestamps taken from data query results (or exact SamplingClock arithmetic); a recomputed or rounded timestamp - will silently fail to match its sample. - - :param values: The timestamps to label (tz-aware datetimes, epoch seconds, or common.Timestamp objects). - :return: A DataTimestamps carrying a TimestampList. - :raises ValueError: if values is empty or the timestamps are not strictly increasing. - """ - if not values: - raise ValueError("timestamp_list() requires a non-empty values list") - - converted = [to_timestamp(value) for value in values] - - previous = converted[0] - for index, current in enumerate(converted[1:], start=1): - if (current.epochSeconds, current.nanoseconds) <= (previous.epochSeconds, previous.nanoseconds): - raise ValueError( - f"timestamp_list() requires strictly increasing timestamps; entry {index} " - f"({current.epochSeconds}.{current.nanoseconds:09d}) does not follow entry {index - 1} " - f"({previous.epochSeconds}.{previous.nanoseconds:09d})" - ) - previous = current - - timestamps = common_pb2.DataTimestamps() - timestamps.timestampList.timestamps.extend(converted) - return timestamps - - -def _timestamp_count(timestamps: common_pb2.DataTimestamps) -> int: - """ - Returns the number of timestamps a DataTimestamps describes, for validating parallel-array lengths. - - An empty axis is rejected here rather than allowed to surface later as a confusing column-length mismatch. - The axis builders already make this unreachable -- sampling_clock() requires count >= 1 and timestamp_list() - requires a non-empty list -- but a hand-built DataTimestamps can still carry a zero-count SamplingClock or an - empty TimestampList, and both report their oneof arm as set. Rejecting them keeps this in step with - expand_data_timestamps(), which applies the same rule on the read path. - - :param timestamps: The time axis to measure. - :return: The number of timestamps on the axis. - :raises ValueError: if neither axis form is set, or if the axis describes no timestamps. - """ - axis = timestamps.WhichOneof("value") - if axis == "samplingClock": - count = timestamps.samplingClock.count - if count < 1: - raise ValueError(f"DataTimestamps samplingClock requires count >= 1, got {count}") - return count - if axis == "timestampList": - count = len(timestamps.timestampList.timestamps) - if count < 1: - raise ValueError("DataTimestamps timestampList requires at least one timestamp, got an empty list") - return count - raise ValueError("DataTimestamps must specify either a samplingClock or a timestampList") +# sampling_clock(), timestamp_list(), and timestamp_count() moved to data_frame.py in issue #6 Phase 2, once +# calculations frames became a second caller. They are re-exported here (and from dp_python_lib.client) so existing +# imports keep working; __all__ names them explicitly so the re-export reads as deliberate rather than as an unused +# import. _timestamp_count is the private alias this module has always used. +__all__ = [ + "DeleteSampleStatusesApiResult", + "QuerySampleStatusesApiResult", + "QuerySampleStatusesRequestParams", + "SampleStatusClient", + "SampleStatusColumn", + "SampleStatusFrame", + "SaveSampleStatusesApiResult", + "SaveSampleStatusesRequestParams", + "sampling_clock", + "timestamp_list", +] class SampleStatusColumn: diff --git a/tests/unit/test_data_frame.py b/tests/unit/test_data_frame.py new file mode 100644 index 0000000..7e8a76c --- /dev/null +++ b/tests/unit/test_data_frame.py @@ -0,0 +1,356 @@ +import os +import sys +import unittest +from datetime import datetime, timezone + +# Add src directory to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src")) + +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.grpc import common_pb2 + +T0 = datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc) +T1 = datetime(2026, 7, 14, 19, 0, 0, tzinfo=timezone.utc) + + +def _axis(count=3): + """A SamplingClock axis of the given length, at 1 Hz.""" + return dfb.sampling_clock(T0, 1_000_000_000, count) + + +class TestSamplingClock(unittest.TestCase): + """The axis builders relocated here from sample_status_client in Phase 2; behavior is unchanged.""" + + def test_builds_clock(self): + axis = dfb.sampling_clock(T0, 1_000_000, 3) + self.assertEqual(axis.samplingClock.startTime.epochSeconds, int(T0.timestamp())) + self.assertEqual(axis.samplingClock.periodNanos, 1_000_000) + self.assertEqual(axis.samplingClock.count, 3) + + def test_rejects_non_positive_period(self): + for period in (0, -1): + with self.assertRaises(ValueError): + dfb.sampling_clock(T0, period, 3) + + def test_rejects_count_below_one(self): + with self.assertRaises(ValueError): + dfb.sampling_clock(T0, 1_000_000, 0) + + def test_still_importable_from_sample_status_client(self): + # The relocation must not break existing imports. + from dp_python_lib.client.sample_status_client import sampling_clock, timestamp_list + + self.assertIs(sampling_clock, dfb.sampling_clock) + self.assertIs(timestamp_list, dfb.timestamp_list) + + +class TestTimestampList(unittest.TestCase): + def test_builds_list(self): + axis = dfb.timestamp_list([100, 200, 300]) + self.assertEqual([t.epochSeconds for t in axis.timestampList.timestamps], [100, 200, 300]) + + def test_rejects_empty(self): + with self.assertRaises(ValueError): + dfb.timestamp_list([]) + + def test_rejects_non_increasing(self): + for values in ([100, 300, 200], [100, 100]): + with self.assertRaises(ValueError): + dfb.timestamp_list(values) + + def test_rejects_non_increasing_at_nanosecond_precision(self): + with self.assertRaises(ValueError): + dfb.timestamp_list([100.000_002, 100.000_001]) + + +class TestTimestampCount(unittest.TestCase): + def test_counts_both_axis_forms(self): + self.assertEqual(dfb.timestamp_count(dfb.sampling_clock(T0, 1_000, 7)), 7) + self.assertEqual(dfb.timestamp_count(dfb.timestamp_list([1, 2])), 2) + + def test_rejects_unset_axis(self): + with self.assertRaises(ValueError): + dfb.timestamp_count(common_pb2.DataTimestamps()) + + def test_rejects_hand_built_empty_axes(self): + # The builders make these unreachable, but a hand-built axis can still carry them. + empty_clock = common_pb2.DataTimestamps() + empty_clock.samplingClock.count = 0 + empty_clock.samplingClock.periodNanos = 1 + with self.assertRaises(ValueError): + dfb.timestamp_count(empty_clock) + + empty_list = common_pb2.DataTimestamps() + empty_list.timestampList.SetInParent() + with self.assertRaises(ValueError): + dfb.timestamp_count(empty_list) + + +class TestScalarColumnBuilders(unittest.TestCase): + """Each typed scalar builder produces the right message with the right values.""" + + def test_each_builder_and_field(self): + cases = [ + (dfb.double_column("d", [1.0, 2.0]), common_pb2.DoubleColumn, [1.0, 2.0]), + (dfb.int64_column("i64", [1, 2]), common_pb2.Int64Column, [1, 2]), + (dfb.int32_column("i32", [1, 2]), common_pb2.Int32Column, [1, 2]), + (dfb.bool_column("b", [True, False]), common_pb2.BoolColumn, [True, False]), + (dfb.string_column("s", ["a", "b"]), common_pb2.StringColumn, ["a", "b"]), + ] + for column, expected_type, expected_values in cases: + with self.subTest(column=column.name): + self.assertIsInstance(column, expected_type) + self.assertEqual(list(column.values), expected_values) + + def test_float_column_is_float32(self): + column = dfb.float_column("f", [1.5, 2.5]) + self.assertIsInstance(column, common_pb2.FloatColumn) + self.assertEqual(list(column.values), [1.5, 2.5]) + + def test_enum_column_carries_enum_id(self): + column = dfb.enum_column("state", [0, 1, 2], enum_id="beam-state") + self.assertEqual(list(column.values), [0, 1, 2]) + self.assertEqual(column.enumId, "beam-state") + + def test_enum_column_requires_enum_id(self): + with self.assertRaises(ValueError) as ctx: + dfb.enum_column("state", [0], enum_id="") + self.assertIn("enum_id", str(ctx.exception)) + + def test_metadata_is_attached(self): + metadata = dfb.column_metadata(tags=["derived"]) + column = dfb.double_column("d", [1.0], metadata=metadata) + self.assertEqual(list(column.metadata.tags), ["derived"]) + + def test_rejects_empty_name_and_values(self): + for builder in (dfb.double_column, dfb.float_column, dfb.int64_column, dfb.int32_column, dfb.string_column): + with self.subTest(builder=builder.__name__): + with self.assertRaises(ValueError): + builder("", [1]) + with self.assertRaises(ValueError): + builder("name", []) + + def test_empty_values_error_names_the_column(self): + with self.assertRaises(ValueError) as ctx: + dfb.double_column("x_rms", []) + self.assertIn("x_rms", str(ctx.exception)) + + +class TestDataColumnEscapeHatch(unittest.TestCase): + """The legacy DataColumn is the only way to express a gap on a shared axis.""" + + def test_none_becomes_an_unset_oneof(self): + column = dfb.data_column("sparse", [1.0, None, 3.0]) + arms = [v.WhichOneof("value") for v in column.dataValues] + self.assertEqual(arms, ["doubleValue", None, "doubleValue"]) + + def test_maps_each_python_type(self): + column = dfb.data_column("mixed", [True, 7, 1.5, "s", b"bytes"]) + arms = [v.WhichOneof("value") for v in column.dataValues] + self.assertEqual(arms, ["booleanValue", "longValue", "doubleValue", "stringValue", "byteArrayValue"]) + + def test_bool_is_checked_before_int(self): + # bool is a subclass of int, so an int-first check would store True as longValue 1. + column = dfb.data_column("flags", [True, False]) + self.assertEqual([v.WhichOneof("value") for v in column.dataValues], ["booleanValue", "booleanValue"]) + self.assertEqual([v.booleanValue for v in column.dataValues], [True, False]) + + def test_accepts_prebuilt_data_value(self): + prebuilt = common_pb2.DataValue() + prebuilt.uintValue = 42 + column = dfb.data_column("u", [prebuilt]) + self.assertEqual(column.dataValues[0].WhichOneof("value"), "uintValue") + self.assertEqual(column.dataValues[0].uintValue, 42) + + def test_rejects_unmappable_type(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_column("c", [1 + 2j]) + message = str(ctx.exception) + self.assertIn("complex", message) + self.assertIn("index 0", message) + + def test_rejects_empty_name_and_values(self): + with self.assertRaises(ValueError): + dfb.data_column("", [1]) + with self.assertRaises(ValueError): + dfb.data_column("name", []) + + +class TestProvenanceHelpers(unittest.TestCase): + def test_pv_source_sets_the_pv_name_arm(self): + source = dfb.pv_source("BPMS:GUNB:314:X") + self.assertEqual(source.WhichOneof("origin"), "pvName") + self.assertEqual(source.pvName, "BPMS:GUNB:314:X") + self.assertFalse(source.HasField("timeRange")) + + def test_pv_source_with_time_range(self): + source = dfb.pv_source("A:1", (T0, T1)) + self.assertTrue(source.HasField("timeRange")) + self.assertEqual(source.timeRange.beginTime.epochSeconds, int(T0.timestamp())) + self.assertEqual(source.timeRange.endTime.epochSeconds, int(T1.timestamp())) + + def test_calculations_source_sets_the_calculations_arm(self): + source = dfb.calculations_source("calc-1", "f1", "x_rms") + self.assertEqual(source.WhichOneof("origin"), "calculationsColumn") + self.assertEqual(source.calculationsColumn.calculationsId, "calc-1") + self.assertEqual(source.calculationsColumn.frameName, "f1") + self.assertEqual(source.calculationsColumn.columnName, "x_rms") + + def test_sources_reject_empty_identifiers(self): + with self.assertRaises(ValueError): + dfb.pv_source("") + with self.assertRaises(ValueError): + dfb.calculations_source("", "f", "c") + with self.assertRaises(ValueError): + dfb.calculations_source("calc", "", "c") + with self.assertRaises(ValueError): + dfb.calculations_source("calc", "f", "") + + def test_sources_reject_reversed_time_range(self): + with self.assertRaises(ValueError) as ctx: + dfb.pv_source("A:1", (T1, T0)) + self.assertIn("strictly before", str(ctx.exception)) + + def test_provenance_fields(self): + result = dfb.provenance( + source="analysis-rig", + process="1 Hz RMS", + derived_from=[dfb.pv_source("A:1"), dfb.calculations_source("c", "f", "x")], + ) + self.assertEqual(result.source, "analysis-rig") + self.assertEqual(result.process, "1 Hz RMS") + self.assertEqual(len(result.derivedFrom), 2) + + def test_provenance_omits_unset_fields(self): + result = dfb.provenance() + self.assertEqual(result.source, "") + self.assertEqual(result.process, "") + self.assertEqual(len(result.derivedFrom), 0) + + def test_column_metadata_fields(self): + metadata = dfb.column_metadata( + tags=["derived", "reviewed"], + attributes={"unit": "mm"}, + provenance=dfb.provenance(process="p"), + ) + self.assertEqual(list(metadata.tags), ["derived", "reviewed"]) + self.assertEqual({(a.name, a.value) for a in metadata.attributes}, {("unit", "mm")}) + self.assertEqual(metadata.provenance.process, "p") + + def test_column_metadata_empty(self): + metadata = dfb.column_metadata() + self.assertEqual(list(metadata.tags), []) + self.assertEqual(len(metadata.attributes), 0) + self.assertFalse(metadata.HasField("provenance")) + + +class TestDataFrameAssembly(unittest.TestCase): + """data_frame() routes columns by type and enforces the server's shape rules client-side.""" + + def test_routes_each_column_type_to_its_field(self): + frame = dfb.data_frame( + _axis(2), + [ + dfb.double_column("d", [1.0, 2.0]), + dfb.float_column("f", [1.0, 2.0]), + dfb.int64_column("i64", [1, 2]), + dfb.int32_column("i32", [1, 2]), + dfb.bool_column("b", [True, False]), + dfb.string_column("s", ["a", "b"]), + dfb.enum_column("e", [0, 1], enum_id="enum-1"), + dfb.data_column("legacy", [1.0, None]), + ], + ) + self.assertEqual([c.name for c in frame.doubleColumns], ["d"]) + self.assertEqual([c.name for c in frame.floatColumns], ["f"]) + self.assertEqual([c.name for c in frame.int64Columns], ["i64"]) + self.assertEqual([c.name for c in frame.int32Columns], ["i32"]) + self.assertEqual([c.name for c in frame.boolColumns], ["b"]) + self.assertEqual([c.name for c in frame.stringColumns], ["s"]) + self.assertEqual([c.name for c in frame.enumColumns], ["e"]) + self.assertEqual([c.name for c in frame.dataColumns], ["legacy"]) + + def test_copies_the_axis(self): + axis = _axis(3) + frame = dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0, 3.0])]) + self.assertEqual(frame.dataTimestamps.samplingClock.count, 3) + + def test_rejects_empty_column_list(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), []) + self.assertIn("at least one column", str(ctx.exception)) + + def test_rejects_count_mismatch_naming_the_column(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(3), [dfb.double_column("x_rms", [1.0, 2.0])]) + message = str(ctx.exception) + self.assertIn("x_rms", message) + self.assertIn("2 values", message) + self.assertIn("3 timestamps", message) + + def test_rejects_duplicate_names_across_types(self): + # Uniqueness is across ALL column types, not just within one. + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), [dfb.double_column("dup", [1.0]), dfb.string_column("dup", ["a"])]) + self.assertIn("dup", str(ctx.exception)) + + def test_rejects_unnamed_prebuilt_column(self): + unnamed = common_pb2.DoubleColumn() + unnamed.values[:] = [1.0] + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), [unnamed]) + self.assertIn("index 0", str(ctx.exception)) + + def test_rejects_unsupported_column_type(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), ["not a column"]) + self.assertIn("unsupported column type", str(ctx.exception)) + + def test_rejects_empty_axis(self): + with self.assertRaises(ValueError): + dfb.data_frame(common_pb2.DataTimestamps(), [dfb.double_column("d", [1.0])]) + + def test_accepts_prebuilt_array_column(self): + # Array builders are #17's; a hand-built one passes through, with dims giving the sample count. + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([4]) + column.values[:] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] # 2 samples x 4 + frame = dfb.data_frame(_axis(2), [column]) + self.assertEqual([c.name for c in frame.doubleArrayColumns], ["waveform"]) + + def test_array_column_count_uses_dims_product(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([4]) + column.values[:] = [1.0] * 8 # 2 samples, but the axis says 3 + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(3), [column]) + self.assertIn("2 values", str(ctx.exception)) + + def test_array_column_without_dims_is_rejected(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.values[:] = [1.0, 2.0] + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(2), [column]) + self.assertIn("dimensions", str(ctx.exception)) + + def test_serialized_column_gets_name_checks_only(self): + # Serialized payloads are opaque, so the server checks names only; this must not demand a count match. + column = common_pb2.SerializedDataColumn() + column.name = "serialized" + column.encoding = "arrow" + column.payload = b"opaque" + frame = dfb.data_frame(_axis(3), [column]) + self.assertEqual([c.name for c in frame.serializedDataColumns], ["serialized"]) + + def test_serialized_column_still_needs_a_unique_name(self): + column = common_pb2.SerializedDataColumn() + column.name = "dup" + with self.assertRaises(ValueError): + dfb.data_frame(_axis(1), [dfb.double_column("dup", [1.0]), column]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_data_frame_conversions.py b/tests/unit/test_data_frame_conversions.py new file mode 100644 index 0000000..f3b86e7 --- /dev/null +++ b/tests/unit/test_data_frame_conversions.py @@ -0,0 +1,429 @@ +import os +import sys +import unittest +from datetime import datetime, timezone + +# Add src directory to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src")) + +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc +from dp_python_lib.client.annotations_client import calculations +from dp_python_lib.client.query_conversions import Image +from dp_python_lib.grpc import annotation_pb2, common_pb2 + +# The pandas half depends on the optional [analysis] extra; skip those tests cleanly when it is absent. The +# pure-Python read side needs no optional deps and always runs. +try: + import pandas as pd + + _HAVE_ANALYSIS = True +except ImportError: + _HAVE_ANALYSIS = False + +T0 = datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc) +T0_NANOS = int(T0.timestamp()) * 1_000_000_000 + + +def _axis(count=3, period_nanos=1_000_000_000): + return dfb.sampling_clock(T0, period_nanos, count) + + +# ---------------------------------------------------------------------- +# pure Python read side (no optional dependencies) +# ---------------------------------------------------------------------- + + +class TestDataFrameTimestamps(unittest.TestCase): + def test_sampling_clock_expands_in_exact_integer_nanoseconds(self): + # The whole point of the integer path: a float64 cannot hold present-day epoch nanos, so a float + # round-trip would move every one of these. + frame = dfb.data_frame(_axis(3, period_nanos=1), [dfb.double_column("d", [1.0, 2.0, 3.0])]) + self.assertEqual(dfc.data_frame_timestamps(frame), [T0_NANOS, T0_NANOS + 1, T0_NANOS + 2]) + + def test_timestamp_list_is_returned_in_order(self): + axis = dfb.timestamp_list([100, 200, 300]) + frame = dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0, 3.0])]) + self.assertEqual( + dfc.data_frame_timestamps(frame), + [100_000_000_000, 200_000_000_000, 300_000_000_000], + ) + + def test_frame_without_axis_raises(self): + frame = common_pb2.DataFrame() + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_timestamps(frame) + self.assertIn("dataTimestamps", str(ctx.exception)) + + +class TestColumnValues(unittest.TestCase): + """column_values() is a standalone per-column converter, reusable by the bucket query (#16).""" + + def test_each_scalar_column_type(self): + cases = [ + (dfb.double_column("d", [1.5, 2.5]), [1.5, 2.5]), + (dfb.int64_column("i", [1, 2]), [1, 2]), + (dfb.int32_column("i32", [3, 4]), [3, 4]), + (dfb.bool_column("b", [True, False]), [True, False]), + (dfb.string_column("s", ["a", "b"]), ["a", "b"]), + (dfb.enum_column("e", [0, 1], enum_id="x"), [0, 1]), + ] + for column, expected in cases: + with self.subTest(column=column.name): + self.assertEqual(dfc.column_values(column), expected) + + def test_float_column_values_round_to_float32(self): + # float32 storage is lossy by nature; assert the round trip rather than exact float64 equality. + column = dfb.float_column("f", [1.5, 2.25]) + self.assertEqual(dfc.column_values(column), [1.5, 2.25]) + + def test_data_column_unset_value_becomes_none(self): + # The one representation of a gap in this API. + column = dfb.data_column("sparse", [1.0, None, 3.0]) + self.assertEqual(dfc.column_values(column), [1.0, None, 3.0]) + + def test_data_column_complex_arms(self): + column = common_pb2.DataColumn() + column.name = "mixed" + column.dataValues.add().stringValue = "s" + image = column.dataValues.add() + image.imageValue.image = b"png-bytes" + image.imageValue.fileType = common_pb2.Image.FileType.PNG + + values = dfc.column_values(column) + self.assertEqual(values[0], "s") + self.assertEqual(values[1], Image(b"png-bytes", "PNG")) + + def test_array_column_is_reshaped_per_sample(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([3]) + column.values[:] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] + self.assertEqual(dfc.column_values(column), [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) + + def test_multidimensional_array_stays_flat_within_a_sample(self): + column = common_pb2.Int32ArrayColumn() + column.name = "grid" + column.dimensions.dims.extend([2, 2]) + column.values[:] = [1, 2, 3, 4, 5, 6, 7, 8] + self.assertEqual(dfc.column_values(column), [[1, 2, 3, 4], [5, 6, 7, 8]]) + + def test_array_column_without_dims_raises(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.values[:] = [1.0, 2.0] + with self.assertRaises(ValueError) as ctx: + dfc.column_values(column) + self.assertIn("dimensions", str(ctx.exception)) + + def test_array_column_with_indivisible_values_raises(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([3]) + column.values[:] = [1.0, 2.0, 3.0, 4.0] # not a whole multiple of 3 + with self.assertRaises(ValueError) as ctx: + dfc.column_values(column) + self.assertIn("not a whole multiple", str(ctx.exception)) + + def test_image_column_yields_bytes_per_sample(self): + column = common_pb2.ImageColumn() + column.name = "frames" + column.images.extend([b"a", b"b"]) + self.assertEqual(dfc.column_values(column), [b"a", b"b"]) + + def test_unsupported_type_raises(self): + with self.assertRaises(ValueError): + dfc.column_values("not a column") + + +class TestDataFrameColumns(unittest.TestCase): + def test_collects_every_column_type_by_name(self): + frame = dfb.data_frame( + _axis(2), + [ + dfb.double_column("d", [1.0, 2.0]), + dfb.string_column("s", ["a", "b"]), + dfb.data_column("legacy", [1.0, None]), + ], + ) + self.assertEqual(dfc.data_frame_columns(frame), {"d": [1.0, 2.0], "s": ["a", "b"], "legacy": [1.0, None]}) + + def test_duplicate_names_raise_rather_than_dropping(self): + # data_frame() prevents this, but a hand-built or older-server frame can still carry it. + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(_axis(1)) + for field in (frame.doubleColumns, frame.stringColumns): + column = field.add() + column.name = "dup" + frame.doubleColumns[0].values[:] = [1.0] + frame.stringColumns[0].values[:] = ["a"] + + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_columns(frame) + self.assertIn("dup", str(ctx.exception)) + + def test_serialized_columns_are_skipped(self): + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(_axis(1)) + double = frame.doubleColumns.add() + double.name = "d" + double.values[:] = [1.0] + serialized = frame.serializedDataColumns.add() + serialized.name = "opaque" + serialized.payload = b"blob" + + self.assertEqual(dfc.data_frame_columns(frame), {"d": [1.0]}) + + +class TestColumnMetadataDict(unittest.TestCase): + def test_full_metadata(self): + column = dfb.double_column( + "d", + [1.0], + metadata=dfb.column_metadata( + tags=["derived"], + attributes={"unit": "mm"}, + provenance=dfb.provenance( + source="rig", + process="RMS", + derived_from=[dfb.pv_source("A:1"), dfb.calculations_source("c", "f", "x")], + ), + ), + ) + result = dfc.column_metadata_dict(column) + + self.assertEqual(result["tags"], ["derived"]) + self.assertEqual(result["attributes"], {"unit": "mm"}) + self.assertEqual(result["provenance"]["source"], "rig") + self.assertEqual(result["provenance"]["process"], "RMS") + self.assertEqual(result["provenance"]["derived_from"][0]["pv_name"], "A:1") + self.assertIsNone(result["provenance"]["derived_from"][0]["calculations_column"]) + self.assertEqual( + result["provenance"]["derived_from"][1]["calculations_column"], + {"calculations_id": "c", "frame_name": "f", "column_name": "x"}, + ) + + def test_absent_metadata(self): + result = dfc.column_metadata_dict(dfb.double_column("d", [1.0])) + self.assertEqual(result, {"tags": [], "attributes": {}, "provenance": None}) + + +# ---------------------------------------------------------------------- +# pandas bridges ([analysis] extra) +# ---------------------------------------------------------------------- + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestDataFrameToPandas(unittest.TestCase): + def test_basic_conversion(self): + frame = dfb.data_frame( + _axis(3), + [dfb.double_column("x_rms", [12.7, 12.8, 12.9]), dfb.string_column("label", ["a", "b", "c"])], + ) + df = dfc.data_frame_to_pandas(frame) + + self.assertEqual(list(df.columns), ["x_rms", "label"]) + self.assertEqual(df["x_rms"].tolist(), [12.7, 12.8, 12.9]) + self.assertEqual(len(df), 3) + + def test_index_is_utc_and_nanosecond_exact(self): + # A SamplingClock at 1 ns spacing: only an integer path reproduces these instants. + frame = dfb.data_frame(_axis(3, period_nanos=1), [dfb.double_column("d", [1.0, 2.0, 3.0])]) + df = dfc.data_frame_to_pandas(frame) + + self.assertEqual(str(df.index.tz), "UTC") + self.assertEqual( + [int(v) for v in df.index.view("int64")], + [T0_NANOS, T0_NANOS + 1, T0_NANOS + 2], + ) + + def test_metadata_lands_in_attrs(self): + frame = dfb.data_frame( + _axis(1), + [dfb.double_column("d", [1.0], metadata=dfb.column_metadata(tags=["derived"]))], + ) + df = dfc.data_frame_to_pandas(frame) + + self.assertEqual(df.attrs["column_metadata"]["d"]["tags"], ["derived"]) + + def test_metadata_can_be_excluded(self): + frame = dfb.data_frame(_axis(1), [dfb.double_column("d", [1.0])]) + df = dfc.data_frame_to_pandas(frame, exclude_column_metadata=True) + self.assertNotIn("column_metadata", df.attrs) + + def test_gap_becomes_nan(self): + frame = dfb.data_frame(_axis(3), [dfb.data_column("sparse", [1.0, None, 3.0])]) + df = dfc.data_frame_to_pandas(frame) + self.assertTrue(df["sparse"].isna().tolist() == [False, True, False]) + + def test_misaligned_column_raises(self): + # Hand-built: data_frame() would have caught this on the way out. + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(_axis(3)) + column = frame.doubleColumns.add() + column.name = "short" + column.values[:] = [1.0, 2.0] + + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_to_pandas(frame) + message = str(ctx.exception) + self.assertIn("short", message) + self.assertIn("index-aligned", message) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestDataFrameFromPandas(unittest.TestCase): + def _index(self, periods=3): + return pd.date_range("2026-07-14 18:00", periods=periods, freq="s", tz="UTC") + + def test_dtype_mapping(self): + import numpy as np + + df = pd.DataFrame( + { + "d": [1.0, 2.0], + "f": np.array([1.5, 2.5], dtype="float32"), + "i64": np.array([1, 2], dtype="int64"), + "i32": np.array([1, 2], dtype="int32"), + "b": [True, False], + "s": ["a", "b"], + }, + index=self._index(2), + ) + frame = dfc.data_frame_from_pandas(df) + + self.assertEqual([c.name for c in frame.doubleColumns], ["d"]) + self.assertEqual([c.name for c in frame.floatColumns], ["f"]) + self.assertEqual([c.name for c in frame.int64Columns], ["i64"]) + self.assertEqual([c.name for c in frame.int32Columns], ["i32"]) + self.assertEqual([c.name for c in frame.boolColumns], ["b"]) + self.assertEqual([c.name for c in frame.stringColumns], ["s"]) + + def test_index_becomes_a_timestamp_list(self): + # Never a SamplingClock: inferring uniformity from an index that merely looks regular would move + # timestamps, which this API cannot tolerate. + df = pd.DataFrame({"d": [1.0, 2.0, 3.0]}, index=self._index()) + frame = dfc.data_frame_from_pandas(df) + + self.assertEqual(frame.dataTimestamps.WhichOneof("value"), "timestampList") + self.assertEqual(len(frame.dataTimestamps.timestampList.timestamps), 3) + + def test_round_trip_preserves_values_and_instants(self): + index = self._index() + df = pd.DataFrame({"d": [1.0, 2.0, 3.0], "s": ["a", "b", "c"]}, index=index) + + back = dfc.data_frame_to_pandas(dfc.data_frame_from_pandas(df)) + + self.assertEqual(back["d"].tolist(), [1.0, 2.0, 3.0]) + self.assertEqual(back["s"].tolist(), ["a", "b", "c"]) + self.assertEqual( + [int(v) for v in back.index.view("int64")], + [int(v) for v in index.view("int64")], + ) + + def test_nan_is_fail_loud_with_sparsity_guidance(self): + df = pd.DataFrame({"x": [1.0, None, 3.0]}, index=self._index()) + + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + message = str(ctx.exception) + self.assertIn("x", message) + self.assertIn("missing values", message) + self.assertIn("data_column", message) + + def test_naive_index_is_rejected(self): + df = pd.DataFrame({"x": [1.0]}, index=pd.date_range("2026-07-14", periods=1, freq="s")) + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + self.assertIn("timezone-aware", str(ctx.exception)) + + def test_non_datetime_index_is_rejected(self): + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(pd.DataFrame({"x": [1.0]})) + self.assertIn("DatetimeIndex", str(ctx.exception)) + + def test_non_increasing_index_is_rejected(self): + index = pd.DatetimeIndex(["2026-07-14 18:00:01", "2026-07-14 18:00:00"], tz="UTC") + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(pd.DataFrame({"x": [1.0, 2.0]}, index=index)) + self.assertIn("strictly increasing", str(ctx.exception)) + + def test_no_columns_is_rejected(self): + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(pd.DataFrame(index=self._index())) + self.assertIn("at least one column", str(ctx.exception)) + + def test_unmappable_dtype_is_rejected(self): + df = pd.DataFrame({"c": [1 + 2j, 2 + 0j, 3 + 0j]}, index=self._index()) + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + message = str(ctx.exception) + self.assertIn("complex128", message) + self.assertIn("no typed-column mapping", message) + + def test_object_column_holding_non_strings_is_rejected(self): + df = pd.DataFrame({"o": [{"a": 1}, {"b": 2}, {"c": 3}]}, index=self._index()) + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + self.assertIn("non-string value", str(ctx.exception)) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestCalculationsBridges(unittest.TestCase): + def _index(self, periods=3): + return pd.date_range("2026-07-14 18:00", periods=periods, freq="s", tz="UTC") + + def test_round_trip_multiple_frames(self): + frames = { + "bpm-statistics": pd.DataFrame({"x_rms": [1.0, 2.0, 3.0]}, index=self._index()), + "rf-statistics": pd.DataFrame({"y_rms": [4.0, 5.0, 6.0]}, index=self._index()), + } + calcs = dfc.calculations_from_dataframes(frames) + + self.assertEqual( + {f.name for f in calcs.calculationDataFrames}, + {"bpm-statistics", "rf-statistics"}, + ) + + back = dfc.calculations_to_dataframes(calcs) + self.assertEqual(set(back), set(frames)) + self.assertEqual(back["bpm-statistics"]["x_rms"].tolist(), [1.0, 2.0, 3.0]) + self.assertEqual(back["rf-statistics"]["y_rms"].tolist(), [4.0, 5.0, 6.0]) + + def test_to_dataframes_none_is_empty(self): + self.assertEqual(dfc.calculations_to_dataframes(None), {}) + + def test_to_dataframes_no_frames_is_empty(self): + self.assertEqual(dfc.calculations_to_dataframes(annotation_pb2.Calculations()), {}) + + def test_to_dataframes_rejects_duplicate_frame_names(self): + calcs = annotation_pb2.Calculations() + for _ in range(2): + entry = calcs.calculationDataFrames.add() + entry.name = "dup" + entry.frame.CopyFrom(dfb.data_frame(_axis(1), [dfb.double_column("d", [1.0])])) + + with self.assertRaises(ValueError) as ctx: + dfc.calculations_to_dataframes(calcs) + self.assertIn("dup", str(ctx.exception)) + + def test_from_dataframes_rejects_empty(self): + with self.assertRaises(ValueError) as ctx: + dfc.calculations_from_dataframes({}) + self.assertIn("at least one frame", str(ctx.exception)) + + def test_from_dataframes_rejects_empty_frame_name(self): + with self.assertRaises(ValueError): + dfc.calculations_from_dataframes({"": pd.DataFrame({"x": [1.0]}, index=self._index(1))}) + + def test_metadata_survives_to_pandas(self): + frame = dfb.data_frame( + _axis(1), + [dfb.double_column("d", [1.0], metadata=dfb.column_metadata(tags=["derived"]))], + ) + back = dfc.calculations_to_dataframes(calculations({"f1": frame})) + self.assertEqual(back["f1"].attrs["column_metadata"]["d"]["tags"], ["derived"]) + + +if __name__ == "__main__": + unittest.main() From 99c0c0694561686b3c9dd69dfc2fde004c0b111f Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Wed, 9 Sep 2026 14:41:57 -0600 Subject: [PATCH 02/11] docs: datasets-and-annotations cookbook recipe and docs (issue #6, Phase 4) Completes issue #6 with the documentation and the remaining integration coverage. doc/cookbook/datasets-and-annotations.md continues the cookbook's shared worked example -- a dataset over the first hour of the CXI_3443 shift, an orbit-drift annotation, a 1 Hz RMS calculation carrying provenance back to the source PV, and the export -- and covers the things most likely to bite: - The archive-existence rule, including why its error text ("no PV metadata found for names") misdescribes the check it failed. - Calculations belonging to their annotation, so a full-replace save without them deletes them. - Gaps: the typed columns are dense, so data_column() or a separate frame. - Why the pandas direction never infers a SamplingClock, and why NaN is fail-loud. - That there is no download: file_path is a server-side path. Integration test grew the Phase 4 legs: builder-made calculations read back through data_frame_conversions on a sub-second axis (so the round trip exercises nanosecond arithmetic rather than whole seconds a float could also represent), provenance survival, and four export cases. 25 tests, 12 subtests, all passing against dp-service fddf692. Also: README moves the three Annotation Service bullets from TODO to Current state; CLAUDE.md gains the usage section promised in PR 1; the cookbook README lists the recipe and extends the worked-example description; and the snippet checker's preamble gains the new names plus the recipe's shared handles. The CLAUDE.md snippet was extracted and RUN against the live server, not just type-checked -- save, query, batch fetch, calculations read-back in both plain Python and pandas, export, and teardown all succeed end to end. 699 tests pass with the [analysis] extra; 601 pass and 56 skip cleanly without it. ruff, format, and all 101 cookbook snippets clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- .dev/tools/check-cookbook-snippets.py | 29 + CLAUDE.md | 62 +- README.md | 16 +- doc/cookbook/README.md | 3 + doc/cookbook/datasets-and-annotations.md | 572 ++++++++++++++++++ plan/tickets/6/plan.md | 9 + .../test_datasets_annotations_integration.py | 138 +++++ 7 files changed, 822 insertions(+), 7 deletions(-) create mode 100644 doc/cookbook/datasets-and-annotations.md diff --git a/.dev/tools/check-cookbook-snippets.py b/.dev/tools/check-cookbook-snippets.py index 2e30bfc..71e400c 100755 --- a/.dev/tools/check-cookbook-snippets.py +++ b/.dev/tools/check-cookbook-snippets.py @@ -104,10 +104,26 @@ QuerySampleStatusesRequestParams, sampling_clock, timestamp_list, + DataSetClient, + DataSetQuery, + DataSetQuery as DS, + SaveDataSetRequestParams, + data_block, + AnnotationsClient, + AnnotationQuery, + AnnotationQuery as AQ, + SaveAnnotationRequestParams, + calculations, + ExportClient, + ExportFormat, + ExportDataRequestParams, + calculations_spec, ) from dp_python_lib.client import query_conversions as qc from dp_python_lib.client import sample_status_conversions as ssc +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc client: MldpClient = MldpClient() begin: datetime = datetime(2024, 1, 1, tzinfo=timezone.utc) @@ -117,6 +133,19 @@ # building the query. Snippets that demonstrate query *construction* build their own. params: QueryParams = QueryParams( begin_time=begin, end_time=end, pv_selector=PV.name_list(["BPMS:GUNB:314:X"])) + +# The datasets-and-annotations recipe is one continuous worked example: it saves a dataset, then an +# annotation on it, then calculations, then exports and deletes them. These are the handles the +# recipe establishes in its own earlier snippets and legitimately carries forward, the way `client` +# and `params` are carried above. Server-assigned ids are strings. +t0: datetime = datetime(2026, 2, 2, 18, 0, tzinfo=timezone.utc) +t1: datetime = datetime(2026, 2, 2, 19, 0, tzinfo=timezone.utc) +# Typed `str` because the recipe's snippets narrow the Optional accessors with an assert before +# carrying the id forward -- which is the pattern conventions.md teaches, so the declared type here +# is the post-narrowing one. +dataset_id: str = "6aa1bb271a768e97db44d426" +annotation_id: str = "6aa1bb271a768e97db44d427" +calculations_id: str = "6aa1bb271a768e97db44d428" # --- end preamble --- """ diff --git a/CLAUDE.md b/CLAUDE.md index cc93563..153955b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -529,10 +529,62 @@ Notes: ### DataSets, Annotations, and Export API (Annotation Service) -Phase 1 of issue #6 (`plan/tickets/6/plan.md`) added three feature clients on the `annotation` facade: +Issue #6 (`plan/tickets/6/plan.md`) added three feature clients on the `annotation` facade: `client.annotation.datasets` (`DataSetClient`), `client.annotation.annotations` (`AnnotationsClient`), and -`client.annotation.export` (`ExportClient`). The full usage section lands with the cookbook recipe in PR 2; the -invariants worth knowing before touching this code: +`client.annotation.export` (`ExportClient`). A DataSet names a region of the archive; an Annotation describes one +or more DataSets and may own a Calculations payload of derived values; export writes any of it to a file on the +server. Worked example: `doc/cookbook/datasets-and-annotations.md`. + +```python +from datetime import datetime, timezone +from dp_python_lib.client import ( + MldpClient, SaveDataSetRequestParams, SaveAnnotationRequestParams, ExportDataRequestParams, ExportFormat, + DataSetQuery as DS, AnnotationQuery as AQ, data_block, calculations, calculations_spec, sampling_clock, +) +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc + +client = MldpClient() +ds, an, ex = client.annotation.datasets, client.annotation.annotations, client.annotation.export +t0 = datetime(2026, 2, 2, 18, tzinfo=timezone.utc) +t1 = datetime(2026, 2, 2, 19, tzinfo=timezone.utc) + +# a region of the archive: one DataBlock per (time range, PV list). Every PV must already have +# ingested data -- see the archive-existence invariant below. +dataset_id = ds.save_dataset(SaveDataSetRequestParams( + name="CXI shift, hour 1", owner_id="cmcchesney", + data_blocks=[data_block(t0, t1, ["BPMS:GUNB:314:X"])], + tags=["cxi-3443"], attributes={"EXP": "CXI_3443"}, modified_by="cmcchesney")).dataset_id + +# derived values with column-level provenance, one frame per time axis +frame = dfb.data_frame( + sampling_clock(start_time=t0, period_nanos=1_000_000_000, count=3), + [dfb.double_column("x_rms", [0.31, 0.29, 0.33], metadata=dfb.column_metadata( + provenance=dfb.provenance(process="1 Hz RMS", + derived_from=[dfb.pv_source("BPMS:GUNB:314:X", (t0, t1))])))]) +saved = an.save_annotation(SaveAnnotationRequestParams( + name="Orbit drift", owner_id="cmcchesney", dataset_ids=[dataset_id], + tags=["reviewed"], calculations=calculations({"orbit-rms": frame}))) + +# find it again; query results carry ids, so resolve a page's datasets in ONE call +for a in an.iter_annotations([AQ.tags(["reviewed"]), AQ.attributes("EXP")]): # key-only search + print(a.name, a.dataSetIds, bool(a.calculationsId)) +datasets = ds.get_datasets([i for a in an.iter_annotations([AQ.datasets([dataset_id])]) + for i in a.dataSetIds]) + +# read the calculations back (get_annotation is the only method returning them inline) +calcs = an.get_calculations(saved.calculations_id).calculations +columns = dfc.data_frame_columns(calcs.calculationDataFrames[0].frame) # plain Python, no extras +frames = dfc.calculations_to_dataframes(calcs) # pandas, [analysis] extra + +# export; then tear down annotations first, since delete_dataset is refused while referenced +ex.export_data(ExportDataRequestParams(ExportFormat.HDF5, dataset_id=dataset_id, + calculations_spec=calculations_spec(saved.calculations_id))) +an.delete_annotation(saved.annotation_id) +ds.delete_dataset(dataset_id) +``` + +Invariants worth knowing before touching this code: - **`saveDataSet` requires every PV named in a data block to already exist in the archive** — that is, to have *ingested data*. The server's error text says `no PV metadata found for names: [...]`, but the check is a @@ -579,6 +631,10 @@ invariants worth knowing before touching this code: large `$in` and an oversized request message. A short result is logged at WARNING, since an id withheld for any other reason is indistinguishable from a dangling one. - `patchDataSet` / `patchAnnotation` are reserved "not implemented" placeholders and are not wrapped. +- Calculations are built with `data_frame.py` and read back with `data_frame_conversions.py`; both are shared with + #16/#17 rather than local to this area. `data_frame_from_pandas()` always emits a `TimestampList`, never an + inferred `SamplingClock`, and rejects `NaN` fail-loud: a dense typed column cannot express a gap, so use + `data_column()` or a separate frame. ### Sample Status API (Annotation Service) diff --git a/README.md b/README.md index 7750d67..3db05bf 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,18 @@ for their service. transparent paging (`query_samples()` / `iter_query_samples()`) and server-streaming (`iter_query_samples_stream()`). Results convert to pandas DataFrames, NumPy arrays, and Excel via the optional `[analysis]` extra. +- **DataSets** — `client.annotation.datasets`. Name a region of the archive (time ranges plus the + PVs covered over them) so it can be found, annotated, and exported later: `save_dataset()`, + `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, and a + `get_datasets(ids)` batch fetch, with the `DataSetQuery` criterion helpers. +- **Annotations and calculations** — `client.annotation.annotations`. Attach conclusions and + derived values to those datasets, with column-level provenance recording what each column came + from: `save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, + `delete_annotation()`, `get_calculations()`, and the `AnnotationQuery` helpers. Calculations are + built with the `data_frame` builders and read back with `data_frame_conversions`, which also + bridges to pandas under the optional `[analysis]` extra. +- **Export** — `client.annotation.export`. Export a saved dataset, ad-hoc data blocks, and/or + calculations to HDF5, CSV, or XLSX. The file is written on the server; there is no retrieval RPC. - **Provider registration** — `client.ingestion_client.register_provider()`. The rest of the ingestion API is not yet implemented. @@ -97,10 +109,6 @@ Note the v2 query API comes from unreleased dp-grpc work and will not work again - `queryTable()` — PV time-series data in tabular format - `queryPvStats()` — archive ingestion statistics for PVs - `queryProviders()` / `queryProviderStats()` — provider information and ingestion statistics -- **Annotation Service** - - `saveDataSet()` / `queryDataSets()` — datasets over collections of PVs and time ranges - - `saveAnnotation()` / `queryAnnotations()` — annotations targeting a dataset - - `exportData()` — export datasets to common file formats - **Ingestion Stream Service** - `subscribeDataEvent()` — notification when a data condition in the ingestion stream triggers diff --git a/doc/cookbook/README.md b/doc/cookbook/README.md index 441c1db..9e1d983 100644 --- a/doc/cookbook/README.md +++ b/doc/cookbook/README.md @@ -26,6 +26,7 @@ client. | [Recording machine configuration](machine-configuration.md) | Defining configurations, recording when each was active, closing and opening intervals, and answering "what was the machine doing at 18:04?" | | [Querying time-series data](query.md) | Retrieving samples by PV name, by metadata, or by machine configuration, and converting results to pandas / NumPy / Excel | | [Labeling samples](sample-status.md) | Recording per-sample status codes, reading them back, and querying data with flagged samples excluded | +| [DataSets and annotations](datasets-and-annotations.md) | Naming a region of the archive, attaching analysis results with column-level provenance, round-tripping calculations through pandas, and exporting | **Not yet covered: getting data in.** `IngestionClient` currently exposes only `register_provider()`, so there is no ingestion recipe. @@ -45,6 +46,8 @@ query recipes is the data the earlier recipes create: `MODE=09`), activated over a shift with `DEST=CXI` and `EXP=CXI_3443`. - Queries that retrieve those PVs by name, by *"every monitor in GUNB"*, and by *"whatever ran during the CXI shift"*. +- A dataset naming the first hour of that shift, an annotation recording an orbit drift, and a 1 Hz + RMS calculation attached to it with provenance pointing back at the source PV. Attribute names and values are the facility's; tag values are illustrative placeholders. diff --git a/doc/cookbook/datasets-and-annotations.md b/doc/cookbook/datasets-and-annotations.md new file mode 100644 index 0000000..eebc52d --- /dev/null +++ b/doc/cookbook/datasets-and-annotations.md @@ -0,0 +1,572 @@ +# DataSets and Annotations + +Naming a region of the archive so you can find it again, attaching derived values to it with a +record of what they were computed from, and exporting the result. + +> **Verified against:** dp-grpc `rel-1.16.0`. +> The modernized DataSet / Annotation / Export API is **new in 1.16.0** and will not work against a +> `rel-1.15.0` server, which answers these calls with `UNIMPLEMENTED`. + +See [API conventions](conventions.md) for result checking, paging, and time handling. + +Examples use `client.annotation.datasets`, `client.annotation.annotations`, and +`client.annotation.export`. Note that `client.annotation` itself is `None` unless an annotation +channel is configured, so guard on it before reaching through. + +### Imports used by the examples + +```python +# cookbook:skip +from datetime import datetime, timezone + +from dp_python_lib.client import ( + MldpClient, + SaveDataSetRequestParams, + SaveAnnotationRequestParams, + ExportDataRequestParams, + ExportFormat, + DataSetQuery as DS, + AnnotationQuery as AQ, + data_block, + calculations, + calculations_spec, + sampling_clock, +) +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc +``` + +## Contents + +- [Model](#model) — three objects and how they relate +- [Naming a region of the archive](#naming-a-region-of-the-archive) +- [Finding datasets again](#finding-datasets-again) +- [Attaching an analysis result](#attaching-an-analysis-result) +- [Recording where the numbers came from](#recording-where-the-numbers-came-from) +- [Reading calculations back](#reading-calculations-back) +- [Going through pandas](#going-through-pandas) +- [Updating without losing the calculations](#updating-without-losing-the-calculations) +- [Exporting](#exporting) +- [Tearing down](#tearing-down) +- [Also worth knowing](#also-worth-knowing) + +## Model + +Three objects, in a chain: + +A **DataSet** names a region of the archive: a list of **DataBlocks**, each one a time range plus +the PVs covered over it. It holds no data — only the coordinates of data that already exists. + +An **Annotation** describes one or more DataSets: a name, a description, tags, and optionally a +**Calculations** payload of derived values. Annotations are how analysis results get attached to +the raw data they came from. + +**Calculations** are named frames, each one a time axis plus typed columns of values, with +optional per-column **provenance** recording what each column was derived from. + +``` +DataSet ──described by──> Annotation ──owns──> Calculations +(where the (what you (the derived + data is) concluded) numbers) +``` + +Two consequences worth internalizing before you start. + +**A DataSet can only name PVs that already have archived data.** `save_dataset()` checks that +every PV in every data block exists in the archive — not merely that it has PV metadata saved. A +dataset over a PV that has never been ingested is rejected: + +``` +no PV metadata found for names: [BPMS:GUNB:314:X] +``` + +That message is misleading: the check is against ingested data, not the metadata catalogue. If you +see it for a PV you know you catalogued, the PV has no samples. + +**Calculations belong to their annotation.** They have no independent lifecycle: saving an +annotation is the only way to write them, deleting the annotation deletes them, and re-saving an +annotation without them deletes them too. See +[updating without losing the calculations](#updating-without-losing-the-calculations). + +## Naming a region of the archive + +A `DataBlock` is one time range and the PVs measured over it. Build one per (range, PV list) pair; +a dataset is a list of them, so a study spanning two disjoint windows is one dataset with two +blocks. + +```python +# cookbook:partial +t0 = datetime(2026, 2, 2, 18, 0, tzinfo=timezone.utc) +t1 = datetime(2026, 2, 2, 19, 0, tzinfo=timezone.utc) + +saved = client.annotation.datasets.save_dataset(SaveDataSetRequestParams( + name="CXI shift, hour 1", + owner_id="cmcchesney", + data_blocks=[data_block(t0, t1, ["BPMS:GUNB:314:X", "BPMS:GUNB:314:Y"])], + description="First hour of the CXI_3443 shift, both transverse BPM planes.", + tags=["cxi-3443", "shift-study"], + attributes={"EXP": "CXI_3443", "DEST": "CXI"}, + modified_by="cmcchesney", +)) +if saved.result_status.is_error: + raise RuntimeError(saved.result_status.message) + +saved_id = saved.dataset_id +assert saved_id is not None # guaranteed once is_error is False; accessors are Optional +print(saved_id) # server-assigned id, e.g. '6aa1bb271a768e97db44d426' +``` + +`data_block()` requires `begin < end` and a non-empty PV list. That check exists here because the +server does not make it: it validates only that each bound is non-zero, so a reversed block would +otherwise be stored happily. + +Note the tags come back **normalized** — lowercased, deduplicated, and sorted — so a tag saved as +`CXI-3443` reads back as `cxi-3443`, and queries must match the lowercase form. + +## Finding datasets again + +Seven criteria, combined with the usual [AND across / OR within](conventions.md#query-criteria) +rules: + +```python +# cookbook:partial +# Everything from this experiment, in this area of the machine. +for dataset in client.annotation.datasets.iter_datasets([ + DS.attributes("EXP", ["CXI_3443"]), + DS.pv_names(["BPMS:GUNB:314:X"]), +]): + print(dataset.name, len(dataset.dataBlocks)) +``` + +Two criteria behave differently from the older helpers in this library: + +**`attributes()` takes an optional value list.** Omit it to ask *"does this key exist at all?"*, +regardless of value: + +```python +# cookbook:partial +# Every dataset tagged with an experiment, whichever one. +tagged = list(client.annotation.datasets.iter_datasets([DS.attributes("EXP")])) +``` + +**`criteria` itself is optional.** An empty or omitted list matches everything, so browsing the +whole collection is a legitimate call rather than something to work around: + +```python +# cookbook:partial +for dataset in client.annotation.datasets.iter_datasets(): + print(dataset.id, dataset.name) +``` + +`text()` is a full-text search across the name and description together. Only **one** text +criterion is allowed per query — two cannot be ANDed — and the client rejects a second one before +the RPC: + +```python +# cookbook:partial +# One text criterion: fine. +found = list(client.annotation.datasets.iter_datasets([DS.text("shift")])) + +# Two: ValueError, naming the rule. Combine the terms into one search instead. +``` + +### Fetching many datasets at once + +Annotations carry `dataSetIds`, not the datasets themselves. Resolving them one call at a time is +an N+1, so `get_datasets()` does it in a single paged query: + +```python +# cookbook:partial +ids = [i for a in client.annotation.annotations.iter_annotations([AQ.tags(["reviewed"])]) + for i in a.dataSetIds] +datasets = client.annotation.datasets.get_datasets(ids) # dict: id -> DataSet + +for dataset_id, dataset in datasets.items(): + print(dataset_id, dataset.name) +``` + +Ids are deduplicated, an empty list returns `{}` without an RPC, and ids that resolve to nothing +are simply **absent from the dict** rather than raising — a dangling `dataSetIds` entry is a normal +consequence of deletion, not an error. + +## Attaching an analysis result + +An annotation on its own records a conclusion: + +```python +# cookbook:partial +result = client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( + name="Orbit drift during CXI_3443", + owner_id="cmcchesney", + dataset_ids=[dataset_id], + description="Slow horizontal drift, ~0.3 mm over the hour. Suspect thermal.", + tags=["reviewed", "orbit"], + modified_by="cmcchesney", +)) +if result.result_status.is_error: + raise RuntimeError(result.result_status.message) + +saved_annotation_id = result.annotation_id +assert saved_annotation_id is not None +``` + +To attach *numbers*, build a `Calculations` payload. A frame is one time axis plus its columns: + +```python +# cookbook:partial +# Your analysis output: one value per sample, however you computed it. +x_rms_values = [0.31, 0.29, 0.33] # in reality, 3600 of them +y_rms_values = [0.12, 0.14, 0.11] + +# A 1 Hz RMS series: one sample per second, count matching the values above. +axis = sampling_clock(start_time=t0, period_nanos=1_000_000_000, count=len(x_rms_values)) + +frame = dfb.data_frame(axis, [ + dfb.double_column("x_rms", x_rms_values), + dfb.double_column("y_rms", y_rms_values), +]) + +result = client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( + name="1 Hz orbit RMS, CXI_3443 hour 1", + owner_id="cmcchesney", + dataset_ids=[dataset_id], + tags=["reviewed"], + calculations=calculations({"orbit-rms": frame}), + modified_by="cmcchesney", +)) +saved_calculations_id = result.calculations_id +assert saved_calculations_id is not None +``` + +`calculations()` takes a **dict** of frame name to frame, which makes frame-name uniqueness true by +construction — the server rejects duplicates, and a list would let you build one. + +The column builders validate the shape before anything goes over the wire: every column needs a +non-blank name and a value count matching the axis, and names must be unique **across all column +types** in a frame. A mismatch names the offending column: + +``` +data_frame() column 'x_rms' has 3599 values but the time axis has 3600 timestamps; +every column must carry exactly one value per sample +``` + +Size limits — string length, array element counts, image and struct bytes — are deliberately *not* +checked here. Those are deployment policy and can change; the server enforces them. + +### Columns with gaps + +The typed columns are dense: a `DoubleColumn` has one value per sample, with no way to mark one +absent. When a column genuinely has holes, use `data_column()`, whose values are `DataValue` +messages that can be left unset: + +```python +# cookbook:partial +# None becomes a genuinely absent value, not a fabricated zero. +sparse = dfb.data_column("beam_current", [12.7, None, 12.9]) +``` + +The alternative — and often the better one — is to give the sparse column **its own frame** over +only the timestamps where it has values, built with `timestamp_list()`. A frame per sampling rate +is the natural shape when your columns genuinely differ. + +## Recording where the numbers came from + +A calculated column without provenance is a number nobody can check later. `ColumnMetadata` records +what a column was derived from, per column: + +```python +# cookbook:partial +metadata = dfb.column_metadata( + tags=["derived"], + attributes={"unit": "mm"}, + provenance=dfb.provenance( + source="orbit-analysis-rig", + process="1 Hz RMS over 10 kHz BPM samples", + derived_from=[dfb.pv_source("BPMS:GUNB:314:X", (t0, t1))], + ), +) + +axis = sampling_clock(start_time=t0, period_nanos=1_000_000_000, count=3) +x_rms_values = [0.31, 0.29, 0.33] + +frame = dfb.data_frame(axis, [dfb.double_column("x_rms", x_rms_values, metadata=metadata)]) +``` + +`pv_source()` names an archived PV; `calculations_source()` names a column of *other* calculations, +for values derived from earlier derived values: + +```python +# cookbook:partial +chained = dfb.provenance( + process="10-minute moving average of the 1 Hz RMS", + derived_from=[dfb.calculations_source(calculations_id, "orbit-rms", "x_rms")], +) +``` + +Both accept an optional `(begin, end)` pair narrowing which part of the source was used. + +These are **soft references**. Deleting the annotation that owns the referenced calculations leaves +the link dangling; nothing resolves or cleans it up, and readers are expected to tolerate that. + +## Reading calculations back + +`get_annotation()` is the only method that returns calculations **inline**: + +```python +# cookbook:partial +read = client.annotation.annotations.get_annotation(annotation_id) +if read.result_status.is_error: + raise RuntimeError(read.result_status.message) + +calcs = read.calculations # None if the annotation has none +``` + +`query_annotations()` deliberately does not: results carry the `calculationsId` with the content +left empty, so listing a hundred annotations does not drag a hundred payloads with it. Follow the +id when you want the content: + +```python +# cookbook:partial +for annotation in client.annotation.annotations.iter_annotations([AQ.tags(["reviewed"])]): + if not annotation.calculationsId: + continue + fetched = client.annotation.annotations.get_calculations(annotation.calculationsId) + if fetched.result_status.is_error: + raise RuntimeError(fetched.result_status.message) + payload = fetched.calculations + assert payload is not None + print(annotation.name, len(payload.calculationDataFrames)) +``` + +Without pandas, `data_frame_conversions` returns plain Python: + +```python +# cookbook:partial +calcs = client.annotation.annotations.get_annotation(annotation_id).calculations +assert calcs is not None + +entry = calcs.calculationDataFrames[0] + +epoch_nanos = dfc.data_frame_timestamps(entry.frame) # list[int], exact +columns = dfc.data_frame_columns(entry.frame) # {"x_rms": [...], "y_rms": [...]} + +print(entry.name, len(epoch_nanos), list(columns)) +``` + +Timestamps are computed in **integer nanoseconds** throughout. A `SamplingClock` expands as +`startTime + i * periodNanos` with no float step anywhere: present-day epoch nanoseconds need about +61 bits, and a float64 carries 53, so routing through float seconds would silently move every +timestamp by hundreds of nanoseconds. + +## Going through pandas + +With the [`analysis` extra](conventions.md#optional-dependencies) installed, whole frames convert +in one call: + +```python +# cookbook:partial +calcs = client.annotation.annotations.get_annotation(annotation_id).calculations + +frames = dfc.calculations_to_dataframes(calcs) # {"orbit-rms": DataFrame} +df = frames["orbit-rms"] + +print(df.index.tz) # UTC +print(df.attrs["column_metadata"]["x_rms"]["provenance"]["process"]) +``` + +The index is a UTC `DatetimeIndex` built directly from int64 nanoseconds, and per-column metadata +lands in `df.attrs["column_metadata"]` — the same convention the +[query recipe](query.md) uses. + +The reverse direction turns analysis output back into a payload: + +```python +# cookbook:partial +calcs = client.annotation.annotations.get_annotation(annotation_id).calculations +df = dfc.calculations_to_dataframes(calcs)["orbit-rms"] + +calcs_to_save = dfc.calculations_from_dataframes({"orbit-rms": df}) +``` + +Two things to know about it. + +**Missing values are rejected, loudly.** A dense typed column cannot represent a gap, so a `NaN` +anywhere raises rather than inventing a value or dropping a row: + +``` +column 'x_rms' contains missing values (first at row position(s) [17]), which a dense typed +column cannot represent. Either give the sparse column its own frame over only the timestamps +where it has values, or build it with data_frame.data_column(), whose DataValues can be left unset. +``` + +**The index always becomes an explicit `TimestampList`**, never an inferred `SamplingClock`. A +clock is only correct if the spacing is exactly uniform at nanosecond precision, and inferring that +from an index that merely *looks* regular would quietly change your timestamps. When the data +really is a regular clock, say so with `sampling_clock()` and build the frame directly. + +The dtype mapping is `float64`→Double, `float32`→Float, `int64`→Int64, `int32`→Int32, `bool`→Bool, +and object/string→String. Anything else — complex, datetime columns, categoricals — raises rather +than guessing. + +## Updating without losing the calculations + +`save_annotation()` is a [full replace](conventions.md#save-semantics-full-replace), and that +**includes the calculations**. Re-saving an annotation without them does not leave them alone: it +clears the reference and deletes the stored object. + +```python +# cookbook:partial +# WRONG -- silently deletes the calculations this annotation owned +client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( + name="Orbit drift during CXI_3443 (revised)", + owner_id="cmcchesney", + dataset_ids=[dataset_id], + annotation_id=annotation_id, +)) +``` + +Read the current state and carry the calculations forward: + +```python +# cookbook:partial +read = client.annotation.annotations.get_annotation(annotation_id) +if read.result_status.is_error: + raise RuntimeError(read.result_status.message) + +existing = read.annotation +assert existing is not None # guaranteed once is_error is False + +client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( + name="Orbit drift during CXI_3443 (revised)", + owner_id=existing.ownerId, + dataset_ids=list(existing.dataSetIds), + annotation_ids=list(existing.annotationIds), + description=existing.description, + tags=list(existing.tags), + attributes={a.name: a.value for a in existing.attributes}, + calculations=read.calculations, # carried forward + annotation_id=annotation_id, + modified_by="cmcchesney", +)) +``` + +A replace that *does* carry new calculations returns a **new** `calculations_id`; the previous +object is deleted, not orphaned. An annotation saved with no calculations at all reports +`calculations_id == ""` — empty string, not `None`. `None` means the call failed. + +## Exporting + +`export_data()` writes a file **on the server**, in HDF5, CSV, or XLSX. Sources merge: a saved +dataset by id, ad-hoc data blocks, calculations, or any combination — at least one is required. + +```python +# cookbook:partial +export = client.annotation.export.export_data(ExportDataRequestParams( + ExportFormat.HDF5, + dataset_id=dataset_id, + calculations_spec=calculations_spec(calculations_id), +)) +if export.result_status.is_error: + raise RuntimeError(export.result_status.message) + +print(export.file_path) # a path on the SERVER's filesystem +print(export.file_url) # '' unless the deployment publishes exports over HTTP +``` + +`ExportFormat` accepts either the member or its string value, so `ExportFormat.CSV` and +`"csv"` both work; anything else raises a `ValueError` naming the valid formats, which is what +makes the server-rejected `EXPORT_FORMAT_UNSPECIFIED` unreachable. + +For a one-off export of a region you do not want to keep, skip the dataset entirely: + +```python +# cookbook:partial +client.annotation.export.export_data(ExportDataRequestParams( + "csv", + data_blocks=[data_block(t0, t1, ["BPMS:GUNB:314:X"])], +)) +``` + +`calculations_spec()` narrows what gets included. Omit `frame_columns` for everything; supply it to +pick specific columns of specific frames, excluding any frame you do not mention: + +```python +# cookbook:partial +spec = calculations_spec(calculations_id, {"orbit-rms": ["x_rms"]}) +``` + +**There is no download.** `file_path` is a path on the server's filesystem, and no RPC retrieves +the file — so this library offers no download convenience rather than pretending otherwise. How you +collect the file is a deployment question: a shared mount, `file_url` if the deployment publishes +over HTTP, or an out-of-band copy. + +Two rejections to expect: the tabular formats (CSV, XLSX) can only represent **scalar** columns, so +exporting array, image, or struct columns to them fails — use HDF5. And an unknown dataset or +calculations id is rejected rather than producing an empty file. + +## Tearing down + +Order matters. A dataset cannot be deleted while any annotation references it: + +```python +# cookbook:partial +refused = client.annotation.datasets.delete_dataset(dataset_id) +print(refused.result_status.is_error) # True, naming a referencing annotation +``` + +There is deliberately no `cascade=True`. Deleting a dataset and every annotation about it is two +destructive operations behind one flag; the honest shape is the two-step the server designed: + +```python +# cookbook:partial +for annotation in client.annotation.annotations.iter_annotations([AQ.datasets([dataset_id])]): + deleted = client.annotation.annotations.delete_annotation(annotation.id) + if deleted.result_status.is_error: + raise RuntimeError(deleted.result_status.message) + +client.annotation.datasets.delete_dataset(dataset_id) +``` + +Deleting an annotation **cascades to its calculations**. It does not clean up incoming references: +other annotations' `annotationIds` entries, and `derivedFrom` provenance links naming the deleted +calculations, are left dangling. + +## Also worth knowing + +- **Deleting something that does not exist is an error**, not a silent success. Both + `delete_dataset()` and `delete_annotation()` report `no ... record found for id: ...` on a second + delete, so a teardown that runs twice reports failures the second time. +- **`updatedTime` is unset on create.** It appears once a record has been replaced at least once. +- **Ids are ObjectIds.** A malformed id is rejected as malformed rather than reported as "not + found"; the distinction is useful when debugging. +- **Page tokens are opaque keyset tokens**, and a malformed or wrong-query one is **rejected**. + This differs from the older metadata queries, which silently restart from the beginning + ([dp-service #193](https://github.com/osprey-dcs/dp-service/issues/193)). Do not construct or + reuse tokens across queries. +- **`patchDataSet` and `patchAnnotation` are not wrapped.** They are reserved placeholders that + return "not implemented"; use the full-replace `save_*` methods. +- **Array, image, and struct column builders do not exist yet.** `data_frame.py` covers the typed + scalar columns and the `DataColumn` escape hatch; the rest are + [issue #17](https://github.com/osprey-dcs/dp-python-lib/issues/17)'s to design with ingestion data + in hand. Meanwhile, build those column protos directly and pass them to `data_frame()`, which + accepts pre-built columns alongside the ones its builders return. +- **Serialized columns are skipped on read.** `data_frame_columns()` ignores + `serializedDataColumns`, whose payloads this library does not decode; read the field directly if + you need it. + +### How far these examples have been verified + +The dataset and annotation lifecycle **has** been exercised against a live 1.16.0 Annotation +Service by `tests/integration/test_datasets_annotations_integration.py`: save/get round trips, +every criterion including the key-only attribute search, lowercase tag normalization, paging, +rejected page tokens, the `get_datasets()` batch fetch, calculations inline on `get_annotation()` +versus id-only on `query_annotations()`, the full-replace clearing behavior, the delete cascade, and +the referenced-dataset refusal. + +That test ingests its own samples first, because of the archive-existence rule described in +[Model](#model) — there is no ingestion client yet +([issue #17](https://github.com/osprey-dcs/dp-python-lib/issues/17)), so it uses the generated stub +directly. + +The `x_rms_values` in these examples stand in for real analysis output. The **numbers** are +illustrative; the calls around them are the verified part. diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index 4602525..21a9cdb 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -44,6 +44,15 @@ collision surfaced and was resolved: the `data_frame()` *function* is deliberately not re-exported from `dp_python_lib.client`, because binding that name would shadow the `data_frame` *module* and break the `from dp_python_lib.client import data_frame as dfb` form this plan's own reference snippet uses. +- **Phase 4 implemented 2026-09-09.** `doc/cookbook/datasets-and-annotations.md` (continuing the shared worked + example: a dataset over the CXI shift's first hour, an orbit-drift annotation, a 1 Hz RMS calculation with + provenance, and the export), the cookbook README table and worked-example note, the `README.md` move of the three + Annotation Service bullets from TODO to Current state, and the `CLAUDE.md` usage section. The integration test + grew the Phase 4 legs — builder-made calculations read back through `data_frame_conversions` with a sub-second + axis (so the round trip exercises nanosecond arithmetic rather than whole seconds), provenance survival, and four + export cases — for 25 tests, 12 subtests, all passing against dp-service `fddf692`. The checker preamble gained + the new names plus the recipe's shared worked-example handles. The `CLAUDE.md` snippet was extracted and **run + against the live server**, not just type-checked: every documented call succeeds end to end. ## Overview diff --git a/tests/integration/test_datasets_annotations_integration.py b/tests/integration/test_datasets_annotations_integration.py index 888edfe..9483107 100644 --- a/tests/integration/test_datasets_annotations_integration.py +++ b/tests/integration/test_datasets_annotations_integration.py @@ -10,6 +10,8 @@ # Add src directory to path for imports sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src")) +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc from dp_python_lib.client.annotations_client import ( AnnotationQuery, SaveAnnotationRequestParams, @@ -20,6 +22,7 @@ SaveDataSetRequestParams, data_block, ) +from dp_python_lib.client.export_client import ExportDataRequestParams, ExportFormat, calculations_spec from dp_python_lib.client.mldp_client import MldpClient from dp_python_lib.grpc import common_pb2, ingestion_pb2, ingestion_pb2_grpc @@ -68,6 +71,7 @@ def setUpClass(cls): cls.client = MldpClient() cls.datasets = cls.client.annotation.datasets cls.annotations = cls.client.annotation.annotations + cls.export = cls.client.annotation.export cls._verify_modernized_api_available() @@ -529,6 +533,140 @@ def test_delete_annotation_twice_is_a_business_error(self): self.assertTrue(second.result_status.is_error) self.assertIsNone(second.annotation_id) + # ------------------------------------------------------------------ + # Builder-made calculations, read back through the conversions (Phase 2) + # ------------------------------------------------------------------ + + CALC_START = datetime(2024, 2, 2, 18, 0, 0, tzinfo=timezone.utc) + CALC_PERIOD_NANOS = 250_000_000 + CALC_VALUES = (12.7, 12.8, 12.9) + + def _builder_calculations(self): + """ + A Calculations payload built entirely through data_frame.py, with column-level provenance. + + Deliberately uses a sub-second period, so the axis round trip exercises the nanosecond arithmetic rather + than whole seconds a float could also represent. + """ + axis = dfb.sampling_clock(self.CALC_START, self.CALC_PERIOD_NANOS, len(self.CALC_VALUES)) + column = dfb.double_column( + "x_rms", + list(self.CALC_VALUES), + metadata=dfb.column_metadata( + tags=["derived"], + attributes={"unit": "mm"}, + provenance=dfb.provenance( + source="itest-rig", + process="1 Hz RMS", + derived_from=[dfb.pv_source(self.pv_name, (self.begin_time, self.end_time))], + ), + ), + ) + return calculations({"orbit-rms": dfb.data_frame(axis, [column])}) + + def test_builder_calculations_round_trip_is_nanosecond_exact(self): + """ + Save calculations built by data_frame.py, read them back, and check the axis reproduces exactly. + + This is the leg that would fail silently if any part of the path routed timestamps through float seconds: + a float64 cannot represent present-day epoch nanoseconds, so the expanded positions would drift. + """ + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + fetched = self.annotations.get_calculations(saved.calculations_id) + self.assertFalse(fetched.result_status.is_error, fetched.result_status.message) + + frames = fetched.calculations.calculationDataFrames + self.assertEqual([f.name for f in frames], ["orbit-rms"]) + frame = frames[0].frame + + start_nanos = int(self.CALC_START.timestamp()) * 1_000_000_000 + expected = [start_nanos + i * self.CALC_PERIOD_NANOS for i in range(len(self.CALC_VALUES))] + self.assertEqual(dfc.data_frame_timestamps(frame), expected) + + self.assertEqual(dfc.data_frame_columns(frame), {"x_rms": list(self.CALC_VALUES)}) + + def test_builder_provenance_survives_the_round_trip(self): + """Column-level provenance is the reason calculations are worth storing; it must come back intact.""" + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + frame = self.annotations.get_calculations(saved.calculations_id).calculations.calculationDataFrames[0].frame + metadata = dfc.column_metadata_dict(frame.doubleColumns[0]) + + self.assertEqual(metadata["tags"], ["derived"]) + self.assertEqual(metadata["attributes"], {"unit": "mm"}) + self.assertEqual(metadata["provenance"]["source"], "itest-rig") + self.assertEqual(metadata["provenance"]["process"], "1 Hz RMS") + self.assertEqual(metadata["provenance"]["derived_from"][0]["pv_name"], self.pv_name) + + # ------------------------------------------------------------------ + # Export (Phase 4) + # ------------------------------------------------------------------ + + def test_calculations_only_csv_export(self): + """ + A calculations-only export needs no ingested data of its own, which makes it the cheapest end-to-end + check that exportData() works. + """ + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + result = self.export.export_data( + ExportDataRequestParams( + ExportFormat.CSV, + calculations_spec=calculations_spec(saved.calculations_id), + ) + ) + + self.assertFalse(result.result_status.is_error, result.result_status.message) + self.assertTrue(result.file_path, "a successful export must report a server-side file path") + # file_url is empty unless the deployment publishes over HTTP; empty is normal, not a failure. + self.assertIsNotNone(result.file_url) + + def test_export_accepts_a_bare_format_string(self): + """ExportFormat coercion is part of the params contract, so exercise it against the real server too.""" + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + result = self.export.export_data( + ExportDataRequestParams("csv", calculations_spec=calculations_spec(saved.calculations_id)) + ) + + self.assertFalse(result.result_status.is_error, result.result_status.message) + + def test_export_with_a_column_filter(self): + """calculations_spec() narrows the export to named columns of named frames.""" + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + result = self.export.export_data( + ExportDataRequestParams( + ExportFormat.CSV, + calculations_spec=calculations_spec(saved.calculations_id, {"orbit-rms": ["x_rms"]}), + ) + ) + + self.assertFalse(result.result_status.is_error, result.result_status.message) + + def test_export_of_an_unknown_calculations_id_is_rejected(self): + result = self.export.export_data( + ExportDataRequestParams(ExportFormat.CSV, calculations_spec=calculations_spec("000000000000000000000000")) + ) + + self.assertTrue(result.result_status.is_error) + self.assertIsNone(result.file_path) + + def test_dataset_export_to_hdf5(self): + """The dataset path exports the archived samples the run ingested.""" + dataset_id = self._save_dataset() + + result = self.export.export_data(ExportDataRequestParams(ExportFormat.HDF5, dataset_id=dataset_id)) + + self.assertFalse(result.result_status.is_error, result.result_status.message) + self.assertTrue(result.file_path) + if __name__ == "__main__": unittest.main() From f101ff6ef94faec67049ad92f242f6aa4b353d21 Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Thu, 10 Sep 2026 11:42:55 -0600 Subject: [PATCH 03/11] docs: complete the PR #44 review fixes for Phase 2-4 (issue #6) Two corrections that only apply once data_frame.py exists on this branch, and so could not be made on the PR 1 branch: - calculations(): PR 1 reworded this docstring to say the data_frame builders "arrive in the follow-up PR", which is true there and stale here. Point at the builders directly now that they exist. - datasets-and-annotations cookbook: record the DataBlock interval semantics established by reading dp-service. The range is half-open [begin, end) like everything else in the library, so back-to-back blocks cover the boundary sample exactly once -- but HDF5 export is bucket-granular and writes overlapping buckets whole and untrimmed, so it can include out-of-range samples and can write a straddling bucket twice. That exception was documented nowhere. Also note the new bare-string rejection in data_block(). 670 unit tests pass; ruff lint and format clean; cookbook snippet checker passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- doc/cookbook/datasets-and-annotations.md | 16 ++++++++++++++-- src/dp_python_lib/client/annotations_client.py | 8 ++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/doc/cookbook/datasets-and-annotations.md b/doc/cookbook/datasets-and-annotations.md index eebc52d..a912b5b 100644 --- a/doc/cookbook/datasets-and-annotations.md +++ b/doc/cookbook/datasets-and-annotations.md @@ -117,8 +117,20 @@ print(saved_id) # server-assigned id, e.g. '6aa1bb271a768e97d ``` `data_block()` requires `begin < end` and a non-empty PV list. That check exists here because the -server does not make it: it validates only that each bound is non-zero, so a reversed block would -otherwise be stored happily. +server does not make it: it validates only that each bound is non-zero and never compares the two, +so a reversed block would otherwise be stored happily. It also rejects a bare string for the PV +list, which would otherwise be iterated into one PV name per character. + +A block's range is [half-open](conventions.md#half-open-ranges), `[begin, end)`, like every other +range in the library, so back-to-back blocks — one ending at `T`, the next starting at `T` — cover +the sample at `T` exactly once. `saveDataSet` itself never compares the bounds; the interval only +acquires meaning at export, where it reaches the same per-sample trimming that `query_samples()` +does. + +**HDF5 export is the exception.** It is bucket-granular: every bucket that *overlaps* the block is +written whole and untrimmed, so an HDF5 file can contain samples outside the range you asked for, +and back-to-back blocks sharing a straddling bucket write it twice. CSV and XLSX trim to the exact +range. Nothing client-side can change this — it is a property of the export format. Note the tags come back **normalized** — lowercased, deduplicated, and sorted — so a tag saved as `CXI-3443` reads back as `cxi-3443`, and queries must match the lowercase form. diff --git a/src/dp_python_lib/client/annotations_client.py b/src/dp_python_lib/client/annotations_client.py index 9922609..eb3fd03 100644 --- a/src/dp_python_lib/client/annotations_client.py +++ b/src/dp_python_lib/client/annotations_client.py @@ -17,10 +17,10 @@ def calculations(frames: dict[str, common_pb2.DataFrame]) -> annotation_pb2.Calc Taking a dict rather than a list makes frame-name uniqueness true by construction; the server rejects duplicate frame names, and a list would let a caller build one. - Each frame is a common.DataFrame: a time axis plus the columns sampled on it. Assemble that message directly - for now -- the data_frame builders (dp_python_lib.client.data_frame), which validate a frame's internal shape - (column count against the time axis, unique column names, non-empty names and values) so an error names the - offending column, arrive in the follow-up PR for issue #6. + Each frame is a common.DataFrame: a time axis plus the columns sampled on it. Build one with the data_frame + builders (from dp_python_lib.client import data_frame as dfb; dfb.data_frame(...)), which validate the frame's + internal shape -- column count against the time axis, unique column names, non-empty names and values -- so an + error names the offending column. Note the proto's naming trap: the repeated field is 'calculationDataFrames' (singular "calculation") while the message it holds is 'CalculationsDataFrame' (plural). From a6e18656ae6ab1014a4fb197643808ef2421bdeb Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Thu, 10 Sep 2026 12:37:44 -0600 Subject: [PATCH 04/11] fix: address PR #45 review findings (issue #6, PR 2) Five findings from the review of PR #45, two of them defects. Write and read paths disagreed on ragged array columns. data_frame() computed an array column's sample count with floor division, so a value count that was not a whole multiple of prod(dims) rounded into a passing count -- building a frame that data_frame_conversions then refused to read back (5 values with a per-sample size of 2 became "2 samples"). The write path now applies the same whole-multiple rule the read path already had, so what one accepts the other can always read. A regression test asserts both reject the identical input, with prod(dims) extracted into _array_sample_size() now that two checks need it. column_metadata_dict() dropped provenance time ranges. pv_source() and calculations_source() both take an optional (begin, end) narrowing which part of the source was used -- the substantive half of provenance for a derived column -- and it never survived read-back. It is now reported as 'time_range' in epoch nanoseconds, matching the module's integer-nanosecond convention. While there, each source now carries only the origin arm actually set: a pvName source has no 'calculations_column' key and vice versa, rather than the unset arm appearing as an empty string. Absent stays absent, as it does for confidence and reason in the sample status conversions. The integration test asserts the range survives a real server round trip. Three smaller ones: - data_frame_from_pandas() on duplicate column labels failed deep inside as pandas' "truth value of a Series is ambiguous" (df[name] returns a DataFrame, not a Series). It now rejects them up front, naming the offender -- concat and merge produce duplicates easily by accident. - A pandas round trip preserves every value, dtype, and timestamp but not column order, because a DataFrame stores each column kind in its own repeated field. True but undocumented, and easy to read as a bug; now in both docstrings and the cookbook, and pinned by a test rather than left incidental. - data_column() accepted np.float64 (a float subclass) but rejected np.int64 and np.bool_, which subclass nothing it matched -- an arbitrary split for a caller coming from pandas. It now maps by numbers.Integral / numbers.Real, with NumPy's bool matched by type module and name so this module keeps its no-NumPy dependency (the type is bool_ under NumPy 1.x, bool under 2.x). _timestamp_to_nanos is defined locally rather than imported from sample_status_conversions: that copy is private, and reaching across for it would make this module's dependencies read as though it needed the sample status API. 680 unit tests pass (619 pass, 61 skip cleanly in a venv without the [analysis] extra); 25 integration tests and 12 subtests pass against the live server; ruff lint and format clean; all 103 cookbook snippets check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- CLAUDE.md | 4 +- doc/cookbook/datasets-and-annotations.md | 37 +++++- plan/tickets/6/plan.md | 16 +++ src/dp_python_lib/client/data_frame.py | 101 +++++++++++---- .../client/data_frame_conversions.py | 87 ++++++++++--- .../test_datasets_annotations_integration.py | 16 ++- tests/unit/test_data_frame.py | 46 +++++++ tests/unit/test_data_frame_conversions.py | 119 +++++++++++++++++- 8 files changed, 380 insertions(+), 46 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 153955b..695904b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,8 +156,8 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/dataset_client.py` - DataSet client (`save_dataset()`, `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, plus the `get_datasets(ids)` batch fetch that avoids the annotation-listing N+1) with the `DataSetQuery` (`DS`) criterion helpers and the `data_block()` builder. `data_block()` is the only place `begin < end` is checked — the server does not - `src/dp_python_lib/client/annotations_client.py` - Annotations client (`save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, `delete_annotation()`, `get_calculations()`) with the `AnnotationQuery` (`AQ`) criterion helpers and the `calculations()` builder, which takes a `dict[str, DataFrame]` so frame-name uniqueness is true by construction. Note `AnnotationsClient` (feature client) vs `AnnotationClient` (facade) - `src/dp_python_lib/client/export_client.py` - Export client (`export_data()`) with the `ExportFormat` str enum and the `calculations_spec()` builder -- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Array/image/struct/serialized builders are #17's; hand-built ones pass through -- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one list per sample), `data_frame_columns()`, `column_metadata_dict()`. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap), and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock` +- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through +- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one list per sample), `data_frame_columns()`, `column_metadata_dict()`. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap), and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`, and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type - `src/dp_python_lib/client/query_client.py` - v2 time-series query client (sample-oriented) exposed as `client.query`. Low-level wrappers `query_samples()` (unary, one resumable page) and `iter_query_samples()` (transparent paging), plus `iter_query_samples_stream()` (server-streaming, fire-and-consume, lazy). Queries are described by a kind-neutral `QueryParams` built from the `PvQuery` (`PV`) and `ConfigQuery` (`CFG`) criterion helpers; shares a `_build_query_spec()` seam so a future bucket request builder reuses it. Results wrap the raw `ColumnTable` (`.column_table`, `.next_page_token`); `.to_dataframe()`/`.to_numpy()` delegate to `query_conversions` (Phase 2, optional `[analysis]` extra) - `src/dp_python_lib/client/query_conversions.py` - Pythonic conversions for query results (optional `[analysis]` extra: pandas/numpy/openpyxl, imported lazily). `data_value_to_python()` (oneof extractor: scalars→native, timestamp→epoch-nanos, array→list, structure→dict, image→`Image` wrapper, fail-loud on unhandled arm), `column_table_to_dataframe()` (UTC datetime index + one column per DataColumn; dense-alignment and duplicate-column-name fail-loud; ColumnMetadata in `df.attrs`), `column_table_to_numpy()` (dict of 1-D arrays; complex arms stay 1-D object arrays rather than collapsing to 2-D), `dataframe_to_excel()` (thin `to_excel()` wrapper: row-limit guard, tz-drop, complex-cell stringification), and `query_samples_to_dataframe()`/`stream_query_samples_to_dataframes()` whole-query conveniences (unary concats by column name; streaming yields per-page frames lazily) - `src/dp_python_lib/client/service_api_client_base.py` - Base class for the service clients: owns the channel and the one-per-client gRPC stub, and provides `_dispatch()`, the shared three-tier sender that all 18 unary `_send_*` methods delegate to diff --git a/doc/cookbook/datasets-and-annotations.md b/doc/cookbook/datasets-and-annotations.md index a912b5b..bb31185 100644 --- a/doc/cookbook/datasets-and-annotations.md +++ b/doc/cookbook/datasets-and-annotations.md @@ -317,6 +317,25 @@ chained = dfb.provenance( Both accept an optional `(begin, end)` pair narrowing which part of the source was used. +Reading it back, `column_metadata_dict()` gives you the plain-Python view — no extras needed. Each +entry carries **only the origin arm actually set**, plus the time range when there is one, reported +as epoch nanoseconds like every other instant in this library: + +```python +# cookbook:partial +provenance = dfb.provenance(derived_from=[dfb.pv_source("BPMS:GUNB:314:X", (t0, t1))]) +column = dfb.double_column("x_rms", [0.31, 0.29, 0.33], metadata=dfb.column_metadata(provenance=provenance)) + +summary = dfc.column_metadata_dict(column) +print(summary["provenance"]["derived_from"]) +# [{'pv_name': 'BPMS:GUNB:314:X', 'time_range': (1770055200000000000, 1770058800000000000)}] +``` + +A PV source has no `calculations_column` key and a calculations source has no `pv_name` key, rather +than the unset one showing up as an empty string — absence means "not this arm", the same +[absent-vs-empty](conventions.md) discipline the rest of the library follows. A source with no time +range simply has no `time_range` key, never a fabricated `(0, 0)`. + These are **soft references**. Deleting the annotation that owns the referenced calculations leaves the link dangling; nothing resolves or cleans it up, and readers are expected to tolerate that. @@ -418,7 +437,23 @@ really is a regular clock, say so with `sampling_clock()` and build the frame di The dtype mapping is `float64`→Double, `float32`→Float, `int64`→Int64, `int32`→Int32, `bool`→Bool, and object/string→String. Anything else — complex, datetime columns, categoricals — raises rather -than guessing. +than guessing. Duplicate column names are rejected up front: a frame stores one column per name, and +a `concat` or a `merge` with overlapping names produces duplicates easily. + +**A round trip preserves values, but not column order.** A `DataFrame` keeps each column kind in its +own repeated field, so the wire format has no single ordering across kinds. Converting out and back +returns every value, dtype, and timestamp intact, with the columns grouped by type: + +```python +# cookbook:partial +calcs = client.annotation.annotations.get_annotation(annotation_id).calculations +df = dfc.calculations_to_dataframes(calcs)["orbit-rms"] + +restored = dfc.data_frame_to_pandas(dfc.data_frame_from_pandas(df)) +restored = restored[list(df.columns)] # reindex by name to get your order back +``` + +Select by name rather than by position, and reindex when the order matters. ## Updating without losing the calculations diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index 21a9cdb..7df1775 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -53,6 +53,22 @@ export cases — for 25 tests, 12 subtests, all passing against dp-service `fddf692`. The checker preamble gained the new names plus the recipe's shared worked-example handles. The `CLAUDE.md` snippet was extracted and **run against the live server**, not just type-checked: every documented call succeeds end to end. +- **PR #45 review fixes, 2026-09-10.** Five findings, two of them defects: + - `data_frame()` accepted an array column whose value count was not a whole multiple of `prod(dims)`, floor- + dividing it into a passing sample count — building a frame that `data_frame_conversions` then refused to read + back. The write path now applies the same whole-multiple rule the read path already did, so the two agree by + construction; a regression test asserts both reject the identical input. + - `column_metadata_dict()` dropped a provenance source's `timeRange` entirely, losing "which part of the source + was used" — the substantive half of provenance for a derived column. It is now reported as epoch nanoseconds, + and each source carries only the origin arm actually set rather than the unset arm as an empty string. The + integration test asserts the range survives a real server round trip. + - `data_frame_from_pandas()` on duplicate column labels failed deep inside as pandas' "truth value of a Series is + ambiguous"; it now rejects them up front by name. + - A pandas round trip reorders columns (grouped by proto field), which was true but undocumented and easy to read + as a bug. Documented in both docstrings and the cookbook, and pinned by a test. + - `data_column()` accepted `np.float64` (a `float` subclass) but rejected `np.int64` and `np.bool_` (subclasses of + nothing it matched). It now maps by `numbers.Integral` / `numbers.Real`, with NumPy's bool matched by type + module and name so this module keeps its no-NumPy dependency. ## Overview diff --git a/src/dp_python_lib/client/data_frame.py b/src/dp_python_lib/client/data_frame.py index 1418027..1f47ee6 100644 --- a/src/dp_python_lib/client/data_frame.py +++ b/src/dp_python_lib/client/data_frame.py @@ -22,6 +22,7 @@ the ones its builders return. """ +from numbers import Integral, Real from typing import Any from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp @@ -450,6 +451,25 @@ def enum_column( return column +def _is_numpy_bool(value: Any) -> bool: + """ + True if a value is a NumPy boolean scalar. + + NumPy's boolean scalar is a subclass of neither bool nor numbers.Integral, so nothing else in data_column()'s + type mapping catches it -- while np.float64 IS a subclass of float and np.int64 IS numbers.Integral. Left + unhandled it would be the one NumPy scalar the mapping rejected. + + Identified by its type's module and name rather than by importing NumPy, which keeps this module free of the + optional [analysis] dependency. Both spellings are matched: the type is named `bool_` under NumPy 1.x and + `bool` under 2.x, and the module check is what keeps the latter from matching an unrelated class. + + :param value: The value to test. + :return: True if the value is a numpy.bool_ / numpy.bool scalar. + """ + value_type = type(value) + return value_type.__module__ == "numpy" and value_type.__name__ in ("bool_", "bool") + + def data_column( name: str, values: list[Any], metadata: common_pb2.ColumnMetadata | None = None ) -> common_pb2.DataColumn: @@ -466,6 +486,11 @@ def data_column( None -> an unset oneof. Anything else raises: silently coercing an unexpected type would store something the caller did not mean. Pass a pre-built DataValue to use an arm this mapping does not cover. + The integer and float arms are matched by numbers.Integral / numbers.Real, and np.bool_ by name, so NumPy + scalars work too. np.int64 is not a subclass of int and np.bool_ is not a subclass of bool (while np.float64 + IS a subclass of float), so mapping by exact Python type would accept some NumPy scalars and reject others -- + an arbitrary distinction for a caller coming from pandas or NumPy. + :param name: The column's name, unique within its frame. :param values: One value per sample, where None means "no value for this sample". :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). @@ -485,13 +510,15 @@ def data_column( continue if isinstance(value, common_pb2.DataValue): data_value.CopyFrom(value) - elif isinstance(value, bool): - # Checked before int: bool is a subclass of int, so the int branch would swallow it. - data_value.booleanValue = value - elif isinstance(value, int): - data_value.longValue = value - elif isinstance(value, float): - data_value.doubleValue = value + elif isinstance(value, bool) or _is_numpy_bool(value): + # Checked before the integer branch: bool is a subclass of int, so that branch would swallow it. + data_value.booleanValue = bool(value) + elif isinstance(value, Integral): + # Integral rather than int, so a NumPy integer scalar maps like a Python one. + data_value.longValue = int(value) + elif isinstance(value, Real): + # Real rather than float, for symmetry with the integer branch above. + data_value.doubleValue = float(value) elif isinstance(value, str): data_value.stringValue = value elif isinstance(value, bytes): @@ -512,28 +539,50 @@ def data_column( # ---------------------------------------------------------------------- +def _array_sample_size(column: Any) -> int | None: + """ + Returns an array column's per-sample size, prod(dims), or None when it cannot be derived. + + Absent dims are underivable rather than a product of 1 -- an empty product would silently treat every element + as its own sample. A zero or negative dim is equally unusable, so both report as None for the caller to + reject with a message naming the column. + + :param column: An array column (DoubleArrayColumn, Int32ArrayColumn, ...). + :return: The number of values each sample occupies, or None if dims are missing or non-positive. + """ + dims = list(column.dimensions.dims) + if not dims: + return None + product = 1 + for dim in dims: + product *= dim + if product <= 0: + return None + return product + + def _column_sample_count(column: Any) -> int | None: """ Returns the number of samples a column carries, or None when the column has no countable per-sample values. + For an array column the count is len(values) / prod(dims); a value count that is not a whole multiple of + prod(dims) has no sample count at all and reports as None, so _check_column() rejects it rather than letting + floor division round it into a passing count. The read path (data_frame_conversions._reshape_array_values) + applies the same rule, and a frame this accepted but that could not be read back would be the worst outcome. + :param column: A typed column, a legacy DataColumn, or a SerializedDataColumn. - :return: The sample count, or None for a SerializedDataColumn (whose payload is opaque). + :return: The sample count, or None for a SerializedDataColumn (whose payload is opaque) and for an array + column whose dims are missing, non-positive, or do not evenly divide its values. """ if isinstance(column, common_pb2.SerializedDataColumn): return None if isinstance(column, common_pb2.DataColumn): return len(column.dataValues) if isinstance(column, _ARRAY_COLUMN_TYPES): - # Array values are flat: samples x prod(dims). Absent dims are underivable rather than a product of 1 -- - # an empty product would silently treat each element as its own sample. _check_column() reports that - # instead of dividing by zero or accepting a wrong count. - dims = list(column.dimensions.dims) - if not dims: + product = _array_sample_size(column) + if product is None: return None - product = 1 - for dim in dims: - product *= dim - if product <= 0: + if len(column.values) % product != 0: return None return len(column.values) // product return len(column.values) @@ -570,11 +619,19 @@ def _check_column(column: Any, index: int, expected_count: int, seen_names: set[ # Serialized payloads are opaque; the server checks their names only. return - if isinstance(column, _ARRAY_COLUMN_TYPES) and _column_sample_count(column) is None: - raise ValueError( - f"data_frame() cannot determine the sample count of array column '{name}': its dimensions are " - f"missing or zero. Set ArrayDimensions.dims so that values is samples x prod(dims)." - ) + if isinstance(column, _ARRAY_COLUMN_TYPES): + product = _array_sample_size(column) + if product is None: + raise ValueError( + f"data_frame() cannot determine the sample count of array column '{name}': its dimensions are " + f"missing or zero. Set ArrayDimensions.dims so that values is samples x prod(dims)." + ) + if len(column.values) % product != 0: + raise ValueError( + f"data_frame() array column '{name}' has {len(column.values)} values, which is not a whole " + f"multiple of its per-sample size {product} (from dims {list(column.dimensions.dims)}); " + f"values must be samples x prod(dims)" + ) count = _column_sample_count(column) if count == 0: diff --git a/src/dp_python_lib/client/data_frame_conversions.py b/src/dp_python_lib/client/data_frame_conversions.py index 7894d8b..84fbe52 100644 --- a/src/dp_python_lib/client/data_frame_conversions.py +++ b/src/dp_python_lib/client/data_frame_conversions.py @@ -50,6 +50,21 @@ # ImageColumn stores its per-sample payloads in `images` rather than `values`. _IMAGE_COLUMN_FIELD = "imageColumns" +_NANOS_PER_SECOND = 1_000_000_000 + + +def _timestamp_to_nanos(timestamp: common_pb2.Timestamp) -> int: + """ + Converts a common.Timestamp into a single integer of epoch nanoseconds. + + Defined here rather than imported from sample_status_conversions: that module's copy is private, and reaching + across for it would make this module's dependencies read as though it needed the sample status API. + + :param timestamp: The timestamp to convert. + :return: Epoch nanoseconds as a Python int (arbitrary precision, so no overflow). + """ + return timestamp.epochSeconds * _NANOS_PER_SECOND + timestamp.nanoseconds + def _require_pandas(): """Imports and returns pandas, or raises an actionable error if the optional [analysis] extra is missing.""" @@ -181,6 +196,41 @@ def data_frame_columns(frame: common_pb2.DataFrame) -> dict[str, list]: return columns +def _column_source_dict(entry: common_pb2.ColumnProvenance.ColumnSource) -> dict[str, Any]: + """ + Summarizes one ColumnProvenance.ColumnSource -- where a column's values came from -- as a plain dict. + + Only the origin arm actually set is present: a pvName source has no 'calculations_column' key and vice versa, + rather than the unset arm standing in as an empty string. Absence means "not this arm", the same absent-vs- + empty discipline the sample status conversions apply to confidence and reason. + + The optional timeRange -- which part of the source was used -- is reported as epoch nanoseconds, matching this + module's integer-nanosecond convention. It is the substantive half of provenance for a derived column: the + source PV without its window says much less than the pair does. + + :param entry: The ColumnSource to summarize. + :return: A dict carrying the set origin arm and, when present, 'time_range' as (begin_nanos, end_nanos). + """ + result: dict[str, Any] = {} + + origin = entry.WhichOneof("origin") + if origin == "pvName": + result["pv_name"] = entry.pvName + elif origin == "calculationsColumn": + result["calculations_column"] = { + "calculations_id": entry.calculationsColumn.calculationsId, + "frame_name": entry.calculationsColumn.frameName, + "column_name": entry.calculationsColumn.columnName, + } + + if entry.HasField("timeRange"): + result["time_range"] = ( + _timestamp_to_nanos(entry.timeRange.beginTime), + _timestamp_to_nanos(entry.timeRange.endTime), + ) + return result + + def column_metadata_dict(column: Any) -> dict[str, Any]: """ Summarizes a column's ColumnMetadata as a plain dict, for carrying alongside converted values. @@ -198,21 +248,7 @@ def column_metadata_dict(column: Any) -> dict[str, Any]: provenance = { "source": source.source, "process": source.process, - "derived_from": [ - { - "pv_name": entry.pvName, - "calculations_column": ( - { - "calculations_id": entry.calculationsColumn.calculationsId, - "frame_name": entry.calculationsColumn.frameName, - "column_name": entry.calculationsColumn.columnName, - } - if entry.WhichOneof("origin") == "calculationsColumn" - else None - ), - } - for entry in source.derivedFrom - ], + "derived_from": [_column_source_dict(entry) for entry in source.derivedFrom], } return { @@ -252,6 +288,12 @@ def data_frame_to_pandas(frame: common_pb2.DataFrame, exclude_column_metadata: b through float seconds would move present-day timestamps by hundreds of nanoseconds. Per-column ColumnMetadata lands in df.attrs["column_metadata"] (a dict keyed by column name), matching query_conversions' convention. + Note columns come back GROUPED BY TYPE, not in their original order: a DataFrame stores each column kind in its + own repeated field, so the wire format does not preserve a single ordering across kinds. A round trip through + data_frame_from_pandas() therefore preserves every value, dtype, and timestamp, but can reorder the columns + (float64, then int, then bool, then string, matching the proto's field order). Select by name, or reindex to + the order you want. + :param frame: The DataFrame to convert. :param exclude_column_metadata: When True, skip populating df.attrs["column_metadata"]. :return: A pandas.DataFrame indexed by UTC timestamp. @@ -390,6 +432,9 @@ def data_frame_from_pandas(df: Any) -> common_pb2.DataFrame: The index becomes an explicit TimestampList; see _timestamps_from_index() for why a SamplingClock is never inferred. Missing values are rejected fail-loud, since a dense typed column cannot express a gap. + Converting back with data_frame_to_pandas() preserves every value, dtype, and timestamp, but can return the + columns grouped by type rather than in their original order -- see that function's docstring. + :param df: The pandas DataFrame to convert. Must have a tz-aware, strictly increasing DatetimeIndex. :return: A common.DataFrame. :raises ImportError: if the [analysis] extra is not installed. @@ -403,6 +448,18 @@ def data_frame_from_pandas(df: Any) -> common_pb2.DataFrame: if len(df.columns) == 0: raise ValueError("DataFrame must have at least one column") + # Duplicate labels make df[name] a DataFrame rather than a Series, which would otherwise surface far downstream + # as pandas' "truth value of a Series is ambiguous". Column names must be unique within a frame anyway, so + # reject it here with a message naming the offender -- concat() and merge() produce this easily by accident. + names = [str(name) for name in df.columns] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError( + f"DataFrame has more than one column named {', '.join(repr(name) for name in duplicates)}; column " + f"names must be unique within a frame. Rename or drop the duplicates first (e.g. after a concat or " + f"a merge with overlapping names)." + ) + data_timestamps = _timestamps_from_index(df.index) columns = [_column_from_series(str(name), df[name]) for name in df.columns] return builders.data_frame(data_timestamps, columns) diff --git a/tests/integration/test_datasets_annotations_integration.py b/tests/integration/test_datasets_annotations_integration.py index 9483107..22b1e8c 100644 --- a/tests/integration/test_datasets_annotations_integration.py +++ b/tests/integration/test_datasets_annotations_integration.py @@ -27,6 +27,11 @@ from dp_python_lib.grpc import common_pb2, ingestion_pb2, ingestion_pb2_grpc +def _epoch_nanos(when: datetime) -> int: + """Epoch nanoseconds for a tz-aware datetime, for comparing against the conversions' integer-nanos output.""" + return int(when.timestamp()) * 1_000_000_000 + when.microsecond * 1_000 + + class TestDataSetsAnnotationsIntegration(unittest.TestCase): """ Integration tests for DataSetClient and AnnotationsClient that require a running MLDP ecosystem. @@ -599,7 +604,16 @@ def test_builder_provenance_survives_the_round_trip(self): self.assertEqual(metadata["attributes"], {"unit": "mm"}) self.assertEqual(metadata["provenance"]["source"], "itest-rig") self.assertEqual(metadata["provenance"]["process"], "1 Hz RMS") - self.assertEqual(metadata["provenance"]["derived_from"][0]["pv_name"], self.pv_name) + source = metadata["provenance"]["derived_from"][0] + self.assertEqual(source["pv_name"], self.pv_name) + # The source's time range is the substantive half of provenance: which part of the PV's history was used. + # It must survive the server round trip, exact to the nanosecond, like every other instant in this library. + self.assertEqual( + source["time_range"], + (_epoch_nanos(self.begin_time), _epoch_nanos(self.end_time)), + ) + # A pvName source carries no calculations_column key at all, rather than an empty placeholder. + self.assertNotIn("calculations_column", source) # ------------------------------------------------------------------ # Export (Phase 4) diff --git a/tests/unit/test_data_frame.py b/tests/unit/test_data_frame.py index 7e8a76c..b43664c 100644 --- a/tests/unit/test_data_frame.py +++ b/tests/unit/test_data_frame.py @@ -9,6 +9,13 @@ from dp_python_lib.client import data_frame as dfb from dp_python_lib.grpc import common_pb2 +try: + import numpy as np + + _HAVE_NUMPY = True +except ImportError: + _HAVE_NUMPY = False + T0 = datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc) T1 = datetime(2026, 7, 14, 19, 0, 0, tzinfo=timezone.utc) @@ -162,6 +169,24 @@ def test_accepts_prebuilt_data_value(self): self.assertEqual(column.dataValues[0].WhichOneof("value"), "uintValue") self.assertEqual(column.dataValues[0].uintValue, 42) + @unittest.skipUnless(_HAVE_NUMPY, "requires the [analysis] extra (numpy)") + def test_maps_numpy_scalars_like_their_python_counterparts(self): + # np.float64 subclasses float, but np.int64 does not subclass int and np.bool_ does not subclass bool. + # Mapping by exact Python type would accept the first and reject the other two -- an arbitrary split for a + # caller coming from pandas or NumPy. + column = dfb.data_column("np", [np.float64(1.5), np.int64(2), np.int32(3), np.bool_(True), np.float32(0.5)]) + arms = [v.WhichOneof("value") for v in column.dataValues] + self.assertEqual(arms, ["doubleValue", "longValue", "longValue", "booleanValue", "doubleValue"]) + self.assertEqual(column.dataValues[1].longValue, 2) + self.assertIs(column.dataValues[3].booleanValue, True) + + @unittest.skipUnless(_HAVE_NUMPY, "requires the [analysis] extra (numpy)") + def test_numpy_bool_is_checked_before_the_integer_branch(self): + # np.bool_ is not Integral, but this pins the ordering the way test_bool_is_checked_before_int does. + column = dfb.data_column("flags", [np.bool_(True), np.bool_(False)]) + self.assertEqual([v.WhichOneof("value") for v in column.dataValues], ["booleanValue", "booleanValue"]) + self.assertEqual([v.booleanValue for v in column.dataValues], [True, False]) + def test_rejects_unmappable_type(self): with self.assertRaises(ValueError) as ctx: dfb.data_column("c", [1 + 2j]) @@ -328,6 +353,27 @@ def test_array_column_count_uses_dims_product(self): dfb.data_frame(_axis(3), [column]) self.assertIn("2 values", str(ctx.exception)) + def test_array_column_with_ragged_values_is_rejected(self): + # 5 values with a per-sample size of 2 is not a whole number of samples. Floor division would round it to + # a passing count of 2 and build a frame that data_frame_conversions then refuses to read back. + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([2]) + column.values[:] = [1.0, 2.0, 3.0, 4.0, 5.0] + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(2), [column]) + message = str(ctx.exception) + self.assertIn("whole multiple", message) + self.assertIn("waveform", message) + + def test_array_column_accepts_multidimensional_dims(self): + column = common_pb2.DoubleArrayColumn() + column.name = "image" + column.dimensions.dims.extend([2, 3]) + column.values[:] = [float(i) for i in range(12)] # 2 samples x (2 x 3) + frame = dfb.data_frame(_axis(2), [column]) + self.assertEqual([c.name for c in frame.doubleArrayColumns], ["image"]) + def test_array_column_without_dims_is_rejected(self): column = common_pb2.DoubleArrayColumn() column.name = "waveform" diff --git a/tests/unit/test_data_frame_conversions.py b/tests/unit/test_data_frame_conversions.py index f3b86e7..892bcca 100644 --- a/tests/unit/test_data_frame_conversions.py +++ b/tests/unit/test_data_frame_conversions.py @@ -23,6 +23,8 @@ T0 = datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc) T0_NANOS = int(T0.timestamp()) * 1_000_000_000 +T1 = datetime(2026, 7, 14, 19, 0, 0, tzinfo=timezone.utc) +T1_NANOS = int(T1.timestamp()) * 1_000_000_000 def _axis(count=3, period_nanos=1_000_000_000): @@ -186,7 +188,11 @@ def test_full_metadata(self): provenance=dfb.provenance( source="rig", process="RMS", - derived_from=[dfb.pv_source("A:1"), dfb.calculations_source("c", "f", "x")], + derived_from=[ + dfb.pv_source("A:1"), + dfb.calculations_source("c", "f", "x"), + dfb.pv_source("A:2", (T0, T1)), + ], ), ), ) @@ -196,13 +202,35 @@ def test_full_metadata(self): self.assertEqual(result["attributes"], {"unit": "mm"}) self.assertEqual(result["provenance"]["source"], "rig") self.assertEqual(result["provenance"]["process"], "RMS") - self.assertEqual(result["provenance"]["derived_from"][0]["pv_name"], "A:1") - self.assertIsNone(result["provenance"]["derived_from"][0]["calculations_column"]) + # Only the origin arm actually set is present -- the unset arm is absent, not an empty placeholder. + self.assertEqual(result["provenance"]["derived_from"][0], {"pv_name": "A:1"}) self.assertEqual( - result["provenance"]["derived_from"][1]["calculations_column"], - {"calculations_id": "c", "frame_name": "f", "column_name": "x"}, + result["provenance"]["derived_from"][1], + {"calculations_column": {"calculations_id": "c", "frame_name": "f", "column_name": "x"}}, ) + def test_provenance_preserves_source_time_range(self): + """A source's timeRange -- which part of it was used -- survives read-back, as epoch nanoseconds.""" + column = dfb.double_column( + "d", + [1.0], + metadata=dfb.column_metadata( + provenance=dfb.provenance( + derived_from=[ + dfb.pv_source("A:1", (T0, T1)), + dfb.calculations_source("c", "f", "x", (T0, T1)), + dfb.pv_source("A:2"), + ] + ) + ), + ) + + sources = dfc.column_metadata_dict(column)["provenance"]["derived_from"] + self.assertEqual(sources[0]["time_range"], (T0_NANOS, T1_NANOS)) + self.assertEqual(sources[1]["time_range"], (T0_NANOS, T1_NANOS)) + # An absent range is absent, not a fabricated (0, 0) pair. + self.assertNotIn("time_range", sources[2]) + def test_absent_metadata(self): result = dfc.column_metadata_dict(dfb.double_column("d", [1.0])) self.assertEqual(result, {"tags": [], "attributes": {}, "provenance": None}) @@ -425,5 +453,86 @@ def test_metadata_survives_to_pandas(self): self.assertEqual(back["f1"].attrs["column_metadata"]["d"]["tags"], ["derived"]) +class TestWriteAndReadPathsAgree(unittest.TestCase): + """Anything data_frame() accepts must be readable by this module -- the two paths share one shape contract.""" + + def test_ragged_array_rejected_by_both_paths(self): + # Regression: the write path used floor division, so it accepted a value count that was not a whole + # multiple of prod(dims) and produced a frame data_frame_columns() then refused to read. + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([2]) + column.values[:] = [1.0, 2.0, 3.0, 4.0, 5.0] + + with self.assertRaises(ValueError) as write_error: + dfb.data_frame(_axis(2), [column]) + + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(_axis(2)) + frame.doubleArrayColumns.append(column) + with self.assertRaises(ValueError) as read_error: + dfc.data_frame_columns(frame) + + for message in (str(write_error.exception), str(read_error.exception)): + self.assertIn("whole multiple", message) + self.assertIn("waveform", message) + + def test_well_formed_array_survives_the_round_trip(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([2]) + column.values[:] = [1.0, 2.0, 3.0, 4.0] + frame = dfb.data_frame(_axis(2), [column]) + self.assertEqual(dfc.data_frame_columns(frame), {"waveform": [[1.0, 2.0], [3.0, 4.0]]}) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestPandasDuplicateColumnNames(unittest.TestCase): + def test_duplicate_column_names_are_rejected_by_name(self): + # df[name] on a duplicated label returns a DataFrame, not a Series, which would otherwise surface deep + # inside the converter as pandas' "truth value of a Series is ambiguous". + import pandas as pd + + index = pd.DatetimeIndex(pd.to_datetime([T0_NANOS, T0_NANOS + 1], unit="ns", utc=True)) + df = pd.DataFrame([[1.0, 2.0], [3.0, 4.0]], columns=["a", "a"], index=index) + + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + message = str(ctx.exception) + self.assertIn("'a'", message) + self.assertIn("unique", message) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestPandasRoundTripColumnOrder(unittest.TestCase): + """A DataFrame stores each column kind in its own repeated field, so ordering across kinds is not preserved.""" + + def _round_trip(self): + import pandas as pd + + index = pd.DatetimeIndex( + pd.to_datetime([T0_NANOS, T0_NANOS + 1_000_000_000, T0_NANOS + 2_000_000_000], unit="ns", utc=True) + ) + df = pd.DataFrame( + {"a": [1.0, 2.0, 3.0], "b": ["x", "y", "z"], "c": [True, False, True]}, + index=index, + ) + return df, dfc.data_frame_to_pandas(dfc.data_frame_from_pandas(df)) + + def test_columns_come_back_grouped_by_type(self): + # Pinned rather than merely documented: the order is a consequence of the proto's field order, and a + # caller who reindexes by name needs it to be stable. + _, restored = self._round_trip() + self.assertEqual(list(restored.columns), ["a", "c", "b"]) + + def test_values_index_and_dtypes_survive_the_reordering(self): + original, restored = self._round_trip() + realigned = restored[list(original.columns)] + self.assertTrue(original.index.equals(realigned.index)) + self.assertEqual(list(original.dtypes), list(realigned.dtypes)) + for name in original.columns: + self.assertEqual(list(original[name]), list(realigned[name])) + + if __name__ == "__main__": unittest.main() From 9e07226d681675d0d645a4a8ea0472c070a5898d Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Thu, 10 Sep 2026 12:46:18 -0600 Subject: [PATCH 05/11] refactor: one shared to_epoch_nanos() instead of three private copies Follow-up to the PR #45 review fixes. The provenance fix needed a Timestamp -> epoch-nanoseconds conversion, and I added a private copy to data_frame_conversions, justifying it as avoiding a cross-module private import. That was the wrong call twice over: the justification cited a CLAUDE.md note that is scoped to query_support.py's shared query helpers rather than being a general rule, and the note's actual lesson is "shared helpers belong in a shared module", not "duplicate them". The conversion already existed privately in query_conversions (as _timestamp_to_epoch_nanos) and sample_status_conversions (as _timestamp_to_nanos), so my copy made three spellings of one arithmetic identity. It is now one public to_epoch_nanos() beside to_timestamp() in machine_config_client -- the same conversion in the other direction, in the module that data_frame.py, dataset_client.py, query_client.py, and sample_status_client.py already import to_timestamp() from. NANOS_PER_SECOND is public there too, since sample_status_conversions needs it for the inverse divmod. Both existing modules keep their internal names as one-line aliases, so no call site changed. The module name does not advertise a time-conversion role, but four modules already depend on it for exactly that; moving to_timestamp() somewhere better named is a separate change that would touch every one of them. 683 unit tests pass (622 pass, 61 skip without the [analysis] extra); 42 integration tests and 12 subtests pass against the live server; ruff clean; 103 cookbook snippets check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- CLAUDE.md | 2 +- plan/tickets/6/plan.md | 7 ++++++ src/dp_python_lib/client/__init__.py | 2 ++ .../client/data_frame_conversions.py | 20 +++-------------- .../client/machine_config_client.py | 20 +++++++++++++++++ src/dp_python_lib/client/query_conversions.py | 7 +++--- .../client/sample_status_conversions.py | 16 +++++--------- tests/unit/test_machine_config_client.py | 22 +++++++++++++++++++ 8 files changed, 64 insertions(+), 32 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 695904b..236a34d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,7 +150,7 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/ingestion_client.py` - Ingestion service client with methods like `register_provider()` - `src/dp_python_lib/client/annotation_client.py` - Annotation service facade; groups feature-scoped clients sharing the one `DpAnnotationService` channel (`.pv_metadata`, `.machine_config`, `.sample_status`, `.datasets`, `.annotations`, `.export` — every implemented `DpAnnotationService` feature area) - `src/dp_python_lib/client/pv_metadata_client.py` - PV metadata client (`save_pv_metadata()`, `get_pv_metadata()`, `query_pv_metadata()`, `iter_pv_metadata()`, `delete_pv_metadata()`) plus the `PvMetadataQuery` (`Q`) criterion helpers -- `src/dp_python_lib/client/machine_config_client.py` - Machine configuration client covering both configurations (`save_configuration()`, `get_configuration()`, `query_configurations()`, `iter_configurations()`, `delete_configuration()`) and their temporal activations (`save_configuration_activation()`, `get_configuration_activation()`, `query_configuration_activations()`, `iter_configuration_activations()`, `delete_configuration_activation()`, `get_active_configurations()`). Includes the `ConfigurationQuery` (`C`) and `ConfigurationActivationQuery` (`CA`) criterion helpers and the `to_timestamp()` helper (tz-aware datetime / epoch seconds / `common.Timestamp`). Get/delete activation take a composite key (`client_activation_id` XOR `configuration_name`+`start_time`). Activation `end_time` is optional — omit it for an open-ended activation ("still in effect"); the field is then genuinely absent on the wire +- `src/dp_python_lib/client/machine_config_client.py` - Machine configuration client covering both configurations (`save_configuration()`, `get_configuration()`, `query_configurations()`, `iter_configurations()`, `delete_configuration()`) and their temporal activations (`save_configuration_activation()`, `get_configuration_activation()`, `query_configuration_activations()`, `iter_configuration_activations()`, `delete_configuration_activation()`, `get_active_configurations()`). Includes the `ConfigurationQuery` (`C`) and `ConfigurationActivationQuery` (`CA`) criterion helpers and the two shared time converters: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`) and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds). Both live here because four other modules already import `to_timestamp()` from this one; `to_epoch_nanos()` was previously spelled out privately in `query_conversions`, `sample_status_conversions`, and `data_frame_conversions`, which meant three copies of one arithmetic identity. Get/delete activation take a composite key (`client_activation_id` XOR `configuration_name`+`start_time`). Activation `end_time` is optional — omit it for an open-ended activation ("still in effect"); the field is then genuinely absent on the wire - `src/dp_python_lib/client/sample_status_client.py` - Sample status client (`save_sample_statuses()`, `query_sample_statuses()`, `iter_sample_statuses()`, `iter_sample_statuses_stream()`, `delete_sample_statuses()`) plus the `SampleStatusColumn` / `SampleStatusFrame` construction classes. The `sampling_clock()` / `timestamp_list()` axis builders now live in `data_frame.py` (issue #6 Phase 2, once calculations frames became a second caller) and are re-exported here, so existing imports are unaffected. A status's identity key is `(pvName, timestamp, domain, layer)`; `delete_sample_statuses()` requires either `pv_names` or an explicit `all_pvs=True` opt-in for the destructive wildcard - `src/dp_python_lib/client/sample_status_conversions.py` - Per-sample expansion of query results (no optional extras required): `expand_data_timestamps()` (SamplingClock positions computed in **integer nanoseconds**, never float seconds — the exact-match contract depends on it), `bucket_to_rows()` / `buckets_to_rows()` / `iter_rows()` yielding `SampleStatusRow` objects with absent confidence/reason surfaced as `None` rather than fabricated `0.0`/`""` - `src/dp_python_lib/client/dataset_client.py` - DataSet client (`save_dataset()`, `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, plus the `get_datasets(ids)` batch fetch that avoids the annotation-listing N+1) with the `DataSetQuery` (`DS`) criterion helpers and the `data_block()` builder. `data_block()` is the only place `begin < end` is checked — the server does not diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index 7df1775..3efa2c7 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -70,6 +70,13 @@ nothing it matched). It now maps by `numbers.Integral` / `numbers.Real`, with NumPy's bool matched by type module and name so this module keeps its no-NumPy dependency. + Fixing the provenance finding needed a Timestamp -> epoch-nanoseconds conversion, which turned out to exist + privately in three modules already (`query_conversions`, `sample_status_conversions`, and, briefly, a fourth copy + added here). Rather than add to that, it is now one public `to_epoch_nanos()` beside `to_timestamp()` in + `machine_config_client` -- the same conversion in the other direction, in the module four others already import + `to_timestamp()` from. The two private aliases stay as one-line bindings so each module keeps the internal name + it has always used. + ## Overview Wrap the modernized DataSet / Annotation / Calculations / Export area of `DpAnnotationService` in the house diff --git a/src/dp_python_lib/client/__init__.py b/src/dp_python_lib/client/__init__.py index 71eb7c0..a444217 100644 --- a/src/dp_python_lib/client/__init__.py +++ b/src/dp_python_lib/client/__init__.py @@ -67,6 +67,7 @@ SaveConfigurationActivationRequestParams, SaveConfigurationApiResult, SaveConfigurationRequestParams, + to_epoch_nanos, to_timestamp, ) from dp_python_lib.client.mldp_client import MldpClient @@ -180,5 +181,6 @@ "string_column", "timestamp_count", "timestamp_list", + "to_epoch_nanos", "to_timestamp", ] diff --git a/src/dp_python_lib/client/data_frame_conversions.py b/src/dp_python_lib/client/data_frame_conversions.py index 84fbe52..031bcb3 100644 --- a/src/dp_python_lib/client/data_frame_conversions.py +++ b/src/dp_python_lib/client/data_frame_conversions.py @@ -21,6 +21,7 @@ from typing import Any +from dp_python_lib.client.machine_config_client import to_epoch_nanos from dp_python_lib.client.query_conversions import data_value_to_python from dp_python_lib.client.sample_status_conversions import expand_data_timestamps from dp_python_lib.grpc import annotation_pb2, common_pb2 @@ -50,21 +51,6 @@ # ImageColumn stores its per-sample payloads in `images` rather than `values`. _IMAGE_COLUMN_FIELD = "imageColumns" -_NANOS_PER_SECOND = 1_000_000_000 - - -def _timestamp_to_nanos(timestamp: common_pb2.Timestamp) -> int: - """ - Converts a common.Timestamp into a single integer of epoch nanoseconds. - - Defined here rather than imported from sample_status_conversions: that module's copy is private, and reaching - across for it would make this module's dependencies read as though it needed the sample status API. - - :param timestamp: The timestamp to convert. - :return: Epoch nanoseconds as a Python int (arbitrary precision, so no overflow). - """ - return timestamp.epochSeconds * _NANOS_PER_SECOND + timestamp.nanoseconds - def _require_pandas(): """Imports and returns pandas, or raises an actionable error if the optional [analysis] extra is missing.""" @@ -225,8 +211,8 @@ def _column_source_dict(entry: common_pb2.ColumnProvenance.ColumnSource) -> dict if entry.HasField("timeRange"): result["time_range"] = ( - _timestamp_to_nanos(entry.timeRange.beginTime), - _timestamp_to_nanos(entry.timeRange.endTime), + to_epoch_nanos(entry.timeRange.beginTime), + to_epoch_nanos(entry.timeRange.endTime), ) return result diff --git a/src/dp_python_lib/client/machine_config_client.py b/src/dp_python_lib/client/machine_config_client.py index aa82b8b..8117190 100644 --- a/src/dp_python_lib/client/machine_config_client.py +++ b/src/dp_python_lib/client/machine_config_client.py @@ -14,6 +14,26 @@ TimestampInput = datetime | int | float | common_pb2.Timestamp +NANOS_PER_SECOND = 1_000_000_000 + + +def to_epoch_nanos(timestamp: common_pb2.Timestamp) -> int: + """ + Converts a common.Timestamp into a single integer of epoch nanoseconds -- the inverse of to_timestamp(). + + Lives here beside to_timestamp() because it is the same conversion in the other direction, and the modules + that need it (query, sample status, and DataFrame conversions) already import to_timestamp() from here. It + was previously spelled out privately in each of them, which meant three copies of one arithmetic identity. + + Integer arithmetic throughout, on a Python int, so there is no overflow and no precision loss: present-day + epoch nanoseconds need about 61 bits, and a float64 carries 53. + + :param timestamp: The timestamp to convert. + :return: Epoch nanoseconds as a Python int. + """ + return timestamp.epochSeconds * NANOS_PER_SECOND + timestamp.nanoseconds + + def to_timestamp(value: TimestampInput) -> common_pb2.Timestamp: """ Converts a user-supplied time value into a common.Timestamp{epochSeconds, nanoseconds}. diff --git a/src/dp_python_lib/client/query_conversions.py b/src/dp_python_lib/client/query_conversions.py index dc2ab05..55fbe91 100644 --- a/src/dp_python_lib/client/query_conversions.py +++ b/src/dp_python_lib/client/query_conversions.py @@ -22,6 +22,7 @@ from collections.abc import Iterator from typing import Any +from dp_python_lib.client.machine_config_client import to_epoch_nanos from dp_python_lib.grpc import common_pb2, query_pb2 # Excel's hard row ceiling (1,048,576 rows including a header row). @@ -98,9 +99,9 @@ def _require_numpy(): ) -def _timestamp_to_epoch_nanos(ts: common_pb2.Timestamp) -> int: - """Converts a common.Timestamp to an integer count of nanoseconds since the Unix epoch.""" - return ts.epochSeconds * 1_000_000_000 + ts.nanoseconds +# The Timestamp -> epoch-nanoseconds conversion is shared (see machine_config_client.to_epoch_nanos); this private +# alias is the name this module has always used internally. +_timestamp_to_epoch_nanos = to_epoch_nanos def data_value_to_python(value: common_pb2.DataValue) -> Any: diff --git a/src/dp_python_lib/client/sample_status_conversions.py b/src/dp_python_lib/client/sample_status_conversions.py index 36959d3..a381363 100644 --- a/src/dp_python_lib/client/sample_status_conversions.py +++ b/src/dp_python_lib/client/sample_status_conversions.py @@ -21,18 +21,12 @@ from collections.abc import Iterator +from dp_python_lib.client.machine_config_client import NANOS_PER_SECOND, to_epoch_nanos from dp_python_lib.grpc import common_pb2 -_NANOS_PER_SECOND = 1_000_000_000 - - -def _timestamp_to_nanos(timestamp: common_pb2.Timestamp) -> int: - """ - Converts a common.Timestamp into a single integer of epoch nanoseconds. - :param timestamp: The timestamp to convert. - :return: Epoch nanoseconds as a Python int (arbitrary precision, so no overflow). - """ - return timestamp.epochSeconds * _NANOS_PER_SECOND + timestamp.nanoseconds +# The Timestamp -> epoch-nanoseconds conversion is shared (see machine_config_client.to_epoch_nanos); this private +# alias is the name this module has always used internally. +_timestamp_to_nanos = to_epoch_nanos def _nanos_to_timestamp(epoch_nanos: int) -> common_pb2.Timestamp: @@ -42,7 +36,7 @@ def _nanos_to_timestamp(epoch_nanos: int) -> common_pb2.Timestamp: :return: The equivalent common.Timestamp. """ timestamp = common_pb2.Timestamp() - timestamp.epochSeconds, timestamp.nanoseconds = divmod(epoch_nanos, _NANOS_PER_SECOND) + timestamp.epochSeconds, timestamp.nanoseconds = divmod(epoch_nanos, NANOS_PER_SECOND) return timestamp diff --git a/tests/unit/test_machine_config_client.py b/tests/unit/test_machine_config_client.py index 0804c02..97324d4 100644 --- a/tests/unit/test_machine_config_client.py +++ b/tests/unit/test_machine_config_client.py @@ -17,6 +17,7 @@ QueryConfigurationsApiResult, SaveConfigurationApiResult, SaveConfigurationRequestParams, + to_epoch_nanos, to_timestamp, ) from dp_python_lib.grpc import annotation_pb2, common_pb2 @@ -32,6 +33,27 @@ def _response_with_field(field_name): return response +class TestToEpochNanos(unittest.TestCase): + """to_epoch_nanos() is to_timestamp()'s inverse, shared by the three conversions modules.""" + + def test_combines_seconds_and_nanoseconds(self): + ts = common_pb2.Timestamp() + ts.epochSeconds = 1_700_000_000 + ts.nanoseconds = 123_456_789 + self.assertEqual(to_epoch_nanos(ts), 1_700_000_000_123_456_789) + + def test_round_trips_with_to_timestamp_exactly(self): + # The exactness is the point: present-day epoch nanoseconds need ~61 bits and a float64 carries 53, so a + # conversion routed through float seconds would move the instant. + original = 1_770_055_200_123_456_789 + ts = common_pb2.Timestamp() + ts.epochSeconds, ts.nanoseconds = divmod(original, 1_000_000_000) + self.assertEqual(to_epoch_nanos(ts), original) + + def test_zero_timestamp_is_zero(self): + self.assertEqual(to_epoch_nanos(common_pb2.Timestamp()), 0) + + class TestToTimestamp(unittest.TestCase): """Unit tests for the to_timestamp() conversion helper.""" From 4c6a8e1d0cf1c2f78b4256c0e314dc6efda4a7a6 Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Thu, 10 Sep 2026 12:58:53 -0600 Subject: [PATCH 06/11] refactor: move the shared time converters to their own leaf module to_timestamp() was defined in machine_config_client because that was the first module to need it, and six others grew imports of it from there -- dataset_client, query_client, sample_status_client, data_frame, query_conversions, sample_status_conversions -- so datasets, queries, and DataFrames all read as though they depended on the machine configuration API. The previous commit added to_epoch_nanos() beside it, which made a misattributed home the shared home for both directions. Both now live in client/time_conversions.py, along with the TimestampInput alias and NANOS_PER_SECOND. It is a leaf: it imports only stdlib and the generated protos, so any client module can use it with no risk of a cycle, which is what makes it a safe home for conversions everything needs. Every internal caller is repointed at the new module; machine_config_client is now just another caller. No back-compat shim: the feature is unreleased, and a re-export would preserve exactly the misleading imports this removes. The public package surface is unchanged except that TimestampInput is now exported too, which it should have been -- it is the declared type of many public parameters. The converter unit tests move to tests/unit/test_time_conversions.py, next to the module they cover. Done inside this ticket rather than left as a follow-up, at the user's request: the feature is unreleased, so shipping it as-is would put the wart in the release and make the fix a breaking change afterwards. 725 tests pass (683 unit, 42 integration + 12 subtests against the live server; 622 pass and 61 skip without the [analysis] extra); ruff clean; 103 cookbook snippets check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- CLAUDE.md | 4 +- plan/tickets/6/plan.md | 11 ++- src/dp_python_lib/client/__init__.py | 4 +- src/dp_python_lib/client/data_frame.py | 2 +- .../client/data_frame_conversions.py | 2 +- src/dp_python_lib/client/dataset_client.py | 2 +- .../client/machine_config_client.py | 80 +--------------- src/dp_python_lib/client/query_client.py | 2 +- src/dp_python_lib/client/query_conversions.py | 2 +- .../client/sample_status_client.py | 2 +- .../client/sample_status_conversions.py | 2 +- src/dp_python_lib/client/time_conversions.py | 95 +++++++++++++++++++ tests/unit/test_machine_config_client.py | 79 --------------- tests/unit/test_time_conversions.py | 90 ++++++++++++++++++ 14 files changed, 209 insertions(+), 168 deletions(-) create mode 100644 src/dp_python_lib/client/time_conversions.py create mode 100644 tests/unit/test_time_conversions.py diff --git a/CLAUDE.md b/CLAUDE.md index 236a34d..6a331c5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,12 +150,13 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/ingestion_client.py` - Ingestion service client with methods like `register_provider()` - `src/dp_python_lib/client/annotation_client.py` - Annotation service facade; groups feature-scoped clients sharing the one `DpAnnotationService` channel (`.pv_metadata`, `.machine_config`, `.sample_status`, `.datasets`, `.annotations`, `.export` — every implemented `DpAnnotationService` feature area) - `src/dp_python_lib/client/pv_metadata_client.py` - PV metadata client (`save_pv_metadata()`, `get_pv_metadata()`, `query_pv_metadata()`, `iter_pv_metadata()`, `delete_pv_metadata()`) plus the `PvMetadataQuery` (`Q`) criterion helpers -- `src/dp_python_lib/client/machine_config_client.py` - Machine configuration client covering both configurations (`save_configuration()`, `get_configuration()`, `query_configurations()`, `iter_configurations()`, `delete_configuration()`) and their temporal activations (`save_configuration_activation()`, `get_configuration_activation()`, `query_configuration_activations()`, `iter_configuration_activations()`, `delete_configuration_activation()`, `get_active_configurations()`). Includes the `ConfigurationQuery` (`C`) and `ConfigurationActivationQuery` (`CA`) criterion helpers and the two shared time converters: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`) and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds). Both live here because four other modules already import `to_timestamp()` from this one; `to_epoch_nanos()` was previously spelled out privately in `query_conversions`, `sample_status_conversions`, and `data_frame_conversions`, which meant three copies of one arithmetic identity. Get/delete activation take a composite key (`client_activation_id` XOR `configuration_name`+`start_time`). Activation `end_time` is optional — omit it for an open-ended activation ("still in effect"); the field is then genuinely absent on the wire +- `src/dp_python_lib/client/machine_config_client.py` - Machine configuration client covering both configurations (`save_configuration()`, `get_configuration()`, `query_configurations()`, `iter_configurations()`, `delete_configuration()`) and their temporal activations (`save_configuration_activation()`, `get_configuration_activation()`, `query_configuration_activations()`, `iter_configuration_activations()`, `delete_configuration_activation()`, `get_active_configurations()`). Includes the `ConfigurationQuery` (`C`) and `ConfigurationActivationQuery` (`CA`) criterion helpers. The shared time converters it used to own now live in `time_conversions.py`. Get/delete activation take a composite key (`client_activation_id` XOR `configuration_name`+`start_time`). Activation `end_time` is optional — omit it for an open-ended activation ("still in effect"); the field is then genuinely absent on the wire - `src/dp_python_lib/client/sample_status_client.py` - Sample status client (`save_sample_statuses()`, `query_sample_statuses()`, `iter_sample_statuses()`, `iter_sample_statuses_stream()`, `delete_sample_statuses()`) plus the `SampleStatusColumn` / `SampleStatusFrame` construction classes. The `sampling_clock()` / `timestamp_list()` axis builders now live in `data_frame.py` (issue #6 Phase 2, once calculations frames became a second caller) and are re-exported here, so existing imports are unaffected. A status's identity key is `(pvName, timestamp, domain, layer)`; `delete_sample_statuses()` requires either `pv_names` or an explicit `all_pvs=True` opt-in for the destructive wildcard - `src/dp_python_lib/client/sample_status_conversions.py` - Per-sample expansion of query results (no optional extras required): `expand_data_timestamps()` (SamplingClock positions computed in **integer nanoseconds**, never float seconds — the exact-match contract depends on it), `bucket_to_rows()` / `buckets_to_rows()` / `iter_rows()` yielding `SampleStatusRow` objects with absent confidence/reason surfaced as `None` rather than fabricated `0.0`/`""` - `src/dp_python_lib/client/dataset_client.py` - DataSet client (`save_dataset()`, `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, plus the `get_datasets(ids)` batch fetch that avoids the annotation-listing N+1) with the `DataSetQuery` (`DS`) criterion helpers and the `data_block()` builder. `data_block()` is the only place `begin < end` is checked — the server does not - `src/dp_python_lib/client/annotations_client.py` - Annotations client (`save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, `delete_annotation()`, `get_calculations()`) with the `AnnotationQuery` (`AQ`) criterion helpers and the `calculations()` builder, which takes a `dict[str, DataFrame]` so frame-name uniqueness is true by construction. Note `AnnotationsClient` (feature client) vs `AnnotationClient` (facade) - `src/dp_python_lib/client/export_client.py` - Export client (`export_data()`) with the `ExportFormat` str enum and the `calculations_spec()` builder +- `src/dp_python_lib/client/time_conversions.py` - The two shared time converters and the `TimestampInput` alias: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`; naive datetimes raise) and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds), plus `NANOS_PER_SECOND`. Both were defined in `machine_config_client` as the first module to need them, and six others grew imports from there — which read as though datasets, queries, and DataFrames depended on the machine configuration API; `to_epoch_nanos()` had also been written out privately three separate times. **A leaf module**: it imports only stdlib and the generated protos, so any client module can use it without an import cycle. New time conversions belong here - `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through - `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one list per sample), `data_frame_columns()`, `column_metadata_dict()`. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap), and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`, and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type - `src/dp_python_lib/client/query_client.py` - v2 time-series query client (sample-oriented) exposed as `client.query`. Low-level wrappers `query_samples()` (unary, one resumable page) and `iter_query_samples()` (transparent paging), plus `iter_query_samples_stream()` (server-streaming, fire-and-consume, lazy). Queries are described by a kind-neutral `QueryParams` built from the `PvQuery` (`PV`) and `ConfigQuery` (`CFG`) criterion helpers; shares a `_build_query_spec()` seam so a future bucket request builder reuses it. Results wrap the raw `ColumnTable` (`.column_table`, `.next_page_token`); `.to_dataframe()`/`.to_numpy()` delegate to `query_conversions` (Phase 2, optional `[analysis]` extra) @@ -174,6 +175,7 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `tests/unit/test_export_client.py` - Unit tests for ExportClient (`ExportFormat` mapping and unreachable `UNSPECIFIED`, `calculations_spec()`, the zero-source rejection, three-tier error handling) - `tests/unit/test_annotation_client.py` - Unit tests pinning the `AnnotationClient` facade wiring (every feature client present, one shared channel, one stub apiece) - `tests/integration/test_datasets_annotations_integration.py` - Live-server round trip for datasets/annotations/calculations; ingests its own samples first, because `saveDataSet` requires archived PVs +- `tests/unit/test_time_conversions.py` - Unit tests for the shared time converters (`to_timestamp()` input forms and the naive-datetime/bool/unsupported-type rejections; `to_epoch_nanos()` exactness and its round trip with `to_timestamp()`) - `tests/unit/test_data_frame.py` - Unit tests for the data_frame builders (axis relocation, each typed column, `data_column()` bool-before-int and unset-oneof handling, provenance helpers, and every `data_frame()` shape rule incl. array dims and serialized-column name-only checks) - `tests/unit/test_data_frame_conversions.py` - Unit tests for data_frame_conversions (nanosecond-exact expansion, per-column conversion incl. array reshaping, duplicate-name fail-loud, and — skipping cleanly without `[analysis]` — the pandas round trip, dtype mapping, and NaN fail-loud) - `tests/unit/test_query_client.py` - Unit tests for QueryClient (request building, three-tier error handling, unary paging, streaming, `PvQuery`/`ConfigQuery` helpers, `QueryParams` validation) diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index 3efa2c7..ce8c2c5 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -72,10 +72,13 @@ Fixing the provenance finding needed a Timestamp -> epoch-nanoseconds conversion, which turned out to exist privately in three modules already (`query_conversions`, `sample_status_conversions`, and, briefly, a fourth copy - added here). Rather than add to that, it is now one public `to_epoch_nanos()` beside `to_timestamp()` in - `machine_config_client` -- the same conversion in the other direction, in the module four others already import - `to_timestamp()` from. The two private aliases stay as one-line bindings so each module keeps the internal name - it has always used. + added here). That prompted a second look at where the time converters live at all. They were defined in + `machine_config_client`, the first module to need them, and six others had grown imports of `to_timestamp()` from + there -- so datasets, queries, and DataFrames all read as though they depended on the machine configuration API. + Both directions now live in a new leaf module, `client/time_conversions.py`, which imports only stdlib and the + generated protos: `to_timestamp()`, `to_epoch_nanos()`, `TimestampInput`, and `NANOS_PER_SECOND`. Every internal + caller was repointed at it, and `machine_config_client` is now just another caller. Done inside this ticket + rather than deferred, since the feature is unreleased and a follow-up would ship the wart in the release. ## Overview diff --git a/src/dp_python_lib/client/__init__.py b/src/dp_python_lib/client/__init__.py index a444217..1d20093 100644 --- a/src/dp_python_lib/client/__init__.py +++ b/src/dp_python_lib/client/__init__.py @@ -67,8 +67,6 @@ SaveConfigurationActivationRequestParams, SaveConfigurationApiResult, SaveConfigurationRequestParams, - to_epoch_nanos, - to_timestamp, ) from dp_python_lib.client.mldp_client import MldpClient from dp_python_lib.client.pv_metadata_client import ( @@ -101,6 +99,7 @@ timestamp_list, ) from dp_python_lib.client.sample_status_conversions import SampleStatusRow +from dp_python_lib.client.time_conversions import TimestampInput, to_epoch_nanos, to_timestamp __all__ = [ "AnnotationClient", @@ -163,6 +162,7 @@ "SavePvMetadataRequestParams", "SaveSampleStatusesApiResult", "SaveSampleStatusesRequestParams", + "TimestampInput", "bool_column", "calculations", "calculations_source", diff --git a/src/dp_python_lib/client/data_frame.py b/src/dp_python_lib/client/data_frame.py index 1f47ee6..efb4af1 100644 --- a/src/dp_python_lib/client/data_frame.py +++ b/src/dp_python_lib/client/data_frame.py @@ -25,7 +25,7 @@ from numbers import Integral, Real from typing import Any -from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp from dp_python_lib.grpc import common_pb2 # Each typed column message, paired with the DataFrame field it belongs in. data_frame() routes by exact type, so diff --git a/src/dp_python_lib/client/data_frame_conversions.py b/src/dp_python_lib/client/data_frame_conversions.py index 031bcb3..56f1703 100644 --- a/src/dp_python_lib/client/data_frame_conversions.py +++ b/src/dp_python_lib/client/data_frame_conversions.py @@ -21,9 +21,9 @@ from typing import Any -from dp_python_lib.client.machine_config_client import to_epoch_nanos from dp_python_lib.client.query_conversions import data_value_to_python from dp_python_lib.client.sample_status_conversions import expand_data_timestamps +from dp_python_lib.client.time_conversions import to_epoch_nanos from dp_python_lib.grpc import annotation_pb2, common_pb2 # The DataFrame fields holding typed scalar columns, in the proto's declaration order. Each carries `values` diff --git a/src/dp_python_lib/client/dataset_client.py b/src/dp_python_lib/client/dataset_client.py index 372bb23..c62f468 100644 --- a/src/dp_python_lib/client/dataset_client.py +++ b/src/dp_python_lib/client/dataset_client.py @@ -3,10 +3,10 @@ import grpc -from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp from dp_python_lib.client.query_support import check_at_most_one_text_criterion from dp_python_lib.client.result import ApiResultBase from dp_python_lib.client.service_api_client_base import ServiceApiClientBase +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp from dp_python_lib.grpc import annotation_pb2, annotation_pb2_grpc, common_pb2 ID_QUERY_CHUNK_SIZE = 100 diff --git a/src/dp_python_lib/client/machine_config_client.py b/src/dp_python_lib/client/machine_config_client.py index 8117190..eb8bee1 100644 --- a/src/dp_python_lib/client/machine_config_client.py +++ b/src/dp_python_lib/client/machine_config_client.py @@ -1,5 +1,4 @@ import logging -import math from collections.abc import Iterator from datetime import datetime, timezone @@ -7,81 +6,12 @@ from dp_python_lib.client.result import ApiResultBase from dp_python_lib.client.service_api_client_base import ServiceApiClientBase -from dp_python_lib.grpc import annotation_pb2, annotation_pb2_grpc, common_pb2 - -# Accepted input types for API parameters that map to a common.Timestamp: -# a timezone-aware datetime, epoch seconds (int or float), or an already-built Timestamp. -TimestampInput = datetime | int | float | common_pb2.Timestamp - - -NANOS_PER_SECOND = 1_000_000_000 - - -def to_epoch_nanos(timestamp: common_pb2.Timestamp) -> int: - """ - Converts a common.Timestamp into a single integer of epoch nanoseconds -- the inverse of to_timestamp(). - - Lives here beside to_timestamp() because it is the same conversion in the other direction, and the modules - that need it (query, sample status, and DataFrame conversions) already import to_timestamp() from here. It - was previously spelled out privately in each of them, which meant three copies of one arithmetic identity. - Integer arithmetic throughout, on a Python int, so there is no overflow and no precision loss: present-day - epoch nanoseconds need about 61 bits, and a float64 carries 53. - - :param timestamp: The timestamp to convert. - :return: Epoch nanoseconds as a Python int. - """ - return timestamp.epochSeconds * NANOS_PER_SECOND + timestamp.nanoseconds - - -def to_timestamp(value: TimestampInput) -> common_pb2.Timestamp: - """ - Converts a user-supplied time value into a common.Timestamp{epochSeconds, nanoseconds}. - - Accepts: - - a timezone-aware datetime (naive datetimes are rejected to avoid silent local-timezone bugs), - - epoch seconds as an int or float (float fractional part becomes nanoseconds), - - an already-built common.Timestamp (returned as-is). - - :param value: The time value to convert. - :return: An equivalent common.Timestamp. - :raises ValueError: if a datetime is naive (has no tzinfo), or if the resulting epoch seconds are negative - (pre-1970) -- common.Timestamp.epochSeconds is an unsigned (uint64) field and cannot represent them. - :raises TypeError: if value is not one of the supported types. - """ - if isinstance(value, common_pb2.Timestamp): - return value - - if isinstance(value, datetime): - if value.tzinfo is None or value.tzinfo.utcoffset(value) is None: - raise ValueError( - "to_timestamp() requires a timezone-aware datetime; naive datetimes are rejected " - "to avoid silent local-timezone bugs. Use datetime.now(timezone.utc) or attach tzinfo." - ) - epoch = value.timestamp() - return to_timestamp(epoch) - - if isinstance(value, bool): - # bool is a subclass of int; reject it explicitly as it is virtually always a mistake. - raise TypeError("to_timestamp() does not accept bool") - - if isinstance(value, (int, float)): - # Floor the seconds (not truncate toward zero) so the fractional remainder, and therefore - # nanoseconds, is always in [0, 1_000_000_000) even for negative epoch inputs. - epoch_seconds = math.floor(value) - nanoseconds = int(round((float(value) - epoch_seconds) * 1_000_000_000)) - # Guard against float rounding pushing nanoseconds up to a full second. - if nanoseconds >= 1_000_000_000: - epoch_seconds += 1 - nanoseconds -= 1_000_000_000 - timestamp = common_pb2.Timestamp() - timestamp.epochSeconds = epoch_seconds - timestamp.nanoseconds = nanoseconds - return timestamp - - raise TypeError( - f"to_timestamp() expects datetime, int/float epoch seconds, or common.Timestamp, got {type(value).__name__}" - ) +# The shared time converters moved to time_conversions.py in issue #6. They were defined here, as the first +# module to need them, and six others grew imports of them from here -- which read as though datasets, queries, +# and DataFrames depended on the machine configuration API. This module is now just another caller. +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp +from dp_python_lib.grpc import annotation_pb2, annotation_pb2_grpc, common_pb2 class ConfigurationQuery: diff --git a/src/dp_python_lib/client/query_client.py b/src/dp_python_lib/client/query_client.py index 2580d92..fda07ec 100644 --- a/src/dp_python_lib/client/query_client.py +++ b/src/dp_python_lib/client/query_client.py @@ -4,9 +4,9 @@ import grpc -from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp from dp_python_lib.client.result import ApiResultBase from dp_python_lib.client.service_api_client_base import ServiceApiClientBase +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp from dp_python_lib.grpc import common_pb2, query_pb2, query_pb2_grpc diff --git a/src/dp_python_lib/client/query_conversions.py b/src/dp_python_lib/client/query_conversions.py index 55fbe91..784026e 100644 --- a/src/dp_python_lib/client/query_conversions.py +++ b/src/dp_python_lib/client/query_conversions.py @@ -22,7 +22,7 @@ from collections.abc import Iterator from typing import Any -from dp_python_lib.client.machine_config_client import to_epoch_nanos +from dp_python_lib.client.time_conversions import to_epoch_nanos from dp_python_lib.grpc import common_pb2, query_pb2 # Excel's hard row ceiling (1,048,576 rows including a header row). diff --git a/src/dp_python_lib/client/sample_status_client.py b/src/dp_python_lib/client/sample_status_client.py index 436e640..59a2515 100644 --- a/src/dp_python_lib/client/sample_status_client.py +++ b/src/dp_python_lib/client/sample_status_client.py @@ -5,9 +5,9 @@ from dp_python_lib.client.data_frame import sampling_clock, timestamp_list from dp_python_lib.client.data_frame import timestamp_count as _timestamp_count -from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp from dp_python_lib.client.result import ApiResultBase from dp_python_lib.client.service_api_client_base import ServiceApiClientBase +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp from dp_python_lib.grpc import annotation_pb2, annotation_pb2_grpc, common_pb2 # sampling_clock(), timestamp_list(), and timestamp_count() moved to data_frame.py in issue #6 Phase 2, once diff --git a/src/dp_python_lib/client/sample_status_conversions.py b/src/dp_python_lib/client/sample_status_conversions.py index a381363..86b1c2b 100644 --- a/src/dp_python_lib/client/sample_status_conversions.py +++ b/src/dp_python_lib/client/sample_status_conversions.py @@ -21,7 +21,7 @@ from collections.abc import Iterator -from dp_python_lib.client.machine_config_client import NANOS_PER_SECOND, to_epoch_nanos +from dp_python_lib.client.time_conversions import NANOS_PER_SECOND, to_epoch_nanos from dp_python_lib.grpc import common_pb2 # The Timestamp -> epoch-nanoseconds conversion is shared (see machine_config_client.to_epoch_nanos); this private diff --git a/src/dp_python_lib/client/time_conversions.py b/src/dp_python_lib/client/time_conversions.py new file mode 100644 index 0000000..5b43152 --- /dev/null +++ b/src/dp_python_lib/client/time_conversions.py @@ -0,0 +1,95 @@ +""" +Time conversions shared across the client library. + +Every API that takes an instant accepts the same three spellings -- a timezone-aware datetime, epoch seconds, or an +already-built common.Timestamp -- and every conversions module that reads one back wants integer nanoseconds. Those +two directions are `to_timestamp()` and `to_epoch_nanos()`, and they live here rather than in a feature client. + +They were originally defined in machine_config_client, the first module to need them, and four other modules grew +imports of `to_timestamp()` from there -- which read as though datasets, queries, and DataFrames depended on the +machine configuration API. Meanwhile the reverse conversion had been written out privately three separate times. +This module is a leaf: it imports nothing from the client package, so any client module can use it freely. + +Design decisions: + - Naive datetimes are rejected rather than assumed to be UTC or local. A silent local-timezone interpretation is + the kind of bug that surfaces months later as data attributed to the wrong shift. + - `to_epoch_nanos()` is integer arithmetic on a Python int end to end. Present-day epoch nanoseconds need about + 61 bits and a float64 carries 53, so routing the conversion through float seconds would move the instant -- + and sample-status matching is by exact timestamp at nanosecond precision. +""" + +import math +from datetime import datetime + +from dp_python_lib.grpc import common_pb2 + +# Accepted input types for API parameters that map to a common.Timestamp: +# a timezone-aware datetime, epoch seconds (int or float), or an already-built Timestamp. +TimestampInput = datetime | int | float | common_pb2.Timestamp + + +NANOS_PER_SECOND = 1_000_000_000 + + +def to_epoch_nanos(timestamp: common_pb2.Timestamp) -> int: + """ + Converts a common.Timestamp into a single integer of epoch nanoseconds -- the inverse of to_timestamp(). + + Integer arithmetic throughout, on a Python int, so there is no overflow and no precision loss. Every + conversions module in the library reads instants back through this; see the module docstring for why the + integer path is a correctness requirement rather than an optimization. + + :param timestamp: The timestamp to convert. + :return: Epoch nanoseconds as a Python int. + """ + return timestamp.epochSeconds * NANOS_PER_SECOND + timestamp.nanoseconds + + +def to_timestamp(value: TimestampInput) -> common_pb2.Timestamp: + """ + Converts a user-supplied time value into a common.Timestamp{epochSeconds, nanoseconds}. + + Accepts: + - a timezone-aware datetime (naive datetimes are rejected to avoid silent local-timezone bugs), + - epoch seconds as an int or float (float fractional part becomes nanoseconds), + - an already-built common.Timestamp (returned as-is). + + :param value: The time value to convert. + :return: An equivalent common.Timestamp. + :raises ValueError: if a datetime is naive (has no tzinfo), or if the resulting epoch seconds are negative + (pre-1970) -- common.Timestamp.epochSeconds is an unsigned (uint64) field and cannot represent them. + :raises TypeError: if value is not one of the supported types. + """ + if isinstance(value, common_pb2.Timestamp): + return value + + if isinstance(value, datetime): + if value.tzinfo is None or value.tzinfo.utcoffset(value) is None: + raise ValueError( + "to_timestamp() requires a timezone-aware datetime; naive datetimes are rejected " + "to avoid silent local-timezone bugs. Use datetime.now(timezone.utc) or attach tzinfo." + ) + epoch = value.timestamp() + return to_timestamp(epoch) + + if isinstance(value, bool): + # bool is a subclass of int; reject it explicitly as it is virtually always a mistake. + raise TypeError("to_timestamp() does not accept bool") + + if isinstance(value, (int, float)): + # Floor the seconds (not truncate toward zero) so the fractional remainder, and therefore + # nanoseconds, is always in [0, 1_000_000_000) even for negative epoch inputs. + epoch_seconds = math.floor(value) + nanoseconds = int(round((float(value) - epoch_seconds) * 1_000_000_000)) + # Guard against float rounding pushing nanoseconds up to a full second. + if nanoseconds >= 1_000_000_000: + epoch_seconds += 1 + nanoseconds -= 1_000_000_000 + timestamp = common_pb2.Timestamp() + timestamp.epochSeconds = epoch_seconds + timestamp.nanoseconds = nanoseconds + return timestamp + + raise TypeError( + f"to_timestamp() expects datetime, int/float epoch seconds, or common.Timestamp, got {type(value).__name__}" + ) diff --git a/tests/unit/test_machine_config_client.py b/tests/unit/test_machine_config_client.py index 97324d4..95a0a4a 100644 --- a/tests/unit/test_machine_config_client.py +++ b/tests/unit/test_machine_config_client.py @@ -1,7 +1,6 @@ import os import sys import unittest -from datetime import datetime, timedelta, timezone from unittest.mock import Mock import grpc @@ -17,8 +16,6 @@ QueryConfigurationsApiResult, SaveConfigurationApiResult, SaveConfigurationRequestParams, - to_epoch_nanos, - to_timestamp, ) from dp_python_lib.grpc import annotation_pb2, common_pb2 @@ -33,82 +30,6 @@ def _response_with_field(field_name): return response -class TestToEpochNanos(unittest.TestCase): - """to_epoch_nanos() is to_timestamp()'s inverse, shared by the three conversions modules.""" - - def test_combines_seconds_and_nanoseconds(self): - ts = common_pb2.Timestamp() - ts.epochSeconds = 1_700_000_000 - ts.nanoseconds = 123_456_789 - self.assertEqual(to_epoch_nanos(ts), 1_700_000_000_123_456_789) - - def test_round_trips_with_to_timestamp_exactly(self): - # The exactness is the point: present-day epoch nanoseconds need ~61 bits and a float64 carries 53, so a - # conversion routed through float seconds would move the instant. - original = 1_770_055_200_123_456_789 - ts = common_pb2.Timestamp() - ts.epochSeconds, ts.nanoseconds = divmod(original, 1_000_000_000) - self.assertEqual(to_epoch_nanos(ts), original) - - def test_zero_timestamp_is_zero(self): - self.assertEqual(to_epoch_nanos(common_pb2.Timestamp()), 0) - - -class TestToTimestamp(unittest.TestCase): - """Unit tests for the to_timestamp() conversion helper.""" - - def test_passthrough_timestamp(self): - ts = common_pb2.Timestamp() - ts.epochSeconds = 123 - ts.nanoseconds = 456 - self.assertIs(to_timestamp(ts), ts) - - def test_int_epoch_seconds(self): - ts = to_timestamp(1_700_000_000) - self.assertEqual(ts.epochSeconds, 1_700_000_000) - self.assertEqual(ts.nanoseconds, 0) - - def test_float_epoch_seconds_with_fraction(self): - ts = to_timestamp(1_700_000_000.25) - self.assertEqual(ts.epochSeconds, 1_700_000_000) - self.assertEqual(ts.nanoseconds, 250_000_000) - - def test_aware_datetime_utc(self): - dt = datetime(2023, 11, 14, 22, 13, 20, tzinfo=timezone.utc) - expected_epoch = int(dt.timestamp()) - ts = to_timestamp(dt) - self.assertEqual(ts.epochSeconds, expected_epoch) - self.assertEqual(ts.nanoseconds, 0) - - def test_aware_datetime_nonzero_offset(self): - tz = timezone(timedelta(hours=-5)) - dt = datetime(2023, 11, 14, 17, 13, 20, tzinfo=tz) # same instant as the UTC test above - ts = to_timestamp(dt) - self.assertEqual(ts.epochSeconds, int(dt.timestamp())) - - def test_negative_epoch_rejected_by_uint64_field(self): - # common.Timestamp.epochSeconds is uint64, so pre-1970 (negative) epochs cannot be represented. - # Flooring keeps nanoseconds normalized, and the negative seconds are cleanly rejected at assignment - # (ValueError: out of range) rather than silently producing an incorrect (seconds, nanos) pair. - with self.assertRaises(ValueError): - to_timestamp(-1.25) - with self.assertRaises(ValueError): - to_timestamp(-5) - - def test_naive_datetime_raises(self): - with self.assertRaises(ValueError): - # DTZ001: the naive datetime is the point of this test -- it must be rejected. - to_timestamp(datetime(2023, 11, 14, 22, 13, 20)) # noqa: DTZ001 - - def test_bool_raises(self): - with self.assertRaises(TypeError): - to_timestamp(True) - - def test_unsupported_type_raises(self): - with self.assertRaises(TypeError): - to_timestamp("2023-11-14") - - class TestConfigurationQuery(unittest.TestCase): """Unit tests for the ConfigurationQuery criterion helpers.""" diff --git a/tests/unit/test_time_conversions.py b/tests/unit/test_time_conversions.py new file mode 100644 index 0000000..a8df038 --- /dev/null +++ b/tests/unit/test_time_conversions.py @@ -0,0 +1,90 @@ +import os +import sys +import unittest +from datetime import datetime, timedelta, timezone + +# Add src directory to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src")) + +from dp_python_lib.client.time_conversions import to_epoch_nanos, to_timestamp +from dp_python_lib.grpc import common_pb2 + + +class TestToEpochNanos(unittest.TestCase): + """to_epoch_nanos() is to_timestamp()'s inverse, shared by the three conversions modules.""" + + def test_combines_seconds_and_nanoseconds(self): + ts = common_pb2.Timestamp() + ts.epochSeconds = 1_700_000_000 + ts.nanoseconds = 123_456_789 + self.assertEqual(to_epoch_nanos(ts), 1_700_000_000_123_456_789) + + def test_round_trips_with_to_timestamp_exactly(self): + # The exactness is the point: present-day epoch nanoseconds need ~61 bits and a float64 carries 53, so a + # conversion routed through float seconds would move the instant. + original = 1_770_055_200_123_456_789 + ts = common_pb2.Timestamp() + ts.epochSeconds, ts.nanoseconds = divmod(original, 1_000_000_000) + self.assertEqual(to_epoch_nanos(ts), original) + + def test_zero_timestamp_is_zero(self): + self.assertEqual(to_epoch_nanos(common_pb2.Timestamp()), 0) + + +class TestToTimestamp(unittest.TestCase): + """Unit tests for the to_timestamp() conversion helper.""" + + def test_passthrough_timestamp(self): + ts = common_pb2.Timestamp() + ts.epochSeconds = 123 + ts.nanoseconds = 456 + self.assertIs(to_timestamp(ts), ts) + + def test_int_epoch_seconds(self): + ts = to_timestamp(1_700_000_000) + self.assertEqual(ts.epochSeconds, 1_700_000_000) + self.assertEqual(ts.nanoseconds, 0) + + def test_float_epoch_seconds_with_fraction(self): + ts = to_timestamp(1_700_000_000.25) + self.assertEqual(ts.epochSeconds, 1_700_000_000) + self.assertEqual(ts.nanoseconds, 250_000_000) + + def test_aware_datetime_utc(self): + dt = datetime(2023, 11, 14, 22, 13, 20, tzinfo=timezone.utc) + expected_epoch = int(dt.timestamp()) + ts = to_timestamp(dt) + self.assertEqual(ts.epochSeconds, expected_epoch) + self.assertEqual(ts.nanoseconds, 0) + + def test_aware_datetime_nonzero_offset(self): + tz = timezone(timedelta(hours=-5)) + dt = datetime(2023, 11, 14, 17, 13, 20, tzinfo=tz) # same instant as the UTC test above + ts = to_timestamp(dt) + self.assertEqual(ts.epochSeconds, int(dt.timestamp())) + + def test_negative_epoch_rejected_by_uint64_field(self): + # common.Timestamp.epochSeconds is uint64, so pre-1970 (negative) epochs cannot be represented. + # Flooring keeps nanoseconds normalized, and the negative seconds are cleanly rejected at assignment + # (ValueError: out of range) rather than silently producing an incorrect (seconds, nanos) pair. + with self.assertRaises(ValueError): + to_timestamp(-1.25) + with self.assertRaises(ValueError): + to_timestamp(-5) + + def test_naive_datetime_raises(self): + with self.assertRaises(ValueError): + # DTZ001: the naive datetime is the point of this test -- it must be rejected. + to_timestamp(datetime(2023, 11, 14, 22, 13, 20)) # noqa: DTZ001 + + def test_bool_raises(self): + with self.assertRaises(TypeError): + to_timestamp(True) + + def test_unsupported_type_raises(self): + with self.assertRaises(TypeError): + to_timestamp("2023-11-14") + + +if __name__ == "__main__": + unittest.main() From 86621d19bff9880801793e5f6622a437bafd1a80 Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Thu, 10 Sep 2026 13:02:52 -0600 Subject: [PATCH 07/11] fix: address Copilot review findings (issue #6, PR 2) Six findings from Copilot's review of f101ff6. A seventh -- array columns accepted by floor division -- duplicated a finding already fixed in a6e1865. ImageColumn was unusable. _column_sample_count() fell through to len(column.values) for it, but an ImageColumn keeps one payload per sample in `images` and has no `values` field at all, so every prebuilt image column -- a kind data_frame() documents as supported and routes to imageColumns -- raised AttributeError instead of being assembled. A pandas index in any unit but nanoseconds was serialized wrong. _timestamps_from_index() read the index through a raw int64 view, which is expressed in the index's OWN storage unit; pandas 2+ keeps second, millisecond, and microsecond resolutions, so a datetime64[us] index produced timestamps 1000x too early with no error. It now reads Timestamp.value, which is nanoseconds whatever the unit, and rejects NaT, whose integer form is the int64 minimum and would otherwise serialize as a real instant. This one had a test that should have caught it: the round trip compared raw int64 views on both sides, so it passed while both were equally wrong. It now compares the instants. date_range() returns a microsecond-unit index under pandas 3, so the fixture was already exercising the broken path. Four smaller ones: - Column names were checked for emptiness, but the documented rule is non-blank; " " passed. Applied in data_frame() and in the builders. - data_frame_timestamps() documented that it rejects an empty axis and did not: expand_data_timestamps() returns [] for a set-but-empty timestampList, whose oneof still reports as set, so a corrupt frame became a zero-row table instead of failing loudly. - Array dims were used to delimit samples and then discarded, leaving [2, 2] and [4] indistinguishable, though plan D7 asks for the dims alongside the values. Added column_dimensions() / data_frame_column_dimensions() rather than changing column_values()'s one-entry-per-sample contract, which the cookbook and integration tests depend on. - The cookbook's first snippet bound `saved_id` while every later snippet read `dataset_id`, and likewise for the annotation and calculations ids. Every snippet type-checked because the checker preamble pre-seeds those names, so the recipe was broken only when read end to end -- which is how a reader reads it. The preamble now records that seeding a name cannot prove the recipe binds it. 734 tests pass (692 unit, 42 integration + subtests against the live server); ruff clean; 103 cookbook snippets check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- .dev/tools/check-cookbook-snippets.py | 20 +++-- CLAUDE.md | 4 +- doc/cookbook/datasets-and-annotations.md | 14 +-- plan/tickets/6/plan.md | 21 +++++ src/dp_python_lib/client/data_frame.py | 24 +++-- .../client/data_frame_conversions.py | 87 ++++++++++++++++--- tests/unit/test_data_frame.py | 33 +++++++ tests/unit/test_data_frame_conversions.py | 72 ++++++++++++++- 8 files changed, 237 insertions(+), 38 deletions(-) diff --git a/.dev/tools/check-cookbook-snippets.py b/.dev/tools/check-cookbook-snippets.py index 71e400c..ca173a0 100755 --- a/.dev/tools/check-cookbook-snippets.py +++ b/.dev/tools/check-cookbook-snippets.py @@ -140,12 +140,20 @@ # and `params` are carried above. Server-assigned ids are strings. t0: datetime = datetime(2026, 2, 2, 18, 0, tzinfo=timezone.utc) t1: datetime = datetime(2026, 2, 2, 19, 0, tzinfo=timezone.utc) -# Typed `str` because the recipe's snippets narrow the Optional accessors with an assert before -# carrying the id forward -- which is the pattern conventions.md teaches, so the declared type here -# is the post-narrowing one. -dataset_id: str = "6aa1bb271a768e97db44d426" -annotation_id: str = "6aa1bb271a768e97db44d427" -calculations_id: str = "6aa1bb271a768e97db44d428" +# Declared without an annotation so mypy infers `str` for the standalone snippets that consume these, +# while the recipe's own snippets can still rebind them from an Optional accessor and narrow with an +# assert, as conventions.md teaches. An explicit `str` would conflict with those real assignments; an +# explicit `str | None` would force every later snippet to re-narrow a handle the recipe already did. +# +# Seeding a handle here cannot prove the recipe actually binds that name -- a snippet binding `saved_id` +# while later ones read `dataset_id` type-checked cleanly and was still broken end to end. Names carried +# across snippets are verified by reading the recipe as one continuous script, not by this preamble. +# `str | None` is what the accessors return; the recipe narrows with an assert before use, and the +# standalone snippets below do the same, so consumers see a plain `str`. +dataset_id: str | None = "6aa1bb271a768e97db44d426" +annotation_id: str | None = "6aa1bb271a768e97db44d427" +calculations_id: str | None = "6aa1bb271a768e97db44d428" +assert dataset_id is not None and annotation_id is not None and calculations_id is not None # --- end preamble --- """ diff --git a/CLAUDE.md b/CLAUDE.md index 6a331c5..1b1cca6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -157,8 +157,8 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/annotations_client.py` - Annotations client (`save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, `delete_annotation()`, `get_calculations()`) with the `AnnotationQuery` (`AQ`) criterion helpers and the `calculations()` builder, which takes a `dict[str, DataFrame]` so frame-name uniqueness is true by construction. Note `AnnotationsClient` (feature client) vs `AnnotationClient` (facade) - `src/dp_python_lib/client/export_client.py` - Export client (`export_data()`) with the `ExportFormat` str enum and the `calculations_spec()` builder - `src/dp_python_lib/client/time_conversions.py` - The two shared time converters and the `TimestampInput` alias: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`; naive datetimes raise) and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds), plus `NANOS_PER_SECOND`. Both were defined in `machine_config_client` as the first module to need them, and six others grew imports from there — which read as though datasets, queries, and DataFrames depended on the machine configuration API; `to_epoch_nanos()` had also been written out privately three separate times. **A leaf module**: it imports only stdlib and the generated protos, so any client module can use it without an import cycle. New time conversions belong here -- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through -- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one list per sample), `data_frame_columns()`, `column_metadata_dict()`. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap), and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`, and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type +- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Column names must be **non-blank**, not merely non-empty. Sample counts come from the right field per kind: `dataValues` for a `DataColumn`, `images` for an `ImageColumn` (which has no `values` field at all), and `len(values) / prod(dims)` for an array column. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through +- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one flat list per sample, with the shape recoverable from `column_dimensions()` / `data_frame_column_dimensions()` so `[2,2]` and `[4]` stay distinguishable), `data_frame_columns()`, `column_metadata_dict()`. An axis that is set but empty is rejected rather than converted to a zero-row table. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap), and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`; reads each instant as `Timestamp.value` (**always nanoseconds**, unlike a raw int64 view, which is in the index's own storage unit — a `datetime64[us]` index viewed as int64 lands 1000× too early); rejects `NaT`, whose integer form is a valid-looking instant; and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type - `src/dp_python_lib/client/query_client.py` - v2 time-series query client (sample-oriented) exposed as `client.query`. Low-level wrappers `query_samples()` (unary, one resumable page) and `iter_query_samples()` (transparent paging), plus `iter_query_samples_stream()` (server-streaming, fire-and-consume, lazy). Queries are described by a kind-neutral `QueryParams` built from the `PvQuery` (`PV`) and `ConfigQuery` (`CFG`) criterion helpers; shares a `_build_query_spec()` seam so a future bucket request builder reuses it. Results wrap the raw `ColumnTable` (`.column_table`, `.next_page_token`); `.to_dataframe()`/`.to_numpy()` delegate to `query_conversions` (Phase 2, optional `[analysis]` extra) - `src/dp_python_lib/client/query_conversions.py` - Pythonic conversions for query results (optional `[analysis]` extra: pandas/numpy/openpyxl, imported lazily). `data_value_to_python()` (oneof extractor: scalars→native, timestamp→epoch-nanos, array→list, structure→dict, image→`Image` wrapper, fail-loud on unhandled arm), `column_table_to_dataframe()` (UTC datetime index + one column per DataColumn; dense-alignment and duplicate-column-name fail-loud; ColumnMetadata in `df.attrs`), `column_table_to_numpy()` (dict of 1-D arrays; complex arms stay 1-D object arrays rather than collapsing to 2-D), `dataframe_to_excel()` (thin `to_excel()` wrapper: row-limit guard, tz-drop, complex-cell stringification), and `query_samples_to_dataframe()`/`stream_query_samples_to_dataframes()` whole-query conveniences (unary concats by column name; streaming yields per-page frames lazily) - `src/dp_python_lib/client/service_api_client_base.py` - Base class for the service clients: owns the channel and the one-per-client gRPC stub, and provides `_dispatch()`, the shared three-tier sender that all 18 unary `_send_*` methods delegate to diff --git a/doc/cookbook/datasets-and-annotations.md b/doc/cookbook/datasets-and-annotations.md index bb31185..1433faf 100644 --- a/doc/cookbook/datasets-and-annotations.md +++ b/doc/cookbook/datasets-and-annotations.md @@ -111,9 +111,9 @@ saved = client.annotation.datasets.save_dataset(SaveDataSetRequestParams( if saved.result_status.is_error: raise RuntimeError(saved.result_status.message) -saved_id = saved.dataset_id -assert saved_id is not None # guaranteed once is_error is False; accessors are Optional -print(saved_id) # server-assigned id, e.g. '6aa1bb271a768e97db44d426' +dataset_id = saved.dataset_id +assert dataset_id is not None # guaranteed once is_error is False; accessors are Optional +print(dataset_id) # server-assigned id, e.g. '6aa1bb271a768e97db44d426' ``` `data_block()` requires `begin < end` and a non-empty PV list. That check exists here because the @@ -218,8 +218,8 @@ result = client.annotation.annotations.save_annotation(SaveAnnotationRequestPara if result.result_status.is_error: raise RuntimeError(result.result_status.message) -saved_annotation_id = result.annotation_id -assert saved_annotation_id is not None +annotation_id = result.annotation_id +assert annotation_id is not None ``` To attach *numbers*, build a `Calculations` payload. A frame is one time axis plus its columns: @@ -246,8 +246,8 @@ result = client.annotation.annotations.save_annotation(SaveAnnotationRequestPara calculations=calculations({"orbit-rms": frame}), modified_by="cmcchesney", )) -saved_calculations_id = result.calculations_id -assert saved_calculations_id is not None +calculations_id = result.calculations_id +assert calculations_id is not None ``` `calculations()` takes a **dict** of frame name to frame, which makes frame-name uniqueness true by diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index ce8c2c5..cbf6ce1 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -80,6 +80,27 @@ caller was repointed at it, and `machine_config_client` is now just another caller. Done inside this ticket rather than deferred, since the feature is unreleased and a follow-up would ship the wart in the release. +- **Copilot review of PR #45, 2026-09-10.** Seven findings against `f101ff6`; one (array floor division) duplicated a + finding already fixed, and the other six were real: + - `data_frame()` counted every column as `len(column.values)`, but an `ImageColumn` keeps its per-sample payloads + in `images` and has no `values` field, so every prebuilt image column -- a kind the module documents as + supported -- raised `AttributeError` instead of being assembled. + - `_timestamps_from_index()` read the index as a raw int64 view, which is expressed in the index's own storage + unit. pandas 2+ keeps second/millisecond/microsecond resolutions, so a `datetime64[us]` index was serialized + 1000x too early, silently. It now reads `Timestamp.value`, which is nanoseconds regardless of unit, and + rejects `NaT` (whose integer form is a valid-looking instant). A round-trip test had compared raw int64 views + on both sides, so it passed while both were equally wrong. + - Column-name validation accepted whitespace-only names, though the documented rule is non-blank. + - `data_frame_timestamps()` documented that it rejects an empty axis but returned `[]` for a set-but-empty + `timestampList`, converting a corrupt frame to a zero-row table. + - Array dims were consumed to delimit samples and then discarded, so `[2, 2]` and `[4]` were indistinguishable -- + D7 calls for the dims to travel alongside the values. Added `column_dimensions()` / + `data_frame_column_dimensions()` rather than changing `column_values()`'s one-entry-per-sample contract. + - The cookbook's first snippet bound `saved_id` while every later snippet read `dataset_id` (likewise + `saved_annotation_id` / `saved_calculations_id`). Every snippet type-checked because the checker preamble + pre-seeded those names -- the recipe was broken only when read end to end, which is how a reader reads it. + The preamble now documents that seeding a name cannot prove the recipe binds it. + ## Overview Wrap the modernized DataSet / Annotation / Calculations / Export area of `DpAnnotationService` in the house diff --git a/src/dp_python_lib/client/data_frame.py b/src/dp_python_lib/client/data_frame.py index efb4af1..e9923f3 100644 --- a/src/dp_python_lib/client/data_frame.py +++ b/src/dp_python_lib/client/data_frame.py @@ -325,8 +325,8 @@ def _build_scalar_column( :return: The constructed typed column message. :raises ValueError: if name is empty or values is empty. """ - if not name: - raise ValueError(f"{builder_name}() requires a non-empty name") + if not name or not name.strip(): + raise ValueError(f"{builder_name}() requires a non-blank name, got {name!r}") if not values: raise ValueError(f"{builder_name}() requires a non-empty values list for column '{name}'") @@ -497,8 +497,8 @@ def data_column( :return: A common.DataColumn. :raises ValueError: if name or values is empty, or if a value's type has no DataValue mapping. """ - if not name: - raise ValueError("data_column() requires a non-empty name") + if not name or not name.strip(): + raise ValueError(f"data_column() requires a non-blank name, got {name!r}") if not values: raise ValueError(f"data_column() requires a non-empty values list for column '{name}'") @@ -570,7 +570,10 @@ def _column_sample_count(column: Any) -> int | None: floor division round it into a passing count. The read path (data_frame_conversions._reshape_array_values) applies the same rule, and a frame this accepted but that could not be read back would be the worst outcome. - :param column: A typed column, a legacy DataColumn, or a SerializedDataColumn. + An ImageColumn counts its `images`, and a DataColumn its `dataValues`; only the scalar and array columns keep + their per-sample entries in `values`. + + :param column: A typed column, a legacy DataColumn, an ImageColumn, or a SerializedDataColumn. :return: The sample count, or None for a SerializedDataColumn (whose payload is opaque) and for an array column whose dims are missing, non-positive, or do not evenly divide its values. """ @@ -578,6 +581,10 @@ def _column_sample_count(column: Any) -> int | None: return None if isinstance(column, common_pb2.DataColumn): return len(column.dataValues) + if isinstance(column, common_pb2.ImageColumn): + # ImageColumn holds one encoded payload per sample in `images`; it has no `values` field at all, so the + # generic path below would raise AttributeError on a column this function claims to support. + return len(column.images) if isinstance(column, _ARRAY_COLUMN_TYPES): product = _array_sample_size(column) if product is None: @@ -606,8 +613,11 @@ def _check_column(column: Any, index: int, expected_count: int, seen_names: set[ ) name = column.name - if not name: - raise ValueError(f"data_frame() requires a non-empty name for every column; column at index {index} has none") + if not name or not name.strip(): + # Blank, not merely empty: the rule is a non-blank name, and a whitespace-only one is not a name. + raise ValueError( + f"data_frame() requires a non-blank name for every column; column at index {index} has {name!r}" + ) if name in seen_names: raise ValueError( f"data_frame() requires unique column names within a frame; '{name}' appears more than once " diff --git a/src/dp_python_lib/client/data_frame_conversions.py b/src/dp_python_lib/client/data_frame_conversions.py index 56f1703..cc980fc 100644 --- a/src/dp_python_lib/client/data_frame_conversions.py +++ b/src/dp_python_lib/client/data_frame_conversions.py @@ -51,6 +51,15 @@ # ImageColumn stores its per-sample payloads in `images` rather than `values`. _IMAGE_COLUMN_FIELD = "imageColumns" +# The array column message types, whose flat values are delimited by their declared dims. +_ARRAY_COLUMN_MESSAGE_TYPES = ( + common_pb2.DoubleArrayColumn, + common_pb2.FloatArrayColumn, + common_pb2.Int32ArrayColumn, + common_pb2.Int64ArrayColumn, + common_pb2.BoolArrayColumn, +) + def _require_pandas(): """Imports and returns pandas, or raises an actionable error if the optional [analysis] extra is missing.""" @@ -77,7 +86,14 @@ def data_frame_timestamps(frame: common_pb2.DataFrame) -> list[int]: """ if not frame.HasField("dataTimestamps"): raise ValueError("DataFrame has no dataTimestamps; every frame must carry a time axis") - return expand_data_timestamps(frame.dataTimestamps) + + epoch_nanos = expand_data_timestamps(frame.dataTimestamps) + if not epoch_nanos: + # expand_data_timestamps() returns [] for a set-but-empty timestampList, which reports its oneof arm as + # set. Rejecting it here honors this function's documented contract and matches timestamp_count() on the + # write side; otherwise a corrupt frame converts to a zero-row table instead of failing loudly. + raise ValueError("DataFrame time axis describes no timestamps; every frame must cover at least one sample") + return epoch_nanos def _reshape_array_values(column: Any) -> list[list]: @@ -108,6 +124,41 @@ def _reshape_array_values(column: Any) -> list[list]: return [values[i : i + product] for i in range(0, len(values), product)] +def column_dimensions(column: Any) -> list[int] | None: + """ + Returns an array column's declared dimensions, or None for any other column kind. + + column_values() reshapes an array column into one flat list per sample, which delimits the samples but does + not preserve the shape WITHIN one: a 2x2 sample and a 4-element sample both come back as four values. Plan + D7 calls for the dims to travel alongside those values, and this is the accessor that supplies them, kept + separate so column_values()'s "one entry per sample" contract stays uniform across all column kinds. + + :param column: Any column message. + :return: The dims as a list of ints for an array column, or None if the column is not an array column. + """ + if not isinstance(column, _ARRAY_COLUMN_MESSAGE_TYPES): + return None + return list(column.dimensions.dims) + + +def data_frame_column_dimensions(frame: common_pb2.DataFrame) -> dict[str, list[int]]: + """ + Returns the declared dimensions of every array column in a frame, keyed by column name. + + Pairs with data_frame_columns(): that gives one flat list per sample, this gives the shape those values have. + Non-array columns are absent from the result rather than mapped to None, so a caller can test membership. + + :param frame: The DataFrame to inspect. + :return: A dict mapping each array column's name to its dims; empty when the frame has no array columns. + """ + dimensions: dict[str, list[int]] = {} + for column in iter_frame_columns(frame): + dims = column_dimensions(column) + if dims is not None: + dimensions[column.name] = dims + return dimensions + + def column_values(column: Any) -> list: """ Extracts one Python value per sample from any supported column message. @@ -120,6 +171,9 @@ def column_values(column: Any) -> list: bytes payload per sample; and a legacy DataColumn is converted per value by data_value_to_python(), so an unset oneof becomes None -- the only representation of a gap in this API. + An array column's per-sample list is flat: the dims that give it shape are available separately from + column_dimensions(), so that every column kind here yields exactly one entry per sample. + :param column: A typed column, a legacy DataColumn, or an ImageColumn. :return: One value per sample, in axis order. :raises ValueError: if the column type is unsupported, or an array column's dims do not divide its values. @@ -128,16 +182,7 @@ def column_values(column: Any) -> list: return [data_value_to_python(value) for value in column.dataValues] if isinstance(column, common_pb2.ImageColumn): return list(column.images) - if isinstance( - column, - ( - common_pb2.DoubleArrayColumn, - common_pb2.FloatArrayColumn, - common_pb2.Int32ArrayColumn, - common_pb2.Int64ArrayColumn, - common_pb2.BoolArrayColumn, - ), - ): + if isinstance(column, _ARRAY_COLUMN_MESSAGE_TYPES): return _reshape_array_values(column) if hasattr(column, "values"): return list(column.values) @@ -311,6 +356,10 @@ def _timestamps_from_index(index: Any) -> common_pb2.DataTimestamps: that merely looks regular would quietly change the timestamps -- the one thing this API cannot tolerate. Build a clock explicitly with sampling_clock() when that is what the data is. + Each instant is read as Timestamp.value, which is nanoseconds whatever the index's own storage unit; a raw + int64 view is in that unit, so a datetime64[us] index would land 1000x too early. NaT is rejected rather than + passed through, since its integer form is a valid-looking instant. + :param index: A pandas DatetimeIndex. :return: A DataTimestamps carrying a TimestampList. :raises ValueError: if the index is empty, not a DatetimeIndex, or not strictly increasing. @@ -331,9 +380,19 @@ def _timestamps_from_index(index: Any) -> common_pb2.DataTimestamps: "e.g. df.index = df.index.tz_localize('UTC')" ) - epoch_nanos = [int(value) for value in index.view("int64")] if hasattr(index, "view") else None - if epoch_nanos is None: - epoch_nanos = [int(value) for value in index.astype("int64")] + # NaT has an integer representation (the int64 minimum), so a raw integer view would serialize it as a real + # instant. Reject it before converting: an absent timestamp is not a time axis position. + if index.isna().any(): + missing = [int(position) for position in index.isna().nonzero()[0][:5]] + raise ValueError( + f"DataFrame index contains NaT at row position(s) {missing}; every row must have a real timestamp to " + f"become a time axis. Drop those rows, or supply the timestamps they should carry." + ) + + # Timestamp.value is always nanoseconds regardless of the index's own storage unit. A raw int64 view is NOT: + # pandas 2+ keeps second, millisecond, and microsecond resolutions, and viewing a datetime64[us] index as + # int64 yields microseconds -- silently placing every instant 1000x too early. + epoch_nanos = [entry.value for entry in index] timestamps = common_pb2.DataTimestamps() previous = None diff --git a/tests/unit/test_data_frame.py b/tests/unit/test_data_frame.py index b43664c..4f9835c 100644 --- a/tests/unit/test_data_frame.py +++ b/tests/unit/test_data_frame.py @@ -194,6 +194,11 @@ def test_rejects_unmappable_type(self): self.assertIn("complex", message) self.assertIn("index 0", message) + def test_rejects_whitespace_only_name(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_column(" ", [1]) + self.assertIn("non-blank", str(ctx.exception)) + def test_rejects_empty_name_and_values(self): with self.assertRaises(ValueError): dfb.data_column("", [1]) @@ -353,6 +358,34 @@ def test_array_column_count_uses_dims_product(self): dfb.data_frame(_axis(3), [column]) self.assertIn("2 values", str(ctx.exception)) + def test_accepts_prebuilt_image_column(self): + # ImageColumn keeps one payload per sample in `images`; it has no `values` field at all, so a generic + # len(column.values) count raises AttributeError on a column type data_frame() claims to support. + column = common_pb2.ImageColumn() + column.name = "camera" + column.images.extend([b"frame-0", b"frame-1"]) + frame = dfb.data_frame(_axis(2), [column]) + self.assertEqual([c.name for c in frame.imageColumns], ["camera"]) + + def test_image_column_count_is_validated_against_the_axis(self): + column = common_pb2.ImageColumn() + column.name = "camera" + column.images.extend([b"only-one"]) + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(3), [column]) + self.assertIn("1 values", str(ctx.exception)) + + def test_rejects_whitespace_only_column_name(self): + # The rule is a non-blank name; " " is empty of content while passing a bare falsiness check. + for blank in (" ", "\t", "\n"): + with self.subTest(name=blank): + column = common_pb2.DoubleColumn() + column.name = blank + column.values[:] = [1.0] + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), [column]) + self.assertIn("non-blank", str(ctx.exception)) + def test_array_column_with_ragged_values_is_rejected(self): # 5 values with a per-sample size of 2 is not a whole number of samples. Floor division would round it to # a passing count of 2 and build a frame that data_frame_conversions then refuses to read back. diff --git a/tests/unit/test_data_frame_conversions.py b/tests/unit/test_data_frame_conversions.py index 892bcca..c8dc7cd 100644 --- a/tests/unit/test_data_frame_conversions.py +++ b/tests/unit/test_data_frame_conversions.py @@ -344,9 +344,12 @@ def test_round_trip_preserves_values_and_instants(self): self.assertEqual(back["d"].tolist(), [1.0, 2.0, 3.0]) self.assertEqual(back["s"].tolist(), ["a", "b", "c"]) + # Compare the instants themselves, not raw int64 views: a view is in the index's own storage unit, and + # date_range() yields a microsecond-unit index here while the converter always emits nanoseconds. The + # earlier form compared two different units and only passed because both sides were equally wrong. self.assertEqual( - [int(v) for v in back.index.view("int64")], - [int(v) for v in index.view("int64")], + [entry.value for entry in back.index], + [entry.value for entry in index], ) def test_nan_is_fail_loud_with_sparsity_guidance(self): @@ -534,5 +537,70 @@ def test_values_index_and_dtypes_survive_the_reordering(self): self.assertEqual(list(original[name]), list(realigned[name])) +class TestEmptyAxisIsRejected(unittest.TestCase): + def test_set_but_empty_timestamp_list_is_rejected(self): + # expand_data_timestamps() returns [] for this, and the oneof still reports as set, so without an explicit + # check a corrupt frame converts to a zero-row table instead of failing loudly. + frame = common_pb2.DataFrame() + frame.dataTimestamps.timestampList.SetInParent() + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_timestamps(frame) + self.assertIn("no timestamps", str(ctx.exception)) + + +class TestArrayColumnDimensions(unittest.TestCase): + """column_values() flattens each sample; the dims that give it shape are recoverable separately (plan D7).""" + + def _frame(self, dims, values, count): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend(dims) + column.values[:] = values + return dfb.data_frame(_axis(count), [column]) + + def test_dimensions_distinguish_shapes_with_equal_sample_size(self): + # [2, 2] and [4] both yield four values per sample; without the dims they are indistinguishable. + square = self._frame([2, 2], [float(i) for i in range(8)], 2) + flat = self._frame([4], [float(i) for i in range(8)], 2) + + self.assertEqual(dfc.data_frame_columns(square), dfc.data_frame_columns(flat)) + self.assertEqual(dfc.data_frame_column_dimensions(square), {"waveform": [2, 2]}) + self.assertEqual(dfc.data_frame_column_dimensions(flat), {"waveform": [4]}) + + def test_non_array_columns_have_no_dimensions(self): + self.assertIsNone(dfc.column_dimensions(dfb.double_column("d", [1.0]))) + frame = dfb.data_frame(_axis(1), [dfb.double_column("d", [1.0])]) + self.assertEqual(dfc.data_frame_column_dimensions(frame), {}) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestIndexUnitAndNaT(unittest.TestCase): + """The index's storage unit is not guaranteed to be nanoseconds, and NaT has an integer representation.""" + + def test_non_nanosecond_index_units_convert_exactly(self): + import pandas as pd + + # A raw int64 view is in the index's OWN unit, so a datetime64[us] index would land 1000x too early. + for unit, expected_nanos in ( + ("s", 1_700_000_000_000_000_000), + ("ms", 1_700_000_000_123_000_000), + ("us", 1_700_000_000_123_456_000), + ("ns", 1_700_000_000_123_456_789), + ): + with self.subTest(unit=unit): + index = pd.DatetimeIndex(pd.to_datetime([1_700_000_000_123_456_789], unit="ns", utc=True)).as_unit(unit) + timestamp = dfc._timestamps_from_index(index).timestampList.timestamps[0] + actual = timestamp.epochSeconds * 1_000_000_000 + timestamp.nanoseconds + self.assertEqual(actual, expected_nanos) + + def test_nat_in_the_index_is_rejected(self): + import pandas as pd + + index = pd.DatetimeIndex([pd.NaT, pd.Timestamp("2026-01-01", tz="UTC")]) + with self.assertRaises(ValueError) as ctx: + dfc._timestamps_from_index(index) + self.assertIn("NaT", str(ctx.exception)) + + if __name__ == "__main__": unittest.main() From 70d21021dd7c3f812e14f5137fad1aaf092c257f Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Thu, 10 Sep 2026 13:28:13 -0600 Subject: [PATCH 08/11] fix: exact datetime conversion, plus three review findings (issue #6, PR 2) Four findings from Copilot's second pass over 86621d1, all real. to_timestamp() lost sub-microsecond precision on EVERY datetime. The datetime branch computed value.timestamp() and converted the float, but a float64 cannot hold present-day epoch seconds at sub-microsecond resolution: 99.7% of microsecond-precision datetimes came back with a wrong nanosecond field, by up to ~119 ns. 2026-07-14T18:00:00.000001Z produced 954 ns instead of 1000. That is the exact-match contract the whole library rests on -- sample-status matching is by exact timestamp at nanosecond precision, and provenance time ranges are compared the same way -- so this quietly undercut the property several modules take pains to preserve. It predates this ticket; moving the converters into time_conversions.py is what put the code under review. The datetime path is now integer arithmetic off the timedelta from the epoch, which is lossless because a datetime's own resolution is exactly microseconds. Verified exact across 50,000 random microsecond values and every 977th microsecond of a second. Pre-1970 datetimes still raise, now with a message naming the instant. The int/float epoch-seconds path is untouched. Three others: - timestamp_count() accepted a hand-built SamplingClock with periodNanos == 0, which expand_data_timestamps() rejects -- the same write/read asymmetry as the earlier array-dims finding. Every sample after the first would have carried the first sample's timestamp. - The cookbook's calculations save passed no annotation_id, so it created a SECOND annotation while the recipe kept reading the first, calculation-free one; every later get_annotation() read the wrong record. It now rebinds both handles and says why, pointing at the update recipe for the other intent. - The cookbook's replace discarded its result, but a replace carrying calculations stores a new object and deletes the old one, leaving the calculations_id the later export used dangling. It now captures and rebinds. Both cookbook findings share a root cause with the earlier saved_id one: snippets are type-checked in isolation against a seeded preamble, which cannot see that a SEQUENCE of snippets is incoherent. Checked this time by parsing the recipe as one continuous script and confirming each handle is rebound before its next use. 740 tests pass (698 unit, 42 integration + subtests against the live server; 635 pass and 63 skip without the [analysis] extra); ruff clean; 103 cookbook snippets check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- CLAUDE.md | 4 +- doc/cookbook/datasets-and-annotations.md | 19 ++++++++- plan/tickets/6/plan.md | 21 ++++++++++ src/dp_python_lib/client/data_frame.py | 20 +++++++--- src/dp_python_lib/client/time_conversions.py | 26 ++++++++++-- tests/unit/test_data_frame.py | 12 ++++++ tests/unit/test_time_conversions.py | 42 ++++++++++++++++++++ 7 files changed, 131 insertions(+), 13 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1b1cca6..c208bdd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,8 +156,8 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/dataset_client.py` - DataSet client (`save_dataset()`, `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, plus the `get_datasets(ids)` batch fetch that avoids the annotation-listing N+1) with the `DataSetQuery` (`DS`) criterion helpers and the `data_block()` builder. `data_block()` is the only place `begin < end` is checked — the server does not - `src/dp_python_lib/client/annotations_client.py` - Annotations client (`save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, `delete_annotation()`, `get_calculations()`) with the `AnnotationQuery` (`AQ`) criterion helpers and the `calculations()` builder, which takes a `dict[str, DataFrame]` so frame-name uniqueness is true by construction. Note `AnnotationsClient` (feature client) vs `AnnotationClient` (facade) - `src/dp_python_lib/client/export_client.py` - Export client (`export_data()`) with the `ExportFormat` str enum and the `calculations_spec()` builder -- `src/dp_python_lib/client/time_conversions.py` - The two shared time converters and the `TimestampInput` alias: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`; naive datetimes raise) and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds), plus `NANOS_PER_SECOND`. Both were defined in `machine_config_client` as the first module to need them, and six others grew imports from there — which read as though datasets, queries, and DataFrames depended on the machine configuration API; `to_epoch_nanos()` had also been written out privately three separate times. **A leaf module**: it imports only stdlib and the generated protos, so any client module can use it without an import cycle. New time conversions belong here -- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Column names must be **non-blank**, not merely non-empty. Sample counts come from the right field per kind: `dataValues` for a `DataColumn`, `images` for an `ImageColumn` (which has no `values` field at all), and `len(values) / prod(dims)` for an array column. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through +- `src/dp_python_lib/client/time_conversions.py` - The two shared time converters and the `TimestampInput` alias: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`; naive datetimes raise). **The datetime path uses integer arithmetic, never `datetime.timestamp()`** — that returns a float64, which cannot hold present-day epoch seconds at sub-microsecond resolution and moved 99.7% of microsecond datetimes by up to ~119ns, breaking the exact-match contract sample status and provenance depend on and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds), plus `NANOS_PER_SECOND`. Both were defined in `machine_config_client` as the first module to need them, and six others grew imports from there — which read as though datasets, queries, and DataFrames depended on the machine configuration API; `to_epoch_nanos()` had also been written out privately three separate times. **A leaf module**: it imports only stdlib and the generated protos, so any client module can use it without an import cycle. New time conversions belong here +- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Column names must be **non-blank**, not merely non-empty, and `timestamp_count()` rejects a hand-built `SamplingClock` with a non-positive `periodNanos` as well as a zero count — the read path rejects both, and anything `data_frame()` accepts must be readable back. Sample counts come from the right field per kind: `dataValues` for a `DataColumn`, `images` for an `ImageColumn` (which has no `values` field at all), and `len(values) / prod(dims)` for an array column. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through - `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one flat list per sample, with the shape recoverable from `column_dimensions()` / `data_frame_column_dimensions()` so `[2,2]` and `[4]` stay distinguishable), `data_frame_columns()`, `column_metadata_dict()`. An axis that is set but empty is rejected rather than converted to a zero-row table. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap), and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`; reads each instant as `Timestamp.value` (**always nanoseconds**, unlike a raw int64 view, which is in the index's own storage unit — a `datetime64[us]` index viewed as int64 lands 1000× too early); rejects `NaT`, whose integer form is a valid-looking instant; and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type - `src/dp_python_lib/client/query_client.py` - v2 time-series query client (sample-oriented) exposed as `client.query`. Low-level wrappers `query_samples()` (unary, one resumable page) and `iter_query_samples()` (transparent paging), plus `iter_query_samples_stream()` (server-streaming, fire-and-consume, lazy). Queries are described by a kind-neutral `QueryParams` built from the `PvQuery` (`PV`) and `ConfigQuery` (`CFG`) criterion helpers; shares a `_build_query_spec()` seam so a future bucket request builder reuses it. Results wrap the raw `ColumnTable` (`.column_table`, `.next_page_token`); `.to_dataframe()`/`.to_numpy()` delegate to `query_conversions` (Phase 2, optional `[analysis]` extra) - `src/dp_python_lib/client/query_conversions.py` - Pythonic conversions for query results (optional `[analysis]` extra: pandas/numpy/openpyxl, imported lazily). `data_value_to_python()` (oneof extractor: scalars→native, timestamp→epoch-nanos, array→list, structure→dict, image→`Image` wrapper, fail-loud on unhandled arm), `column_table_to_dataframe()` (UTC datetime index + one column per DataColumn; dense-alignment and duplicate-column-name fail-loud; ColumnMetadata in `df.attrs`), `column_table_to_numpy()` (dict of 1-D arrays; complex arms stay 1-D object arrays rather than collapsing to 2-D), `dataframe_to_excel()` (thin `to_excel()` wrapper: row-limit guard, tz-drop, complex-cell stringification), and `query_samples_to_dataframe()`/`stream_query_samples_to_dataframes()` whole-query conveniences (unary concats by column name; streaming yields per-page frames lazily) diff --git a/doc/cookbook/datasets-and-annotations.md b/doc/cookbook/datasets-and-annotations.md index 1433faf..c5dec5e 100644 --- a/doc/cookbook/datasets-and-annotations.md +++ b/doc/cookbook/datasets-and-annotations.md @@ -246,8 +246,16 @@ result = client.annotation.annotations.save_annotation(SaveAnnotationRequestPara calculations=calculations({"orbit-rms": frame}), modified_by="cmcchesney", )) +if result.result_status.is_error: + raise RuntimeError(result.result_status.message) + +# No annotation_id was passed, so this SAVED A SECOND ANNOTATION rather than adding calculations to +# the one above. Rebind both handles: the rest of this recipe works with the annotation that owns +# the calculations. To attach them to the first annotation instead, pass annotation_id= and the +# fields to carry forward -- see "Updating without losing the calculations" below. +annotation_id = result.annotation_id calculations_id = result.calculations_id -assert calculations_id is not None +assert annotation_id is not None and calculations_id is not None ``` `calculations()` takes a **dict** of frame name to frame, which makes frame-name uniqueness true by @@ -483,7 +491,7 @@ if read.result_status.is_error: existing = read.annotation assert existing is not None # guaranteed once is_error is False -client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( +replaced = client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( name="Orbit drift during CXI_3443 (revised)", owner_id=existing.ownerId, dataset_ids=list(existing.dataSetIds), @@ -495,6 +503,13 @@ client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( annotation_id=annotation_id, modified_by="cmcchesney", )) +if replaced.result_status.is_error: + raise RuntimeError(replaced.result_status.message) + +# Carrying calculations through a replace stores a NEW object and deletes the old one, so the id +# you were holding is now dangling. Rebind it, or a later export/read will fail. +calculations_id = replaced.calculations_id +assert calculations_id is not None ``` A replace that *does* carry new calculations returns a **new** `calculations_id`; the previous diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index cbf6ce1..b015668 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -101,6 +101,27 @@ pre-seeded those names -- the recipe was broken only when read end to end, which is how a reader reads it. The preamble now documents that seeding a name cannot prove the recipe binds it. +- **Copilot second-pass review, 2026-09-10.** Four findings against `86621d1`, all real: + - **`to_timestamp()` lost sub-microsecond precision on every datetime.** The datetime branch routed through + `value.timestamp()`, a float64, which cannot hold present-day epoch seconds at that resolution: 99.7% of + microsecond-precision datetimes came back with a wrong nanosecond field, by up to ~119 ns. This is the exact + contract sample-status matching and provenance ranges rest on, and it predates this ticket -- the relocation + into `time_conversions.py` is simply what put it under review. Now integer arithmetic off the timedelta from + the epoch, which is lossless because a datetime's own resolution is exactly microseconds. The float/int epoch + seconds path is unchanged. + - `timestamp_count()` accepted a hand-built `SamplingClock` with `periodNanos == 0`, which + `expand_data_timestamps()` rejects -- the same write/read asymmetry class as the array-dims finding, and one + where every sample after the first would have carried the first sample's timestamp. + - The cookbook's calculations save omitted `annotation_id`, so it created a *second* annotation while the recipe + kept reading the first (calculation-free) one; every later `get_annotation()` read the wrong record. + - The cookbook's replace discarded its result, but a replace carrying calculations stores a new object and + deletes the old, so the `calculations_id` the later export used was dangling. + + The two cookbook findings share a root cause with the earlier `saved_id` one: snippets are type-checked in + isolation against a seeded preamble, which cannot see that a *sequence* of snippets is incoherent. Verified + this time by parsing the recipe as one continuous script and checking each handle is rebound before its next + use. + ## Overview Wrap the modernized DataSet / Annotation / Calculations / Export area of `DpAnnotationService` in the house diff --git a/src/dp_python_lib/client/data_frame.py b/src/dp_python_lib/client/data_frame.py index e9923f3..094a911 100644 --- a/src/dp_python_lib/client/data_frame.py +++ b/src/dp_python_lib/client/data_frame.py @@ -147,21 +147,29 @@ def timestamp_count(timestamps: common_pb2.DataTimestamps) -> int: """ Returns the number of timestamps a DataTimestamps describes, for validating parallel-array lengths. - An empty axis is rejected here rather than allowed to surface later as a confusing column-length mismatch. - The axis builders already make this unreachable -- sampling_clock() requires count >= 1 and timestamp_list() - requires a non-empty list -- but a hand-built DataTimestamps can still carry a zero-count SamplingClock or an - empty TimestampList, and both report their oneof arm as set. Rejecting them keeps this in step with - expand_data_timestamps(), which applies the same rule on the read path. + An empty or malformed axis is rejected here rather than allowed to surface later as a confusing column-length + mismatch. The axis builders already make this unreachable -- sampling_clock() requires count >= 1 and a + positive period, and timestamp_list() requires a non-empty list -- but a hand-built DataTimestamps can still + carry a zero-count or zero-period SamplingClock, or an empty TimestampList, and all report their oneof arm as + set. Rejecting them keeps this in step with expand_data_timestamps(), which applies the same rules on the + read path; anything data_frame() accepts must be readable back. :param timestamps: The time axis to measure. :return: The number of timestamps on the axis. - :raises ValueError: if neither axis form is set, or if the axis describes no timestamps. + :raises ValueError: if neither axis form is set, if the axis describes no timestamps, or if a SamplingClock's + periodNanos is not positive. """ axis = timestamps.WhichOneof("value") if axis == "samplingClock": count = timestamps.samplingClock.count if count < 1: raise ValueError(f"DataTimestamps samplingClock requires count >= 1, got {count}") + period = timestamps.samplingClock.periodNanos + if period <= 0: + # Checked here as well as in sampling_clock(), which a hand-built axis bypasses. The read path + # (expand_data_timestamps) rejects a non-positive period, so accepting one here would build a frame + # the library cannot read back -- and every sample after the first would share the first's timestamp. + raise ValueError(f"DataTimestamps samplingClock requires periodNanos > 0, got {period}") return count if axis == "timestampList": count = len(timestamps.timestampList.timestamps) diff --git a/src/dp_python_lib/client/time_conversions.py b/src/dp_python_lib/client/time_conversions.py index 5b43152..249ed75 100644 --- a/src/dp_python_lib/client/time_conversions.py +++ b/src/dp_python_lib/client/time_conversions.py @@ -19,7 +19,7 @@ """ import math -from datetime import datetime +from datetime import datetime, timezone from dp_python_lib.grpc import common_pb2 @@ -30,6 +30,12 @@ NANOS_PER_SECOND = 1_000_000_000 +_NANOS_PER_MICROSECOND = 1_000 +_SECONDS_PER_DAY = 86_400 +# The reference instant for the datetime -> Timestamp conversion. Subtracting two aware datetimes yields a +# timedelta whose days/seconds/microseconds are exact integers, which is what keeps that path lossless. +_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) + def to_epoch_nanos(timestamp: common_pb2.Timestamp) -> int: """ @@ -69,8 +75,22 @@ def to_timestamp(value: TimestampInput) -> common_pb2.Timestamp: "to_timestamp() requires a timezone-aware datetime; naive datetimes are rejected " "to avoid silent local-timezone bugs. Use datetime.now(timezone.utc) or attach tzinfo." ) - epoch = value.timestamp() - return to_timestamp(epoch) + # Integer arithmetic, NOT value.timestamp(): that returns a float64, which cannot hold present-day epoch + # seconds at sub-microsecond resolution, so the fractional part comes back slightly wrong. It moved 99.7% + # of microsecond-precision datetimes by up to ~119 ns -- fatal for an API whose sample-status matching and + # provenance ranges are exact at nanosecond precision. A datetime's own resolution is exactly + # microseconds, so converting through its integer fields is lossless. + delta = value - _EPOCH + epoch_seconds = delta.days * _SECONDS_PER_DAY + delta.seconds + if epoch_seconds < 0: + raise ValueError( + f"to_timestamp() cannot represent {value.isoformat()}: common.Timestamp.epochSeconds is unsigned, " + f"so instants before 1970-01-01T00:00:00Z have no representation." + ) + timestamp = common_pb2.Timestamp() + timestamp.epochSeconds = epoch_seconds + timestamp.nanoseconds = delta.microseconds * _NANOS_PER_MICROSECOND + return timestamp if isinstance(value, bool): # bool is a subclass of int; reject it explicitly as it is virtually always a mistake. diff --git a/tests/unit/test_data_frame.py b/tests/unit/test_data_frame.py index 4f9835c..bc8deda 100644 --- a/tests/unit/test_data_frame.py +++ b/tests/unit/test_data_frame.py @@ -336,6 +336,18 @@ def test_rejects_unsupported_column_type(self): dfb.data_frame(_axis(1), ["not a column"]) self.assertIn("unsupported column type", str(ctx.exception)) + def test_rejects_zero_period_sampling_clock(self): + # sampling_clock() makes this unreachable, but a hand-built axis bypasses it. expand_data_timestamps() + # rejects a non-positive period on the read side, so accepting it here would build an unreadable frame -- + # and every sample after the first would carry the first one's timestamp. + axis = common_pb2.DataTimestamps() + axis.samplingClock.startTime.epochSeconds = 1_700_000_000 + axis.samplingClock.periodNanos = 0 + axis.samplingClock.count = 3 + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0, 3.0])]) + self.assertIn("periodNanos > 0", str(ctx.exception)) + def test_rejects_empty_axis(self): with self.assertRaises(ValueError): dfb.data_frame(common_pb2.DataTimestamps(), [dfb.double_column("d", [1.0])]) diff --git a/tests/unit/test_time_conversions.py b/tests/unit/test_time_conversions.py index a8df038..ef89788 100644 --- a/tests/unit/test_time_conversions.py +++ b/tests/unit/test_time_conversions.py @@ -31,6 +31,48 @@ def test_zero_timestamp_is_zero(self): self.assertEqual(to_epoch_nanos(common_pb2.Timestamp()), 0) +class TestDatetimeConversionIsExact(unittest.TestCase): + """ + A datetime must not route through datetime.timestamp(). + + That returns a float64, which cannot hold present-day epoch seconds at sub-microsecond resolution, so the + fractional part comes back slightly wrong -- it moved 99.7% of microsecond-precision datetimes by up to ~119ns. + Sample-status matching and provenance ranges are exact at nanosecond precision, so that is a correctness bug, + not a rounding nicety. + """ + + def test_microsecond_datetimes_convert_exactly(self): + for microsecond in (1, 999, 123_456, 250_000, 999_999): + with self.subTest(microsecond=microsecond): + dt = datetime(2026, 7, 14, 18, 0, 0, microsecond, tzinfo=timezone.utc) + self.assertEqual(to_timestamp(dt).nanoseconds, microsecond * 1_000) + + def test_every_microsecond_of_a_second_is_exact(self): + # Exhaustive over the fractional field at a present-day instant: the float path failed almost all of these. + base = datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc) + wrong = [ + microsecond + for microsecond in range(0, 1_000_000, 977) # a prime stride, ~1024 samples across the range + if to_timestamp(base.replace(microsecond=microsecond)).nanoseconds != microsecond * 1_000 + ] + self.assertEqual(wrong, []) + + def test_round_trips_through_to_epoch_nanos(self): + dt = datetime(2026, 7, 14, 18, 0, 0, 123_456, tzinfo=timezone.utc) + expected = int(datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc).timestamp()) * 1_000_000_000 + self.assertEqual(to_epoch_nanos(to_timestamp(dt)), expected + 123_456_000) + + def test_non_utc_offset_yields_the_same_instant(self): + aware = datetime(2026, 7, 14, 13, 0, 0, 123_456, tzinfo=timezone(timedelta(hours=-5))) + utc = datetime(2026, 7, 14, 18, 0, 0, 123_456, tzinfo=timezone.utc) + self.assertEqual(to_epoch_nanos(to_timestamp(aware)), to_epoch_nanos(to_timestamp(utc))) + + def test_epoch_and_pre_epoch(self): + self.assertEqual(to_epoch_nanos(to_timestamp(datetime(1970, 1, 1, tzinfo=timezone.utc))), 0) + with self.assertRaises(ValueError): + to_timestamp(datetime(1969, 12, 31, 23, 59, 59, tzinfo=timezone.utc)) + + class TestToTimestamp(unittest.TestCase): """Unit tests for the to_timestamp() conversion helper.""" From a29064d64722b14e0b46609024931a86f6141fa0 Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Thu, 10 Sep 2026 13:39:40 -0600 Subject: [PATCH 09/11] fix: preserve dtypes and metadata through the pandas round trip (issue #6) Three findings from Copilot's third pass over 70d2102, all real. The pandas round trip discarded every column's metadata. data_frame_to_pandas() parked each column's ColumnMetadata in df.attrs["column_metadata"], and data_frame_from_pandas() ignored it -- so a frame that went out through pandas came back with its tags, attributes, and provenance silently stripped. That is the record of where the numbers came from, which is the reason calculations are worth storing and a headline feature of this ticket; carrying the attrs and then dropping them was worse than never carrying them. Added column_metadata_from_dict(), the inverse of column_metadata_dict(), and threaded it through _column_from_series(). Provenance, both origin arms, and time ranges all survive. The pandas round trip also widened dtypes. Building the frame from untyped Python lists let pandas infer float64 for a FloatColumn and int64 for an Int32Column, so converting back emitted DoubleColumn/Int64Column and the frame changed shape on a trip it should have survived. Each Series is now constructed with the dtype its protobuf column type implies; kinds with no narrow equivalent (strings, arrays, images, DataColumn) are still left to inference. Together these make the round trip lossless: a frame now converts to pandas and back to BYTE EQUALITY, apart from the deliberate SamplingClock -> TimestampList axis change, which is the documented refusal to infer a clock from an index that merely looks regular. Tests pin both the equality and that deliberate exception, and a first attempt at the equality test failed by asserting the axis was preserved -- worth recording, since that is the one thing the round trip is designed not to do. Third: the recipe claimed "Verified against dp-grpc rel-1.16.0", a tag that does not exist -- the newest release is rel-1.15.0. It now separates the target API version (1.16.0, unreleased) from what was actually tested (a dp-service build from main at fddf692). doc/cookbook/sample-status.md carries the same overclaim; left alone as it is not this PR's file. 749 tests pass (707 unit, 42 integration + subtests against the live server; 637 pass and 70 skip without the [analysis] extra); ruff clean; 103 cookbook snippets check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- CLAUDE.md | 2 +- doc/cookbook/datasets-and-annotations.md | 13 +- plan/tickets/6/plan.md | 17 +++ .../client/data_frame_conversions.py | 134 ++++++++++++++++-- tests/unit/test_data_frame_conversions.py | 118 +++++++++++++++ 5 files changed, 269 insertions(+), 15 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c208bdd..77393e7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -158,7 +158,7 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/export_client.py` - Export client (`export_data()`) with the `ExportFormat` str enum and the `calculations_spec()` builder - `src/dp_python_lib/client/time_conversions.py` - The two shared time converters and the `TimestampInput` alias: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`; naive datetimes raise). **The datetime path uses integer arithmetic, never `datetime.timestamp()`** — that returns a float64, which cannot hold present-day epoch seconds at sub-microsecond resolution and moved 99.7% of microsecond datetimes by up to ~119ns, breaking the exact-match contract sample status and provenance depend on and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds), plus `NANOS_PER_SECOND`. Both were defined in `machine_config_client` as the first module to need them, and six others grew imports from there — which read as though datasets, queries, and DataFrames depended on the machine configuration API; `to_epoch_nanos()` had also been written out privately three separate times. **A leaf module**: it imports only stdlib and the generated protos, so any client module can use it without an import cycle. New time conversions belong here - `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Column names must be **non-blank**, not merely non-empty, and `timestamp_count()` rejects a hand-built `SamplingClock` with a non-positive `periodNanos` as well as a zero count — the read path rejects both, and anything `data_frame()` accepts must be readable back. Sample counts come from the right field per kind: `dataValues` for a `DataColumn`, `images` for an `ImageColumn` (which has no `values` field at all), and `len(values) / prod(dims)` for an array column. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through -- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one flat list per sample, with the shape recoverable from `column_dimensions()` / `data_frame_column_dimensions()` so `[2,2]` and `[4]` stay distinguishable), `data_frame_columns()`, `column_metadata_dict()`. An axis that is set but empty is rejected rather than converted to a zero-row table. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap), and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`; reads each instant as `Timestamp.value` (**always nanoseconds**, unlike a raw int64 view, which is in the index's own storage unit — a `datetime64[us]` index viewed as int64 lands 1000× too early); rejects `NaT`, whose integer form is a valid-looking instant; and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type +- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one flat list per sample, with the shape recoverable from `column_dimensions()` / `data_frame_column_dimensions()` so `[2,2]` and `[4]` stay distinguishable), `data_frame_columns()`, `column_metadata_dict()`. An axis that is set but empty is rejected rather than converted to a zero-row table. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`; each Series carries the **narrow dtype its column type implies** — `float32`/`int32`, not pandas' widened inference), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap; rebuilds each column's `ColumnMetadata` from `df.attrs` via `column_metadata_from_dict()`). **A frame round-trips through pandas to byte equality** apart from the deliberate `SamplingClock`→`TimestampList` axis change: column types and provenance both survive, and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`; reads each instant as `Timestamp.value` (**always nanoseconds**, unlike a raw int64 view, which is in the index's own storage unit — a `datetime64[us]` index viewed as int64 lands 1000× too early); rejects `NaT`, whose integer form is a valid-looking instant; and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type - `src/dp_python_lib/client/query_client.py` - v2 time-series query client (sample-oriented) exposed as `client.query`. Low-level wrappers `query_samples()` (unary, one resumable page) and `iter_query_samples()` (transparent paging), plus `iter_query_samples_stream()` (server-streaming, fire-and-consume, lazy). Queries are described by a kind-neutral `QueryParams` built from the `PvQuery` (`PV`) and `ConfigQuery` (`CFG`) criterion helpers; shares a `_build_query_spec()` seam so a future bucket request builder reuses it. Results wrap the raw `ColumnTable` (`.column_table`, `.next_page_token`); `.to_dataframe()`/`.to_numpy()` delegate to `query_conversions` (Phase 2, optional `[analysis]` extra) - `src/dp_python_lib/client/query_conversions.py` - Pythonic conversions for query results (optional `[analysis]` extra: pandas/numpy/openpyxl, imported lazily). `data_value_to_python()` (oneof extractor: scalars→native, timestamp→epoch-nanos, array→list, structure→dict, image→`Image` wrapper, fail-loud on unhandled arm), `column_table_to_dataframe()` (UTC datetime index + one column per DataColumn; dense-alignment and duplicate-column-name fail-loud; ColumnMetadata in `df.attrs`), `column_table_to_numpy()` (dict of 1-D arrays; complex arms stay 1-D object arrays rather than collapsing to 2-D), `dataframe_to_excel()` (thin `to_excel()` wrapper: row-limit guard, tz-drop, complex-cell stringification), and `query_samples_to_dataframe()`/`stream_query_samples_to_dataframes()` whole-query conveniences (unary concats by column name; streaming yields per-page frames lazily) - `src/dp_python_lib/client/service_api_client_base.py` - Base class for the service clients: owns the channel and the one-per-client gRPC stub, and provides `_dispatch()`, the shared three-tier sender that all 18 unary `_send_*` methods delegate to diff --git a/doc/cookbook/datasets-and-annotations.md b/doc/cookbook/datasets-and-annotations.md index c5dec5e..024cf60 100644 --- a/doc/cookbook/datasets-and-annotations.md +++ b/doc/cookbook/datasets-and-annotations.md @@ -3,9 +3,11 @@ Naming a region of the archive so you can find it again, attaching derived values to it with a record of what they were computed from, and exporting the result. -> **Verified against:** dp-grpc `rel-1.16.0`. -> The modernized DataSet / Annotation / Export API is **new in 1.16.0** and will not work against a -> `rel-1.15.0` server, which answers these calls with `UNIMPLEMENTED`. +> **Target API version:** dp-grpc 1.16.0, which is **not yet released** — the newest tag is +> `rel-1.15.0`. The modernized DataSet / Annotation / Export API is new in 1.16.0 and will not work +> against a `rel-1.15.0` server, which answers these calls with `UNIMPLEMENTED`. +> +> **Verified against:** a dp-service build from `main` at commit `fddf692`, carrying the 1.16.0 API. See [API conventions](conventions.md) for result checking, paging, and time handling. @@ -618,8 +620,9 @@ calculations, are left dangling. ### How far these examples have been verified -The dataset and annotation lifecycle **has** been exercised against a live 1.16.0 Annotation -Service by `tests/integration/test_datasets_annotations_integration.py`: save/get round trips, +The dataset and annotation lifecycle **has** been exercised against a live Annotation Service built +from dp-service `main` at `fddf692` (the 1.16.0 API, pre-release) by +`tests/integration/test_datasets_annotations_integration.py`: save/get round trips, every criterion including the key-only attribute search, lowercase tag normalization, paging, rejected page tokens, the `get_datasets()` batch fetch, calculations inline on `get_annotation()` versus id-only on `query_annotations()`, the full-replace clearing behavior, the delete cascade, and diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index b015668..b91ba77 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -122,6 +122,23 @@ this time by parsing the recipe as one continuous script and checking each handle is rebound before its next use. +- **Copilot third-pass review, 2026-09-10.** Three findings against `70d2102`, delivered as summary-level + "previously missed" notes rather than inline comments. All three were real: + - **The pandas round trip widened dtypes.** `data_frame_to_pandas()` handed pandas untyped Python lists, so a + `FloatColumn` came back as `float64` and an `Int32Column` as `int64`; converting back then emitted + `DoubleColumn`/`Int64Column`. Each Series is now built with the dtype its column type implies. + - **The pandas round trip discarded all column metadata.** `data_frame_to_pandas()` populated + `df.attrs["column_metadata"]` and `data_frame_from_pandas()` ignored it, so tags, attributes, and provenance + were silently stripped -- losing exactly the record of where the numbers came from that makes calculations + worth storing, and which this ticket sells as a headline feature. Added `column_metadata_from_dict()`, the + inverse of `column_metadata_dict()`, and threaded it through. A frame now round-trips to **byte equality** + apart from the deliberate `SamplingClock` -> `TimestampList` axis change. + - The recipe claimed to be "Verified against dp-grpc `rel-1.16.0`", a tag that does not exist -- the newest + release is `rel-1.15.0`. It now separates the *target* API version (1.16.0, unreleased) from what was + actually tested (a dp-service build from `main` at `fddf692`). `doc/cookbook/sample-status.md` carries the + same overclaim and is left alone here: it is not this PR's file, and correcting it belongs with whoever + reconciles the cookbook's version banners at release time. + ## Overview Wrap the modernized DataSet / Annotation / Calculations / Export area of `DpAnnotationService` in the house diff --git a/src/dp_python_lib/client/data_frame_conversions.py b/src/dp_python_lib/client/data_frame_conversions.py index cc980fc..83ac0ba 100644 --- a/src/dp_python_lib/client/data_frame_conversions.py +++ b/src/dp_python_lib/client/data_frame_conversions.py @@ -309,6 +309,34 @@ def _check_frame_alignment(frame: common_pb2.DataFrame, columns: dict[str, list] ) +# The protobuf column types whose pandas dtype is narrower than what inference from a plain Python list yields. +# Double/Int64/Bool are listed too so the mapping is explicit rather than half-stated. +_NARROW_DTYPE_BY_COLUMN_TYPE = { + common_pb2.FloatColumn: "float32", + common_pb2.Int32Column: "int32", + common_pb2.DoubleColumn: "float64", + common_pb2.Int64Column: "int64", + common_pb2.BoolColumn: "bool", +} + + +def _narrow_column_dtypes(frame: common_pb2.DataFrame) -> dict[str, str]: + """ + Maps each scalar column's name to the pandas dtype its protobuf type implies. + + Only the columns whose type pins a dtype appear; strings, arrays, images, and DataColumns are absent, so the + caller leaves those to pandas' inference. + + :param frame: The frame whose columns to inspect. + :return: A dict of column name -> pandas dtype string. + """ + return { + column.name: _NARROW_DTYPE_BY_COLUMN_TYPE[type(column)] + for column in iter_frame_columns(frame) + if type(column) in _NARROW_DTYPE_BY_COLUMN_TYPE + } + + def data_frame_to_pandas(frame: common_pb2.DataFrame, exclude_column_metadata: bool = False) -> Any: """ Converts a DataFrame into a pandas DataFrame with a UTC DatetimeIndex. @@ -338,7 +366,19 @@ def data_frame_to_pandas(frame: common_pb2.DataFrame, exclude_column_metadata: b _check_frame_alignment(frame, columns, len(epoch_nanos)) index = pd.DatetimeIndex(pd.to_datetime(pd.Series(epoch_nanos, dtype="int64"), unit="ns", utc=True)) - df = pd.DataFrame(columns, index=index) + + # Build each Series with the dtype its protobuf column type implies. Handing pandas an untyped list widens + # FloatColumn to float64 and Int32Column to int64, so a frame -> pandas -> frame round trip would come back + # as DoubleColumn/Int64Column. Columns with no narrow equivalent (strings, arrays, images, DataColumn) are + # left to pandas' own inference. + narrow_dtypes = _narrow_column_dtypes(frame) + df = pd.DataFrame( + { + name: pd.Series(values, index=index, dtype=narrow_dtypes[name]) if name in narrow_dtypes else values + for name, values in columns.items() + }, + index=index, + ) if not exclude_column_metadata: df.attrs["column_metadata"] = { @@ -408,7 +448,72 @@ def _timestamps_from_index(index: Any) -> common_pb2.DataTimestamps: return timestamps -def _column_from_series(name: str, series: Any) -> Any: +def _timestamp_from_nanos(epoch_nanos: int) -> common_pb2.Timestamp: + """ + Builds a common.Timestamp from integer epoch nanoseconds -- the inverse of to_epoch_nanos(). + + :param epoch_nanos: Epoch nanoseconds. + :return: The equivalent common.Timestamp. + """ + timestamp = common_pb2.Timestamp() + timestamp.epochSeconds, timestamp.nanoseconds = divmod(epoch_nanos, 1_000_000_000) + return timestamp + + +def column_metadata_from_dict(summary: dict[str, Any] | None) -> common_pb2.ColumnMetadata | None: + """ + Rebuilds a ColumnMetadata from the dict column_metadata_dict() produced -- the inverse of that function. + + This is what lets a frame survive a pandas round trip with its provenance intact: data_frame_to_pandas() parks + each column's metadata in df.attrs["column_metadata"], and data_frame_from_pandas() feeds it back through + here. Without it the attrs were carried and then silently dropped, losing exactly the record of where the + numbers came from that makes calculations worth storing. + + :param summary: A dict as returned by column_metadata_dict(), or None. + :return: The equivalent ColumnMetadata, or None when the summary is absent or carries nothing. + """ + if not summary: + return None + + tags = summary.get("tags") or [] + attributes = summary.get("attributes") or {} + provenance_summary = summary.get("provenance") + if not tags and not attributes and not provenance_summary: + return None + + metadata = common_pb2.ColumnMetadata() + if tags: + metadata.tags[:] = list(tags) + for name, value in attributes.items(): + attribute = metadata.attributes.add() + attribute.name = name + attribute.value = value + + if provenance_summary: + provenance = metadata.provenance + if provenance_summary.get("source"): + provenance.source = provenance_summary["source"] + if provenance_summary.get("process"): + provenance.process = provenance_summary["process"] + for entry in provenance_summary.get("derived_from") or []: + source = provenance.derivedFrom.add() + if "pv_name" in entry: + source.pvName = entry["pv_name"] + elif "calculations_column" in entry: + column = entry["calculations_column"] + source.calculationsColumn.calculationsId = column["calculations_id"] + source.calculationsColumn.frameName = column["frame_name"] + source.calculationsColumn.columnName = column["column_name"] + time_range = entry.get("time_range") + if time_range is not None: + begin_nanos, end_nanos = time_range + source.timeRange.beginTime.CopyFrom(_timestamp_from_nanos(begin_nanos)) + source.timeRange.endTime.CopyFrom(_timestamp_from_nanos(end_nanos)) + + return metadata + + +def _column_from_series(name: str, series: Any, metadata: common_pb2.ColumnMetadata | None = None) -> Any: """ Builds the typed column matching a pandas Series' dtype. @@ -420,6 +525,7 @@ def _column_from_series(name: str, series: Any) -> Any: :param name: The column's name. :param series: The pandas Series holding its values. + :param metadata: Optional ColumnMetadata to attach (see column_metadata_from_dict()). :return: The matching typed column message. :raises ValueError: if the dtype has no mapping, or the series contains a missing value. """ @@ -438,15 +544,15 @@ def _column_from_series(name: str, series: Any) -> Any: dtype_name = str(dtype) if dtype_name == "float64": - return builders.double_column(name, [float(v) for v in series]) + return builders.double_column(name, [float(v) for v in series], metadata=metadata) if dtype_name == "float32": - return builders.float_column(name, [float(v) for v in series]) + return builders.float_column(name, [float(v) for v in series], metadata=metadata) if dtype_name in ("int64", "Int64"): - return builders.int64_column(name, [int(v) for v in series]) + return builders.int64_column(name, [int(v) for v in series], metadata=metadata) if dtype_name in ("int32", "Int32"): - return builders.int32_column(name, [int(v) for v in series]) + return builders.int32_column(name, [int(v) for v in series], metadata=metadata) if dtype_name in ("bool", "boolean"): - return builders.bool_column(name, [bool(v) for v in series]) + return builders.bool_column(name, [bool(v) for v in series], metadata=metadata) if dtype_name in ("object", "string", "str") or dtype_name.startswith("string"): values = list(series) if not all(isinstance(value, str) for value in values): @@ -459,7 +565,7 @@ def _column_from_series(name: str, series: Any) -> Any: f"object columns are mapped to StringColumn, so convert the values first or build the column " f"explicitly with a data_frame builder" ) - return builders.string_column(name, values) + return builders.string_column(name, values, metadata=metadata) raise ValueError( f"column '{name}' has dtype {dtype_name}, which has no typed-column mapping. Supported dtypes are " @@ -477,6 +583,9 @@ def data_frame_from_pandas(df: Any) -> common_pb2.DataFrame: The index becomes an explicit TimestampList; see _timestamps_from_index() for why a SamplingClock is never inferred. Missing values are rejected fail-loud, since a dense typed column cannot express a gap. + Per-column metadata is read back from df.attrs["column_metadata"] when present, so a frame that went out + through data_frame_to_pandas() returns with its tags, attributes, and provenance intact. + Converting back with data_frame_to_pandas() preserves every value, dtype, and timestamp, but can return the columns grouped by type rather than in their original order -- see that function's docstring. @@ -505,8 +614,15 @@ def data_frame_from_pandas(df: Any) -> common_pb2.DataFrame: f"a merge with overlapping names)." ) + # data_frame_to_pandas() parks each column's ColumnMetadata here; carrying it back is what keeps provenance + # alive across a round trip. A frame built by hand simply has no attrs, and every column gets None. + metadata_by_column = df.attrs.get("column_metadata") or {} + data_timestamps = _timestamps_from_index(df.index) - columns = [_column_from_series(str(name), df[name]) for name in df.columns] + columns = [ + _column_from_series(str(name), df[name], metadata=column_metadata_from_dict(metadata_by_column.get(str(name)))) + for name in df.columns + ] return builders.data_frame(data_timestamps, columns) diff --git a/tests/unit/test_data_frame_conversions.py b/tests/unit/test_data_frame_conversions.py index c8dc7cd..12cf5e3 100644 --- a/tests/unit/test_data_frame_conversions.py +++ b/tests/unit/test_data_frame_conversions.py @@ -602,5 +602,123 @@ def test_nat_in_the_index_is_rejected(self): self.assertIn("NaT", str(ctx.exception)) +def _columns_of(frame): + """Yields a frame's columns, for rebuilding it on a different time axis.""" + return dfc.iter_frame_columns(frame) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestPandasRoundTripPreservesTypesAndMetadata(unittest.TestCase): + """A frame that goes out through pandas must come back as the same frame, not a widened, stripped one.""" + + def _metadata(self): + return dfb.column_metadata( + tags=["derived"], + attributes={"unit": "mm"}, + provenance=dfb.provenance( + source="rig", + process="1 Hz RMS", + derived_from=[ + dfb.pv_source("BPMS:GUNB:314:X", (T0, T1)), + dfb.calculations_source("cid", "orbit-rms", "x_rms"), + ], + ), + ) + + def _frame(self): + return dfb.data_frame( + _axis(2), + [ + dfb.float_column("f", [1.5, 2.5], metadata=self._metadata()), + dfb.int32_column("i32", [1, 2]), + dfb.int64_column("i64", [3, 4]), + dfb.double_column("d", [1.0, 2.0]), + dfb.bool_column("b", [True, False]), + dfb.string_column("s", ["a", "b"]), + ], + ) + + def test_narrow_dtypes_reach_pandas(self): + # An untyped list would let pandas widen float32 to float64 and int32 to int64. + df = dfc.data_frame_to_pandas(self._frame()) + self.assertEqual(str(df["f"].dtype), "float32") + self.assertEqual(str(df["i32"].dtype), "int32") + self.assertEqual(str(df["i64"].dtype), "int64") + self.assertEqual(str(df["d"].dtype), "float64") + self.assertEqual(str(df["b"].dtype), "bool") + + def test_column_types_survive_the_round_trip(self): + original = self._frame() + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + + def kinds(frame): + return { + column.name: field + for field in ( + "doubleColumns", + "floatColumns", + "int64Columns", + "int32Columns", + "boolColumns", + "stringColumns", + ) + for column in getattr(frame, field) + } + + self.assertEqual(kinds(restored), kinds(original)) + + def test_metadata_and_provenance_survive_the_round_trip(self): + original = self._frame() + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + self.assertEqual(restored.floatColumns[0].metadata, original.floatColumns[0].metadata) + + def test_whole_frame_round_trips_apart_from_the_deliberate_axis_change(self): + # Everything except the axis form comes back byte for byte. The axis is deliberately NOT preserved: a + # SamplingClock becomes an explicit TimestampList, because inferring a clock back from an index that + # merely looks regular would quietly move timestamps. The instants themselves are identical. + original = dfb.data_frame(dfb.timestamp_list([T0, T1]), list(_columns_of(self._frame()))) + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + self.assertEqual(restored, original) + + def test_sampling_clock_axis_becomes_a_timestamp_list_with_the_same_instants(self): + original = self._frame() # a SamplingClock axis + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + + self.assertEqual(original.dataTimestamps.WhichOneof("value"), "samplingClock") + self.assertEqual(restored.dataTimestamps.WhichOneof("value"), "timestampList") + self.assertEqual(dfc.data_frame_timestamps(restored), dfc.data_frame_timestamps(original)) + + def test_excluded_metadata_is_simply_absent(self): + # With exclude_column_metadata=True there are no attrs to carry, and the columns come back bare. + original = self._frame() + df = dfc.data_frame_to_pandas(original, exclude_column_metadata=True) + restored = dfc.data_frame_from_pandas(df) + self.assertFalse(restored.floatColumns[0].HasField("metadata")) + + def test_hand_built_frame_without_attrs_still_converts(self): + import pandas as pd + + index = pd.DatetimeIndex(pd.to_datetime([T0_NANOS, T0_NANOS + 1_000_000_000], unit="ns", utc=True)) + df = pd.DataFrame({"d": [1.0, 2.0]}, index=index) + frame = dfc.data_frame_from_pandas(df) + self.assertFalse(frame.doubleColumns[0].HasField("metadata")) + + +class TestColumnMetadataFromDict(unittest.TestCase): + def test_none_and_empty_summaries_yield_none(self): + self.assertIsNone(dfc.column_metadata_from_dict(None)) + self.assertIsNone(dfc.column_metadata_from_dict({})) + self.assertIsNone(dfc.column_metadata_from_dict({"tags": [], "attributes": {}, "provenance": None})) + + def test_inverts_column_metadata_dict(self): + original = dfb.column_metadata( + tags=["a"], + attributes={"k": "v"}, + provenance=dfb.provenance(process="p", derived_from=[dfb.pv_source("PV", (T0, T1))]), + ) + column = dfb.double_column("d", [1.0], metadata=original) + self.assertEqual(dfc.column_metadata_from_dict(dfc.column_metadata_dict(column)), original) + + if __name__ == "__main__": unittest.main() From 94986fd0b573fb034f9d38423390c84a41f63556 Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Thu, 10 Sep 2026 14:05:49 -0600 Subject: [PATCH 10/11] fix: preserve enum columns through the pandas round trip (issue #6) Three findings from Copilot's fourth pass over a29064d. EnumColumn was the one column kind the previous commit's dtype work missed. Its values field is int32, so leaving it out of the narrow-dtype mapping widened the codes to int64 and rebuilt the column as an Int64Column -- and the enumId went with it, which is worse than the widening: the codes are bare integers, and the id is the only thing that says which enumeration names them. A round trip turned a labeled enum into anonymous numbers. The dtype mapping now covers EnumColumn, and the id rides in df.attrs["enum_ids"]. That is carried even under exclude_column_metadata=True, because unlike tags and provenance it is structural: without it the column cannot be rebuilt as an enum at all. _column_from_series() consults it ahead of EVERY dtype branch, not just the integer ones -- an enum id on a float or string column is a caller error worth naming rather than quietly producing the wrong column kind. Enum columns now round-trip to full frame equality, and a plain Int32Column sitting beside one is unaffected. The other two were my own stale references. Comments in query_conversions and sample_status_conversions still pointed at machine_config_client.to_epoch_nanos, which I wrote in 9e07226 and then invalidated in 4c6a8e1 by moving the function to time_conversions.py without updating the prose. Sweeping for the same mistake turned up two more the review had not flagged: a stale module attribution in this plan's own reference list, and a sentence in CLAUDE.md that an earlier edit had mangled mid-clause, leaving two statements spliced together. All four corrected. 754 tests pass (712 unit, 42 integration + subtests against the live server; 637 pass and 75 skip without the [analysis] extra); ruff clean; 103 cookbook snippets check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- CLAUDE.md | 4 +- plan/tickets/6/plan.md | 15 ++++- .../client/data_frame_conversions.py | 53 ++++++++++++++++- src/dp_python_lib/client/query_conversions.py | 2 +- .../client/sample_status_conversions.py | 2 +- tests/unit/test_data_frame_conversions.py | 57 +++++++++++++++++++ 6 files changed, 125 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 77393e7..a15604b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,9 +156,9 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/dataset_client.py` - DataSet client (`save_dataset()`, `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, plus the `get_datasets(ids)` batch fetch that avoids the annotation-listing N+1) with the `DataSetQuery` (`DS`) criterion helpers and the `data_block()` builder. `data_block()` is the only place `begin < end` is checked — the server does not - `src/dp_python_lib/client/annotations_client.py` - Annotations client (`save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, `delete_annotation()`, `get_calculations()`) with the `AnnotationQuery` (`AQ`) criterion helpers and the `calculations()` builder, which takes a `dict[str, DataFrame]` so frame-name uniqueness is true by construction. Note `AnnotationsClient` (feature client) vs `AnnotationClient` (facade) - `src/dp_python_lib/client/export_client.py` - Export client (`export_data()`) with the `ExportFormat` str enum and the `calculations_spec()` builder -- `src/dp_python_lib/client/time_conversions.py` - The two shared time converters and the `TimestampInput` alias: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`; naive datetimes raise). **The datetime path uses integer arithmetic, never `datetime.timestamp()`** — that returns a float64, which cannot hold present-day epoch seconds at sub-microsecond resolution and moved 99.7% of microsecond datetimes by up to ~119ns, breaking the exact-match contract sample status and provenance depend on and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds), plus `NANOS_PER_SECOND`. Both were defined in `machine_config_client` as the first module to need them, and six others grew imports from there — which read as though datasets, queries, and DataFrames depended on the machine configuration API; `to_epoch_nanos()` had also been written out privately three separate times. **A leaf module**: it imports only stdlib and the generated protos, so any client module can use it without an import cycle. New time conversions belong here +- `src/dp_python_lib/client/time_conversions.py` - The two shared time converters and the `TimestampInput` alias: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`; naive datetimes raise) and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds), plus `NANOS_PER_SECOND`. **The datetime path uses integer arithmetic, never `datetime.timestamp()`** — that returns a float64, which cannot hold present-day epoch seconds at sub-microsecond resolution and moved 99.7% of microsecond datetimes by up to ~119ns, breaking the exact-match contract sample status and provenance depend on. Both were defined in `machine_config_client` as the first module to need them, and six others grew imports from there — which read as though datasets, queries, and DataFrames depended on the machine configuration API; `to_epoch_nanos()` had also been written out privately three separate times. **A leaf module**: it imports only stdlib and the generated protos, so any client module can use it without an import cycle. New time conversions belong here - `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Column names must be **non-blank**, not merely non-empty, and `timestamp_count()` rejects a hand-built `SamplingClock` with a non-positive `periodNanos` as well as a zero count — the read path rejects both, and anything `data_frame()` accepts must be readable back. Sample counts come from the right field per kind: `dataValues` for a `DataColumn`, `images` for an `ImageColumn` (which has no `values` field at all), and `len(values) / prod(dims)` for an array column. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through -- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one flat list per sample, with the shape recoverable from `column_dimensions()` / `data_frame_column_dimensions()` so `[2,2]` and `[4]` stay distinguishable), `data_frame_columns()`, `column_metadata_dict()`. An axis that is set but empty is rejected rather than converted to a zero-row table. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`; each Series carries the **narrow dtype its column type implies** — `float32`/`int32`, not pandas' widened inference), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap; rebuilds each column's `ColumnMetadata` from `df.attrs` via `column_metadata_from_dict()`). **A frame round-trips through pandas to byte equality** apart from the deliberate `SamplingClock`→`TimestampList` axis change: column types and provenance both survive, and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`; reads each instant as `Timestamp.value` (**always nanoseconds**, unlike a raw int64 view, which is in the index's own storage unit — a `datetime64[us]` index viewed as int64 lands 1000× too early); rejects `NaT`, whose integer form is a valid-looking instant; and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type +- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one flat list per sample, with the shape recoverable from `column_dimensions()` / `data_frame_column_dimensions()` so `[2,2]` and `[4]` stay distinguishable), `data_frame_columns()`, `column_metadata_dict()`. An axis that is set but empty is rejected rather than converted to a zero-row table. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`; each Series carries the **narrow dtype its column type implies** — `float32`/`int32`, not pandas' widened inference), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap; rebuilds each column's `ColumnMetadata` from `df.attrs` via `column_metadata_from_dict()`). **A frame round-trips through pandas to byte equality** apart from the deliberate `SamplingClock`→`TimestampList` axis change: column types and provenance both survive. An `EnumColumn`'s codes are int32 and indistinguishable from a plain `Int32Column` by dtype, so its `enumId` rides in `df.attrs["enum_ids"]` — carried even under `exclude_column_metadata=True`, since without it the column cannot be rebuilt as an enum at all, and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`; reads each instant as `Timestamp.value` (**always nanoseconds**, unlike a raw int64 view, which is in the index's own storage unit — a `datetime64[us]` index viewed as int64 lands 1000× too early); rejects `NaT`, whose integer form is a valid-looking instant; and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type - `src/dp_python_lib/client/query_client.py` - v2 time-series query client (sample-oriented) exposed as `client.query`. Low-level wrappers `query_samples()` (unary, one resumable page) and `iter_query_samples()` (transparent paging), plus `iter_query_samples_stream()` (server-streaming, fire-and-consume, lazy). Queries are described by a kind-neutral `QueryParams` built from the `PvQuery` (`PV`) and `ConfigQuery` (`CFG`) criterion helpers; shares a `_build_query_spec()` seam so a future bucket request builder reuses it. Results wrap the raw `ColumnTable` (`.column_table`, `.next_page_token`); `.to_dataframe()`/`.to_numpy()` delegate to `query_conversions` (Phase 2, optional `[analysis]` extra) - `src/dp_python_lib/client/query_conversions.py` - Pythonic conversions for query results (optional `[analysis]` extra: pandas/numpy/openpyxl, imported lazily). `data_value_to_python()` (oneof extractor: scalars→native, timestamp→epoch-nanos, array→list, structure→dict, image→`Image` wrapper, fail-loud on unhandled arm), `column_table_to_dataframe()` (UTC datetime index + one column per DataColumn; dense-alignment and duplicate-column-name fail-loud; ColumnMetadata in `df.attrs`), `column_table_to_numpy()` (dict of 1-D arrays; complex arms stay 1-D object arrays rather than collapsing to 2-D), `dataframe_to_excel()` (thin `to_excel()` wrapper: row-limit guard, tz-drop, complex-cell stringification), and `query_samples_to_dataframe()`/`stream_query_samples_to_dataframes()` whole-query conveniences (unary concats by column name; streaming yields per-page frames lazily) - `src/dp_python_lib/client/service_api_client_base.py` - Base class for the service clients: owns the channel and the one-per-client gRPC stub, and provides `_dispatch()`, the shared three-tier sender that all 18 unary `_send_*` methods delegate to diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index b91ba77..48a0179 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -139,6 +139,19 @@ same overclaim and is left alone here: it is not this PR's file, and correcting it belongs with whoever reconciles the cookbook's version banners at release time. +- **Copilot fourth-pass review, 2026-09-10.** Three findings against `a29064d`, all real: + - **Enum columns lost their kind and their `enumId` through the pandas round trip.** `EnumColumn.values` is + int32, so leaving it out of the narrow-dtype mapping widened the codes to int64 and rebuilt the column as an + `Int64Column` -- and the `enumId`, which is the only thing saying what those codes mean, was dropped + entirely. The dtype mapping now covers it, and the id rides in `df.attrs["enum_ids"]`, carried even under + `exclude_column_metadata=True` because it is structural rather than descriptive. An enum id on a + non-integer column is now a named error instead of a silently wrong column kind. + - Two comments in `query_conversions` and `sample_status_conversions` still pointed at + `machine_config_client.to_epoch_nanos` after the move to `time_conversions.py` -- my own stale references + from `9e07226`, left behind by `4c6a8e1`. A sweep for the same mistake found two more: a stale module + attribution in this plan's own reference list, and a sentence in `CLAUDE.md` that an earlier edit had + mangled mid-clause. All four corrected. + ## Overview Wrap the modernized DataSet / Annotation / Calculations / Export area of `DpAnnotationService` in the house @@ -285,7 +298,7 @@ attributes a reader of this table might reach for do not exist. ### 4. What already exists in this repo to reuse -- `to_timestamp()` / `TimestampInput` (`machine_config_client.py`) — every `DataBlock` and `TimeRange` bound. +- `to_timestamp()` / `TimestampInput` (`time_conversions.py`) — every `DataBlock` and `TimeRange` bound. - `sampling_clock()` / `timestamp_list()` / `_timestamp_count()` (`sample_status_client.py`) — the `DataTimestamps` axis builders a calculations frame needs. They belong in a shared module now that a second caller exists; re-export from `sample_status_client` so nothing breaks. diff --git a/src/dp_python_lib/client/data_frame_conversions.py b/src/dp_python_lib/client/data_frame_conversions.py index 83ac0ba..10eae72 100644 --- a/src/dp_python_lib/client/data_frame_conversions.py +++ b/src/dp_python_lib/client/data_frame_conversions.py @@ -317,6 +317,9 @@ def _check_frame_alignment(frame: common_pb2.DataFrame, columns: dict[str, list] common_pb2.DoubleColumn: "float64", common_pb2.Int64Column: "int64", common_pb2.BoolColumn: "bool", + # EnumColumn's values field is int32 too; without this its codes widen to int64 and it rebuilds as an + # Int64Column. The enumId that gives those codes meaning travels separately -- see _enum_ids(). + common_pb2.EnumColumn: "int32", } @@ -337,6 +340,20 @@ def _narrow_column_dtypes(frame: common_pb2.DataFrame) -> dict[str, str]: } +def _enum_ids(frame: common_pb2.DataFrame) -> dict[str, str]: + """ + Maps each EnumColumn's name to its enumId. + + An enum column's values are bare integer codes; the enumId is what says which enumeration names them, so + dropping it would leave the codes meaningless. It has nowhere to live in a pandas Series, so it rides in + df.attrs["enum_ids"] and is consulted when rebuilding the column. + + :param frame: The frame whose enum columns to inspect. + :return: A dict of column name -> enumId; empty when the frame has no enum columns. + """ + return {column.name: column.enumId for column in frame.enumColumns} + + def data_frame_to_pandas(frame: common_pb2.DataFrame, exclude_column_metadata: bool = False) -> Any: """ Converts a DataFrame into a pandas DataFrame with a UTC DatetimeIndex. @@ -380,6 +397,12 @@ def data_frame_to_pandas(frame: common_pb2.DataFrame, exclude_column_metadata: b index=index, ) + # enum_ids is structural, not metadata: without it an enum column cannot be rebuilt as one at all, so it is + # carried even when exclude_column_metadata drops the descriptive attrs. + enum_ids = _enum_ids(frame) + if enum_ids: + df.attrs["enum_ids"] = enum_ids + if not exclude_column_metadata: df.attrs["column_metadata"] = { column.name: column_metadata_dict(column) for column in iter_frame_columns(frame) @@ -513,19 +536,25 @@ def column_metadata_from_dict(summary: dict[str, Any] | None) -> common_pb2.Colu return metadata -def _column_from_series(name: str, series: Any, metadata: common_pb2.ColumnMetadata | None = None) -> Any: +def _column_from_series( + name: str, series: Any, metadata: common_pb2.ColumnMetadata | None = None, enum_id: str | None = None +) -> Any: """ Builds the typed column matching a pandas Series' dtype. Mapping: float64 -> DoubleColumn, float32 -> FloatColumn, int64 -> Int64Column, int32 -> Int32Column, bool -> BoolColumn, and object/string -> StringColumn. Anything else raises rather than guessing. + An enum_id routes an integer column to EnumColumn instead, since an enum's values are indistinguishable from + plain int32 codes by dtype alone -- the id is what makes it an enumeration. + A missing value anywhere is a fail-loud error: the typed columns are dense repeated fields with no way to mark an absent entry, so a NaN would have to be either invented as a real value or silently dropped. :param name: The column's name. :param series: The pandas Series holding its values. :param metadata: Optional ColumnMetadata to attach (see column_metadata_from_dict()). + :param enum_id: When set, build an EnumColumn carrying this enumeration id rather than an integer column. :return: The matching typed column message. :raises ValueError: if the dtype has no mapping, or the series contains a missing value. """ @@ -543,6 +572,17 @@ def _column_from_series(name: str, series: Any, metadata: common_pb2.ColumnMetad dtype = series.dtype dtype_name = str(dtype) + if enum_id is not None: + # Checked before EVERY dtype branch, not just the integer ones: an EnumColumn's values are int32 codes, so + # dtype alone cannot distinguish it from a plain Int32Column, and a non-integer dtype paired with an enum + # id is a caller error worth naming rather than quietly building the wrong column kind. + if dtype_name not in ("int64", "Int64", "int32", "Int32"): + raise ValueError( + f"column '{name}' carries an enum id ({enum_id!r}) but has dtype {dtype_name}; an EnumColumn's " + f"values must be integer codes" + ) + return builders.enum_column(name, [int(v) for v in series], enum_id, metadata=metadata) + if dtype_name == "float64": return builders.double_column(name, [float(v) for v in series], metadata=metadata) if dtype_name == "float32": @@ -584,7 +624,8 @@ def data_frame_from_pandas(df: Any) -> common_pb2.DataFrame: inferred. Missing values are rejected fail-loud, since a dense typed column cannot express a gap. Per-column metadata is read back from df.attrs["column_metadata"] when present, so a frame that went out - through data_frame_to_pandas() returns with its tags, attributes, and provenance intact. + through data_frame_to_pandas() returns with its tags, attributes, and provenance intact. Enum columns are + rebuilt as enum columns via df.attrs["enum_ids"]; without that id an integer column is just an integer column. Converting back with data_frame_to_pandas() preserves every value, dtype, and timestamp, but can return the columns grouped by type rather than in their original order -- see that function's docstring. @@ -617,10 +658,16 @@ def data_frame_from_pandas(df: Any) -> common_pb2.DataFrame: # data_frame_to_pandas() parks each column's ColumnMetadata here; carrying it back is what keeps provenance # alive across a round trip. A frame built by hand simply has no attrs, and every column gets None. metadata_by_column = df.attrs.get("column_metadata") or {} + enum_ids = df.attrs.get("enum_ids") or {} data_timestamps = _timestamps_from_index(df.index) columns = [ - _column_from_series(str(name), df[name], metadata=column_metadata_from_dict(metadata_by_column.get(str(name)))) + _column_from_series( + str(name), + df[name], + metadata=column_metadata_from_dict(metadata_by_column.get(str(name))), + enum_id=enum_ids.get(str(name)), + ) for name in df.columns ] return builders.data_frame(data_timestamps, columns) diff --git a/src/dp_python_lib/client/query_conversions.py b/src/dp_python_lib/client/query_conversions.py index 784026e..6d51d91 100644 --- a/src/dp_python_lib/client/query_conversions.py +++ b/src/dp_python_lib/client/query_conversions.py @@ -99,7 +99,7 @@ def _require_numpy(): ) -# The Timestamp -> epoch-nanoseconds conversion is shared (see machine_config_client.to_epoch_nanos); this private +# The Timestamp -> epoch-nanoseconds conversion is shared (see time_conversions.to_epoch_nanos); this private # alias is the name this module has always used internally. _timestamp_to_epoch_nanos = to_epoch_nanos diff --git a/src/dp_python_lib/client/sample_status_conversions.py b/src/dp_python_lib/client/sample_status_conversions.py index 86b1c2b..dee8fef 100644 --- a/src/dp_python_lib/client/sample_status_conversions.py +++ b/src/dp_python_lib/client/sample_status_conversions.py @@ -24,7 +24,7 @@ from dp_python_lib.client.time_conversions import NANOS_PER_SECOND, to_epoch_nanos from dp_python_lib.grpc import common_pb2 -# The Timestamp -> epoch-nanoseconds conversion is shared (see machine_config_client.to_epoch_nanos); this private +# The Timestamp -> epoch-nanoseconds conversion is shared (see time_conversions.to_epoch_nanos); this private # alias is the name this module has always used internally. _timestamp_to_nanos = to_epoch_nanos diff --git a/tests/unit/test_data_frame_conversions.py b/tests/unit/test_data_frame_conversions.py index 12cf5e3..a314446 100644 --- a/tests/unit/test_data_frame_conversions.py +++ b/tests/unit/test_data_frame_conversions.py @@ -720,5 +720,62 @@ def test_inverts_column_metadata_dict(self): self.assertEqual(dfc.column_metadata_from_dict(dfc.column_metadata_dict(column)), original) +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestEnumColumnRoundTrip(unittest.TestCase): + """ + An EnumColumn's values are int32 codes, so dtype alone cannot tell it from an Int32Column. The enumId that + gives those codes meaning has nowhere to live in a Series, so it rides in df.attrs["enum_ids"]. + """ + + def _frame(self): + return dfb.data_frame( + dfb.timestamp_list([T0, T1]), + [ + dfb.enum_column("state", [0, 1], enum_id="beam-state", metadata=dfb.column_metadata(tags=["t"])), + dfb.int32_column("plain", [7, 8]), + ], + ) + + def test_enum_codes_are_not_widened(self): + df = dfc.data_frame_to_pandas(self._frame()) + self.assertEqual(str(df["state"].dtype), "int32") + + def test_enum_id_travels_in_attrs(self): + df = dfc.data_frame_to_pandas(self._frame()) + self.assertEqual(df.attrs["enum_ids"], {"state": "beam-state"}) + + def test_enum_column_round_trips_whole(self): + original = self._frame() + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + + self.assertEqual([c.name for c in restored.enumColumns], ["state"]) + self.assertEqual(restored.enumColumns[0].enumId, "beam-state") + # The plain int32 column beside it must NOT become an enum. + self.assertEqual([c.name for c in restored.int32Columns], ["plain"]) + self.assertEqual(restored, original) + + def test_enum_id_survives_excluded_metadata(self): + # enum_ids is structural: without it the column cannot be rebuilt as an enum at all, so it is carried even + # when the descriptive metadata is dropped. + original = self._frame() + df = dfc.data_frame_to_pandas(original, exclude_column_metadata=True) + restored = dfc.data_frame_from_pandas(df) + + self.assertEqual(restored.enumColumns[0].enumId, "beam-state") + self.assertFalse(restored.enumColumns[0].HasField("metadata")) + + def test_enum_id_on_a_non_integer_column_is_rejected(self): + import pandas as pd + + index = pd.DatetimeIndex(pd.to_datetime([T0_NANOS, T0_NANOS + 1], unit="ns", utc=True)) + for dtype, values in (("float64", [1.5, 2.5]), ("bool", [True, False]), ("object", ["a", "b"])): + with self.subTest(dtype=dtype): + df = pd.DataFrame({"x": pd.Series(values, index=index)}, index=index) + df.attrs["enum_ids"] = {"x": "beam-state"} + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + self.assertIn("enum id", str(ctx.exception)) + + if __name__ == "__main__": unittest.main() From 5ce9e076fd97a150cab69f0c787c98190fd9c6fa Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Thu, 10 Sep 2026 14:15:17 -0600 Subject: [PATCH 11/11] fix: validate pre-built timestamp lists, expose image/struct metadata (issue #6) Two findings from Copilot's fifth pass over 94986fd. A hand-built TimestampList bypassed the strict-ordering rule. timestamp_count() checked only that the list was non-empty, so duplicate or decreasing timestamps -- which timestamp_list() rejects outright -- sailed through data_frame() and SampleStatusFrame alike. Two samples claiming one instant is inexpressible in the sample-status identity model, and data_frame_from_pandas() refuses the index such an axis produces, so the result was a frame the library could not round-trip. This is the third instance of one shape, after the array dims and the zero periodNanos: a builder enforces a rule that the shared count/validate helper does not, so a pre-built message walks past it. timestamp_count() now applies the same check timestamp_list() does, and the docstring says the rule belongs to the axis rather than to the builder. ImageColumn and StructColumn lost their structural fields on read. column_values() yields the raw per-sample payloads, but an image's imageDescriptor (width, height, channels, encoding) and a struct's schemaId are what make those bytes interpretable at all -- without them the values are undecodable blobs. Neither was reachable from the conversion output. Added image_descriptor_dict() / column_schema_id() and the frame-level data_frame_image_descriptors() / data_frame_schema_ids(), following the precedent set for array dims and enum ids: the structural field travels in a companion accessor rather than being folded into column_values(), which keeps its "exactly one entry per sample, uniform across every column kind" contract. 761 tests pass (719 unit, 42 integration + subtests against the live server; 644 pass and 75 skip without the [analysis] extra). The stricter axis rule is shared with SampleStatusFrame, whose 96 tests still pass. ruff clean; 103 cookbook snippets check. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019he3UCsAnqTDE2VQ73Djwn --- CLAUDE.md | 4 +- plan/tickets/6/plan.md | 13 ++++ src/dp_python_lib/client/data_frame.py | 27 ++++++-- .../client/data_frame_conversions.py | 68 ++++++++++++++++++- tests/unit/test_data_frame.py | 31 +++++++++ tests/unit/test_data_frame_conversions.py | 47 +++++++++++++ 6 files changed, 180 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a15604b..37f79fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -157,8 +157,8 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/annotations_client.py` - Annotations client (`save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, `delete_annotation()`, `get_calculations()`) with the `AnnotationQuery` (`AQ`) criterion helpers and the `calculations()` builder, which takes a `dict[str, DataFrame]` so frame-name uniqueness is true by construction. Note `AnnotationsClient` (feature client) vs `AnnotationClient` (facade) - `src/dp_python_lib/client/export_client.py` - Export client (`export_data()`) with the `ExportFormat` str enum and the `calculations_spec()` builder - `src/dp_python_lib/client/time_conversions.py` - The two shared time converters and the `TimestampInput` alias: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`; naive datetimes raise) and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds), plus `NANOS_PER_SECOND`. **The datetime path uses integer arithmetic, never `datetime.timestamp()`** — that returns a float64, which cannot hold present-day epoch seconds at sub-microsecond resolution and moved 99.7% of microsecond datetimes by up to ~119ns, breaking the exact-match contract sample status and provenance depend on. Both were defined in `machine_config_client` as the first module to need them, and six others grew imports from there — which read as though datasets, queries, and DataFrames depended on the machine configuration API; `to_epoch_nanos()` had also been written out privately three separate times. **A leaf module**: it imports only stdlib and the generated protos, so any client module can use it without an import cycle. New time conversions belong here -- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Column names must be **non-blank**, not merely non-empty, and `timestamp_count()` rejects a hand-built `SamplingClock` with a non-positive `periodNanos` as well as a zero count — the read path rejects both, and anything `data_frame()` accepts must be readable back. Sample counts come from the right field per kind: `dataValues` for a `DataColumn`, `images` for an `ImageColumn` (which has no `values` field at all), and `len(values) / prod(dims)` for an array column. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through -- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; array columns reshape to one flat list per sample, with the shape recoverable from `column_dimensions()` / `data_frame_column_dimensions()` so `[2,2]` and `[4]` stay distinguishable), `data_frame_columns()`, `column_metadata_dict()`. An axis that is set but empty is rejected rather than converted to a zero-row table. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`; each Series carries the **narrow dtype its column type implies** — `float32`/`int32`, not pandas' widened inference), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap; rebuilds each column's `ColumnMetadata` from `df.attrs` via `column_metadata_from_dict()`). **A frame round-trips through pandas to byte equality** apart from the deliberate `SamplingClock`→`TimestampList` axis change: column types and provenance both survive. An `EnumColumn`'s codes are int32 and indistinguishable from a plain `Int32Column` by dtype, so its `enumId` rides in `df.attrs["enum_ids"]` — carried even under `exclude_column_metadata=True`, since without it the column cannot be rebuilt as an enum at all, and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`; reads each instant as `Timestamp.value` (**always nanoseconds**, unlike a raw int64 view, which is in the index's own storage unit — a `datetime64[us]` index viewed as int64 lands 1000× too early); rejects `NaT`, whose integer form is a valid-looking instant; and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type +- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Column names must be **non-blank**, not merely non-empty, and `timestamp_count()` validates a hand-built axis the way the builders do: a `SamplingClock` needs a positive `periodNanos` as well as a non-zero count, and a `TimestampList` must be **strictly increasing** (duplicates included — two samples cannot claim one instant). The read path rejects all of these, and anything `data_frame()` accepts must be readable back. Shared with `SampleStatusFrame`, which validates the same way. Sample counts come from the right field per kind: `dataValues` for a `DataColumn`, `images` for an `ImageColumn` (which has no `values` field at all), and `len(values) / prod(dims)` for an array column. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through +- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; it yields exactly one entry per sample for every column kind, with the structural fields a payload cannot be interpreted without kept in companion accessors: array dims via `column_dimensions()` / `data_frame_column_dimensions()` so `[2,2]` and `[4]` stay distinguishable, an image's width/height/channels/encoding via `image_descriptor_dict()` / `data_frame_image_descriptors()`, and a struct's `schemaId` via `column_schema_id()` / `data_frame_schema_ids()`), `data_frame_columns()`, `column_metadata_dict()`. An axis that is set but empty is rejected rather than converted to a zero-row table. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`; each Series carries the **narrow dtype its column type implies** — `float32`/`int32`, not pandas' widened inference), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap; rebuilds each column's `ColumnMetadata` from `df.attrs` via `column_metadata_from_dict()`). **A frame round-trips through pandas to byte equality** apart from the deliberate `SamplingClock`→`TimestampList` axis change: column types and provenance both survive. An `EnumColumn`'s codes are int32 and indistinguishable from a plain `Int32Column` by dtype, so its `enumId` rides in `df.attrs["enum_ids"]` — carried even under `exclude_column_metadata=True`, since without it the column cannot be rebuilt as an enum at all, and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`; reads each instant as `Timestamp.value` (**always nanoseconds**, unlike a raw int64 view, which is in the index's own storage unit — a `datetime64[us]` index viewed as int64 lands 1000× too early); rejects `NaT`, whose integer form is a valid-looking instant; and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type - `src/dp_python_lib/client/query_client.py` - v2 time-series query client (sample-oriented) exposed as `client.query`. Low-level wrappers `query_samples()` (unary, one resumable page) and `iter_query_samples()` (transparent paging), plus `iter_query_samples_stream()` (server-streaming, fire-and-consume, lazy). Queries are described by a kind-neutral `QueryParams` built from the `PvQuery` (`PV`) and `ConfigQuery` (`CFG`) criterion helpers; shares a `_build_query_spec()` seam so a future bucket request builder reuses it. Results wrap the raw `ColumnTable` (`.column_table`, `.next_page_token`); `.to_dataframe()`/`.to_numpy()` delegate to `query_conversions` (Phase 2, optional `[analysis]` extra) - `src/dp_python_lib/client/query_conversions.py` - Pythonic conversions for query results (optional `[analysis]` extra: pandas/numpy/openpyxl, imported lazily). `data_value_to_python()` (oneof extractor: scalars→native, timestamp→epoch-nanos, array→list, structure→dict, image→`Image` wrapper, fail-loud on unhandled arm), `column_table_to_dataframe()` (UTC datetime index + one column per DataColumn; dense-alignment and duplicate-column-name fail-loud; ColumnMetadata in `df.attrs`), `column_table_to_numpy()` (dict of 1-D arrays; complex arms stay 1-D object arrays rather than collapsing to 2-D), `dataframe_to_excel()` (thin `to_excel()` wrapper: row-limit guard, tz-drop, complex-cell stringification), and `query_samples_to_dataframe()`/`stream_query_samples_to_dataframes()` whole-query conveniences (unary concats by column name; streaming yields per-page frames lazily) - `src/dp_python_lib/client/service_api_client_base.py` - Base class for the service clients: owns the channel and the one-per-client gRPC stub, and provides `_dispatch()`, the shared three-tier sender that all 18 unary `_send_*` methods delegate to diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index 48a0179..bcf516d 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -152,6 +152,19 @@ attribution in this plan's own reference list, and a sentence in `CLAUDE.md` that an earlier edit had mangled mid-clause. All four corrected. +- **Copilot fifth-pass review, 2026-09-10.** Two findings against `94986fd`, both real: + - **A hand-built `TimestampList` bypassed the strict-ordering rule.** `timestamp_count()` checked only that it + was non-empty, so duplicate or decreasing timestamps -- which `timestamp_list()` rejects -- were accepted by + both `data_frame()` and `SampleStatusFrame`, producing a frame `data_frame_from_pandas()` then refuses. Two + samples claiming one instant is also inexpressible in the sample-status identity model. This is the third + instance of the same shape (after array dims and `periodNanos`): a builder enforces a rule that the shared + count/validate helper does not, so a pre-built message walks straight past it. + - **`ImageColumn` and `StructColumn` lost their structural fields on read.** `column_values()` yields the raw + payloads, but an image's `imageDescriptor` (width/height/channels/encoding) and a struct's `schemaId` are what + make those bytes interpretable, and neither was reachable from the conversion output. Added + `image_descriptor_dict()` / `column_schema_id()` and their frame-level companions, following the precedent + already set for array dims and enum ids rather than folding them into `column_values()`. + ## Overview Wrap the modernized DataSet / Annotation / Calculations / Export area of `DpAnnotationService` in the house diff --git a/src/dp_python_lib/client/data_frame.py b/src/dp_python_lib/client/data_frame.py index 094a911..9ffb87b 100644 --- a/src/dp_python_lib/client/data_frame.py +++ b/src/dp_python_lib/client/data_frame.py @@ -150,14 +150,15 @@ def timestamp_count(timestamps: common_pb2.DataTimestamps) -> int: An empty or malformed axis is rejected here rather than allowed to surface later as a confusing column-length mismatch. The axis builders already make this unreachable -- sampling_clock() requires count >= 1 and a positive period, and timestamp_list() requires a non-empty list -- but a hand-built DataTimestamps can still - carry a zero-count or zero-period SamplingClock, or an empty TimestampList, and all report their oneof arm as - set. Rejecting them keeps this in step with expand_data_timestamps(), which applies the same rules on the - read path; anything data_frame() accepts must be readable back. + carry a zero-count or zero-period SamplingClock, or an empty or out-of-order TimestampList, and all report + their oneof arm as set. Rejecting them keeps this in step with the read path -- expand_data_timestamps() for + the clock rules, data_frame_from_pandas() for the ordering one; anything data_frame() accepts must be readable + back. :param timestamps: The time axis to measure. :return: The number of timestamps on the axis. - :raises ValueError: if neither axis form is set, if the axis describes no timestamps, or if a SamplingClock's - periodNanos is not positive. + :raises ValueError: if neither axis form is set, if the axis describes no timestamps, if a SamplingClock's + periodNanos is not positive, or if a TimestampList is not strictly increasing. """ axis = timestamps.WhichOneof("value") if axis == "samplingClock": @@ -172,9 +173,23 @@ def timestamp_count(timestamps: common_pb2.DataTimestamps) -> int: raise ValueError(f"DataTimestamps samplingClock requires periodNanos > 0, got {period}") return count if axis == "timestampList": - count = len(timestamps.timestampList.timestamps) + entries = timestamps.timestampList.timestamps + count = len(entries) if count < 1: raise ValueError("DataTimestamps timestampList requires at least one timestamp, got an empty list") + # Strict ordering, as timestamp_list() enforces -- a hand-built axis bypasses that builder entirely. + # Duplicate or decreasing timestamps mean two samples claim one instant, which the identity model cannot + # express, and data_frame_from_pandas() rejects the index they produce, so accepting them here would build + # a frame the library cannot round-trip. + previous = entries[0] + for index, current in enumerate(entries[1:], start=1): + if (current.epochSeconds, current.nanoseconds) <= (previous.epochSeconds, previous.nanoseconds): + raise ValueError( + f"DataTimestamps timestampList requires strictly increasing timestamps; entry {index} " + f"({current.epochSeconds}.{current.nanoseconds:09d}) does not follow entry {index - 1} " + f"({previous.epochSeconds}.{previous.nanoseconds:09d})" + ) + previous = current return count raise ValueError("DataTimestamps must specify either a samplingClock or a timestampList") diff --git a/src/dp_python_lib/client/data_frame_conversions.py b/src/dp_python_lib/client/data_frame_conversions.py index 10eae72..e921a9f 100644 --- a/src/dp_python_lib/client/data_frame_conversions.py +++ b/src/dp_python_lib/client/data_frame_conversions.py @@ -159,6 +159,68 @@ def data_frame_column_dimensions(frame: common_pb2.DataFrame) -> dict[str, list[ return dimensions +def image_descriptor_dict(column: Any) -> dict[str, Any] | None: + """ + Returns an ImageColumn's descriptor -- width, height, channels, encoding -- as a plain dict, or None for any + other column kind. + + column_values() yields one encoded payload per sample, which nothing can decode on its own: the descriptor is + what says how to interpret those bytes. It is kept separate for the same reason column_dimensions() is, so + that column_values() stays "exactly one entry per sample" across every column kind. + + :param column: Any column message. + :return: A dict with 'width', 'height', 'channels', and 'encoding' for an ImageColumn, else None. + """ + if not isinstance(column, common_pb2.ImageColumn): + return None + descriptor = column.imageDescriptor + return { + "width": descriptor.width, + "height": descriptor.height, + "channels": descriptor.channels, + "encoding": descriptor.encoding, + } + + +def column_schema_id(column: Any) -> str | None: + """ + Returns a StructColumn's schemaId, or None for any other column kind. + + A struct column's values are opaque serialized payloads; the schema id is what names the schema needed to + interpret them, so the values alone are not usable without it. + + :param column: Any column message. + :return: The schemaId for a StructColumn, else None. + """ + if not isinstance(column, common_pb2.StructColumn): + return None + return column.schemaId + + +def data_frame_image_descriptors(frame: common_pb2.DataFrame) -> dict[str, dict[str, Any]]: + """ + Returns every ImageColumn's descriptor in a frame, keyed by column name. + + Pairs with data_frame_columns(), which gives the payloads these describe. Non-image columns are absent. + + :param frame: The frame to inspect. + :return: A dict of column name -> descriptor dict; empty when the frame has no image columns. + """ + return {column.name: image_descriptor_dict(column) for column in frame.imageColumns} + + +def data_frame_schema_ids(frame: common_pb2.DataFrame) -> dict[str, str]: + """ + Returns every StructColumn's schemaId in a frame, keyed by column name. + + Pairs with data_frame_columns(), which gives the payloads the schema describes. Non-struct columns are absent. + + :param frame: The frame to inspect. + :return: A dict of column name -> schemaId; empty when the frame has no struct columns. + """ + return {column.name: column.schemaId for column in frame.structColumns} + + def column_values(column: Any) -> list: """ Extracts one Python value per sample from any supported column message. @@ -171,8 +233,10 @@ def column_values(column: Any) -> list: bytes payload per sample; and a legacy DataColumn is converted per value by data_value_to_python(), so an unset oneof becomes None -- the only representation of a gap in this API. - An array column's per-sample list is flat: the dims that give it shape are available separately from - column_dimensions(), so that every column kind here yields exactly one entry per sample. + Every column kind here yields exactly one entry per sample. The structural information some kinds carry + alongside their values lives in companion accessors rather than being folded into the result: an array + column's dims in column_dimensions(), an image column's descriptor in image_descriptor_dict(), and a struct + column's schema id in column_schema_id(). Those payloads cannot be interpreted without them. :param column: A typed column, a legacy DataColumn, or an ImageColumn. :return: One value per sample, in axis order. diff --git a/tests/unit/test_data_frame.py b/tests/unit/test_data_frame.py index bc8deda..e888475 100644 --- a/tests/unit/test_data_frame.py +++ b/tests/unit/test_data_frame.py @@ -336,6 +336,37 @@ def test_rejects_unsupported_column_type(self): dfb.data_frame(_axis(1), ["not a column"]) self.assertIn("unsupported column type", str(ctx.exception)) + def test_rejects_out_of_order_prebuilt_timestamp_list(self): + # timestamp_list() enforces strict ordering, but a hand-built axis bypasses that builder. Two samples + # claiming one instant is inexpressible in the identity model, and data_frame_from_pandas() rejects the + # index such an axis produces -- so accepting it here would build an un-round-trippable frame. + for label, seconds in (("decreasing", [2, 1]), ("duplicate", [1, 1])): + with self.subTest(case=label): + axis = common_pb2.DataTimestamps() + for second in seconds: + entry = axis.timestampList.timestamps.add() + entry.epochSeconds = 1_700_000_000 + second + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0])]) + self.assertIn("strictly increasing", str(ctx.exception)) + + def test_rejects_prebuilt_timestamp_list_duplicated_at_nanosecond_precision(self): + axis = common_pb2.DataTimestamps() + for _ in range(2): + entry = axis.timestampList.timestamps.add() + entry.epochSeconds = 1_700_000_000 + entry.nanoseconds = 5 + with self.assertRaises(ValueError): + dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0])]) + + def test_accepts_strictly_increasing_prebuilt_timestamp_list(self): + axis = common_pb2.DataTimestamps() + for second in (1, 2): + entry = axis.timestampList.timestamps.add() + entry.epochSeconds = 1_700_000_000 + second + frame = dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0])]) + self.assertEqual(len(frame.dataTimestamps.timestampList.timestamps), 2) + def test_rejects_zero_period_sampling_clock(self): # sampling_clock() makes this unreachable, but a hand-built axis bypasses it. expand_data_timestamps() # rejects a non-positive period on the read side, so accepting it here would build an unreadable frame -- diff --git a/tests/unit/test_data_frame_conversions.py b/tests/unit/test_data_frame_conversions.py index a314446..e1f21c2 100644 --- a/tests/unit/test_data_frame_conversions.py +++ b/tests/unit/test_data_frame_conversions.py @@ -777,5 +777,52 @@ def test_enum_id_on_a_non_integer_column_is_rejected(self): self.assertIn("enum id", str(ctx.exception)) +class TestStructuralColumnFields(unittest.TestCase): + """ + Some column kinds carry information their values cannot be interpreted without. column_values() keeps its + one-entry-per-sample shape, so those fields travel in companion accessors -- the same split as array dims. + """ + + def _frame(self): + image = common_pb2.ImageColumn() + image.name = "cam" + image.images.extend([b"frame-0", b"frame-1"]) + image.imageDescriptor.width = 640 + image.imageDescriptor.height = 480 + image.imageDescriptor.channels = 3 + image.imageDescriptor.encoding = "rgb8" + + struct = common_pb2.StructColumn() + struct.name = "readings" + struct.schemaId = "schema-1" + struct.values.extend([b"payload-0", b"payload-1"]) + + return dfb.data_frame(dfb.timestamp_list([T0, T1]), [image, struct]) + + def test_image_descriptor_is_recoverable(self): + # Without width/height/channels/encoding the payload bytes cannot be decoded at all. + self.assertEqual( + dfc.data_frame_image_descriptors(self._frame()), + {"cam": {"width": 640, "height": 480, "channels": 3, "encoding": "rgb8"}}, + ) + + def test_struct_schema_id_is_recoverable(self): + self.assertEqual(dfc.data_frame_schema_ids(self._frame()), {"readings": "schema-1"}) + + def test_values_still_yield_one_entry_per_sample(self): + self.assertEqual( + dfc.data_frame_columns(self._frame()), + {"cam": [b"frame-0", b"frame-1"], "readings": [b"payload-0", b"payload-1"]}, + ) + + def test_other_column_kinds_have_neither(self): + plain = dfb.double_column("d", [1.0]) + self.assertIsNone(dfc.image_descriptor_dict(plain)) + self.assertIsNone(dfc.column_schema_id(plain)) + frame = dfb.data_frame(_axis(1), [plain]) + self.assertEqual(dfc.data_frame_image_descriptors(frame), {}) + self.assertEqual(dfc.data_frame_schema_ids(frame), {}) + + if __name__ == "__main__": unittest.main()