diff --git a/.dev/tools/check-cookbook-snippets.py b/.dev/tools/check-cookbook-snippets.py index 2e30bfc..ca173a0 100755 --- a/.dev/tools/check-cookbook-snippets.py +++ b/.dev/tools/check-cookbook-snippets.py @@ -104,10 +104,26 @@ QuerySampleStatusesRequestParams, sampling_clock, timestamp_list, + DataSetClient, + DataSetQuery, + DataSetQuery as DS, + SaveDataSetRequestParams, + data_block, + AnnotationsClient, + AnnotationQuery, + AnnotationQuery as AQ, + SaveAnnotationRequestParams, + calculations, + ExportClient, + ExportFormat, + ExportDataRequestParams, + calculations_spec, ) from dp_python_lib.client import query_conversions as qc from dp_python_lib.client import sample_status_conversions as ssc +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc client: MldpClient = MldpClient() begin: datetime = datetime(2024, 1, 1, tzinfo=timezone.utc) @@ -117,6 +133,27 @@ # building the query. Snippets that demonstrate query *construction* build their own. params: QueryParams = QueryParams( begin_time=begin, end_time=end, pv_selector=PV.name_list(["BPMS:GUNB:314:X"])) + +# The datasets-and-annotations recipe is one continuous worked example: it saves a dataset, then an +# annotation on it, then calculations, then exports and deletes them. These are the handles the +# recipe establishes in its own earlier snippets and legitimately carries forward, the way `client` +# and `params` are carried above. Server-assigned ids are strings. +t0: datetime = datetime(2026, 2, 2, 18, 0, tzinfo=timezone.utc) +t1: datetime = datetime(2026, 2, 2, 19, 0, tzinfo=timezone.utc) +# Declared without an annotation so mypy infers `str` for the standalone snippets that consume these, +# while the recipe's own snippets can still rebind them from an Optional accessor and narrow with an +# assert, as conventions.md teaches. An explicit `str` would conflict with those real assignments; an +# explicit `str | None` would force every later snippet to re-narrow a handle the recipe already did. +# +# Seeding a handle here cannot prove the recipe actually binds that name -- a snippet binding `saved_id` +# while later ones read `dataset_id` type-checked cleanly and was still broken end to end. Names carried +# across snippets are verified by reading the recipe as one continuous script, not by this preamble. +# `str | None` is what the accessors return; the recipe narrows with an assert before use, and the +# standalone snippets below do the same, so consumers see a plain `str`. +dataset_id: str | None = "6aa1bb271a768e97db44d426" +annotation_id: str | None = "6aa1bb271a768e97db44d427" +calculations_id: str | None = "6aa1bb271a768e97db44d428" +assert dataset_id is not None and annotation_id is not None and calculations_id is not None # --- end preamble --- """ diff --git a/CLAUDE.md b/CLAUDE.md index 68d6d64..37f79fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,12 +150,15 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `src/dp_python_lib/client/ingestion_client.py` - Ingestion service client with methods like `register_provider()` - `src/dp_python_lib/client/annotation_client.py` - Annotation service facade; groups feature-scoped clients sharing the one `DpAnnotationService` channel (`.pv_metadata`, `.machine_config`, `.sample_status`, `.datasets`, `.annotations`, `.export` — every implemented `DpAnnotationService` feature area) - `src/dp_python_lib/client/pv_metadata_client.py` - PV metadata client (`save_pv_metadata()`, `get_pv_metadata()`, `query_pv_metadata()`, `iter_pv_metadata()`, `delete_pv_metadata()`) plus the `PvMetadataQuery` (`Q`) criterion helpers -- `src/dp_python_lib/client/machine_config_client.py` - Machine configuration client covering both configurations (`save_configuration()`, `get_configuration()`, `query_configurations()`, `iter_configurations()`, `delete_configuration()`) and their temporal activations (`save_configuration_activation()`, `get_configuration_activation()`, `query_configuration_activations()`, `iter_configuration_activations()`, `delete_configuration_activation()`, `get_active_configurations()`). Includes the `ConfigurationQuery` (`C`) and `ConfigurationActivationQuery` (`CA`) criterion helpers and the `to_timestamp()` helper (tz-aware datetime / epoch seconds / `common.Timestamp`). Get/delete activation take a composite key (`client_activation_id` XOR `configuration_name`+`start_time`). Activation `end_time` is optional — omit it for an open-ended activation ("still in effect"); the field is then genuinely absent on the wire -- `src/dp_python_lib/client/sample_status_client.py` - Sample status client (`save_sample_statuses()`, `query_sample_statuses()`, `iter_sample_statuses()`, `iter_sample_statuses_stream()`, `delete_sample_statuses()`) plus the `sampling_clock()` / `timestamp_list()` axis builders and the `SampleStatusColumn` / `SampleStatusFrame` construction classes. A status's identity key is `(pvName, timestamp, domain, layer)`; `delete_sample_statuses()` requires either `pv_names` or an explicit `all_pvs=True` opt-in for the destructive wildcard +- `src/dp_python_lib/client/machine_config_client.py` - Machine configuration client covering both configurations (`save_configuration()`, `get_configuration()`, `query_configurations()`, `iter_configurations()`, `delete_configuration()`) and their temporal activations (`save_configuration_activation()`, `get_configuration_activation()`, `query_configuration_activations()`, `iter_configuration_activations()`, `delete_configuration_activation()`, `get_active_configurations()`). Includes the `ConfigurationQuery` (`C`) and `ConfigurationActivationQuery` (`CA`) criterion helpers. The shared time converters it used to own now live in `time_conversions.py`. Get/delete activation take a composite key (`client_activation_id` XOR `configuration_name`+`start_time`). Activation `end_time` is optional — omit it for an open-ended activation ("still in effect"); the field is then genuinely absent on the wire +- `src/dp_python_lib/client/sample_status_client.py` - Sample status client (`save_sample_statuses()`, `query_sample_statuses()`, `iter_sample_statuses()`, `iter_sample_statuses_stream()`, `delete_sample_statuses()`) plus the `SampleStatusColumn` / `SampleStatusFrame` construction classes. The `sampling_clock()` / `timestamp_list()` axis builders now live in `data_frame.py` (issue #6 Phase 2, once calculations frames became a second caller) and are re-exported here, so existing imports are unaffected. A status's identity key is `(pvName, timestamp, domain, layer)`; `delete_sample_statuses()` requires either `pv_names` or an explicit `all_pvs=True` opt-in for the destructive wildcard - `src/dp_python_lib/client/sample_status_conversions.py` - Per-sample expansion of query results (no optional extras required): `expand_data_timestamps()` (SamplingClock positions computed in **integer nanoseconds**, never float seconds — the exact-match contract depends on it), `bucket_to_rows()` / `buckets_to_rows()` / `iter_rows()` yielding `SampleStatusRow` objects with absent confidence/reason surfaced as `None` rather than fabricated `0.0`/`""` - `src/dp_python_lib/client/dataset_client.py` - DataSet client (`save_dataset()`, `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, plus the `get_datasets(ids)` batch fetch that avoids the annotation-listing N+1) with the `DataSetQuery` (`DS`) criterion helpers and the `data_block()` builder. `data_block()` is the only place `begin < end` is checked — the server does not - `src/dp_python_lib/client/annotations_client.py` - Annotations client (`save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, `delete_annotation()`, `get_calculations()`) with the `AnnotationQuery` (`AQ`) criterion helpers and the `calculations()` builder, which takes a `dict[str, DataFrame]` so frame-name uniqueness is true by construction. Note `AnnotationsClient` (feature client) vs `AnnotationClient` (facade) - `src/dp_python_lib/client/export_client.py` - Export client (`export_data()`) with the `ExportFormat` str enum and the `calculations_spec()` builder +- `src/dp_python_lib/client/time_conversions.py` - The two shared time converters and the `TimestampInput` alias: `to_timestamp()` (tz-aware datetime / epoch seconds / `common.Timestamp` → `Timestamp`; naive datetimes raise) and its inverse `to_epoch_nanos()` (`Timestamp` → integer epoch nanoseconds), plus `NANOS_PER_SECOND`. **The datetime path uses integer arithmetic, never `datetime.timestamp()`** — that returns a float64, which cannot hold present-day epoch seconds at sub-microsecond resolution and moved 99.7% of microsecond datetimes by up to ~119ns, breaking the exact-match contract sample status and provenance depend on. Both were defined in `machine_config_client` as the first module to need them, and six others grew imports from there — which read as though datasets, queries, and DataFrames depended on the machine configuration API; `to_epoch_nanos()` had also been written out privately three separate times. **A leaf module**: it imports only stdlib and the generated protos, so any client module can use it without an import cycle. New time conversions belong here +- `src/dp_python_lib/client/data_frame.py` - Builders for `common.DataFrame`, the shared time-series payload (also ingestion's `ingestionDataFrame`, so #17 extends this rather than forking it): the `sampling_clock()` / `timestamp_list()` / `timestamp_count()` axis helpers **relocated here from `sample_status_client`** (and re-exported from it, so existing imports keep working), the typed scalar column builders (`double_column`, `float_column`, `int64_column`, `int32_column`, `bool_column`, `string_column`, `enum_column`), the legacy `data_column()` escape hatch (a `None` entry becomes an unset oneof — the only way to express a gap on a shared axis), the provenance helpers (`column_metadata`, `provenance`, `pv_source`, `calculations_source`), and `data_frame()` assembly, which routes columns by type and enforces the server's **shape** rules client-side (non-blank names, non-empty values, count match, name uniqueness across all types) while leaving its size caps server-side. Column names must be **non-blank**, not merely non-empty, and `timestamp_count()` validates a hand-built axis the way the builders do: a `SamplingClock` needs a positive `periodNanos` as well as a non-zero count, and a `TimestampList` must be **strictly increasing** (duplicates included — two samples cannot claim one instant). The read path rejects all of these, and anything `data_frame()` accepts must be readable back. Shared with `SampleStatusFrame`, which validates the same way. Sample counts come from the right field per kind: `dataValues` for a `DataColumn`, `images` for an `ImageColumn` (which has no `values` field at all), and `len(values) / prod(dims)` for an array column. An array column's sample count is `len(values) / prod(dims)`, and a value count that is not a **whole multiple** of `prod(dims)` is rejected rather than floor-divided into a passing count — the read path applies the same rule, so a frame this accepts is always one `data_frame_conversions` can read back. `data_column()` maps integers and floats by `numbers.Integral` / `numbers.Real` (and NumPy's bool by type name, without importing NumPy), so NumPy scalars map like their Python counterparts: `np.float64` subclasses `float` but `np.int64` and `np.bool_` subclass nothing here, and matching on exact Python type would accept some and reject others. Array/image/struct/serialized builders are #17's; hand-built ones pass through +- `src/dp_python_lib/client/data_frame_conversions.py` - Reading a `DataFrame` back. Pure Python (no extras): `data_frame_timestamps()` (integer-nanosecond axis expansion), `column_values()` (standalone per-column converter, written so the bucket query #16 can reuse it; it yields exactly one entry per sample for every column kind, with the structural fields a payload cannot be interpreted without kept in companion accessors: array dims via `column_dimensions()` / `data_frame_column_dimensions()` so `[2,2]` and `[4]` stay distinguishable, an image's width/height/channels/encoding via `image_descriptor_dict()` / `data_frame_image_descriptors()`, and a struct's `schemaId` via `column_schema_id()` / `data_frame_schema_ids()`), `data_frame_columns()`, `column_metadata_dict()`. An axis that is set but empty is rejected rather than converted to a zero-row table. Behind `[analysis]`: `data_frame_to_pandas()` (UTC index built from int64 nanos, `ColumnMetadata` in `df.attrs`; each Series carries the **narrow dtype its column type implies** — `float32`/`int32`, not pandas' widened inference), `data_frame_from_pandas()` (dtype→typed column; a NaN anywhere is fail-loud, since a dense typed column cannot express a gap; rebuilds each column's `ColumnMetadata` from `df.attrs` via `column_metadata_from_dict()`). **A frame round-trips through pandas to byte equality** apart from the deliberate `SamplingClock`→`TimestampList` axis change: column types and provenance both survive. An `EnumColumn`'s codes are int32 and indistinguishable from a plain `Int32Column` by dtype, so its `enumId` rides in `df.attrs["enum_ids"]` — carried even under `exclude_column_metadata=True`, since without it the column cannot be rebuilt as an enum at all, and the `calculations_to_dataframes()` / `calculations_from_dataframes()` bridges. The pandas direction always emits a `TimestampList`, never an inferred `SamplingClock`; reads each instant as `Timestamp.value` (**always nanoseconds**, unlike a raw int64 view, which is in the index's own storage unit — a `datetime64[us]` index viewed as int64 lands 1000× too early); rejects `NaT`, whose integer form is a valid-looking instant; and rejects duplicate column names up front (a duplicated label makes `df[name]` a DataFrame, which would otherwise fail deep inside as pandas' "truth value of a Series is ambiguous"). `column_metadata_dict()` reports **only the origin arm actually set** on each provenance source — a PV source has no `calculations_column` key and vice versa — plus `time_range` as epoch nanoseconds when present; an absent range has no key rather than a fabricated `(0, 0)`. A pandas round trip preserves every value, dtype, and timestamp but **not column order**: a `DataFrame` stores each column kind in its own repeated field, so columns come back grouped by type - `src/dp_python_lib/client/query_client.py` - v2 time-series query client (sample-oriented) exposed as `client.query`. Low-level wrappers `query_samples()` (unary, one resumable page) and `iter_query_samples()` (transparent paging), plus `iter_query_samples_stream()` (server-streaming, fire-and-consume, lazy). Queries are described by a kind-neutral `QueryParams` built from the `PvQuery` (`PV`) and `ConfigQuery` (`CFG`) criterion helpers; shares a `_build_query_spec()` seam so a future bucket request builder reuses it. Results wrap the raw `ColumnTable` (`.column_table`, `.next_page_token`); `.to_dataframe()`/`.to_numpy()` delegate to `query_conversions` (Phase 2, optional `[analysis]` extra) - `src/dp_python_lib/client/query_conversions.py` - Pythonic conversions for query results (optional `[analysis]` extra: pandas/numpy/openpyxl, imported lazily). `data_value_to_python()` (oneof extractor: scalars→native, timestamp→epoch-nanos, array→list, structure→dict, image→`Image` wrapper, fail-loud on unhandled arm), `column_table_to_dataframe()` (UTC datetime index + one column per DataColumn; dense-alignment and duplicate-column-name fail-loud; ColumnMetadata in `df.attrs`), `column_table_to_numpy()` (dict of 1-D arrays; complex arms stay 1-D object arrays rather than collapsing to 2-D), `dataframe_to_excel()` (thin `to_excel()` wrapper: row-limit guard, tz-drop, complex-cell stringification), and `query_samples_to_dataframe()`/`stream_query_samples_to_dataframes()` whole-query conveniences (unary concats by column name; streaming yields per-page frames lazily) - `src/dp_python_lib/client/service_api_client_base.py` - Base class for the service clients: owns the channel and the one-per-client gRPC stub, and provides `_dispatch()`, the shared three-tier sender that all 18 unary `_send_*` methods delegate to @@ -172,6 +175,9 @@ plan documents one change, `CLAUDE.md` documents the invariant it established. - `tests/unit/test_export_client.py` - Unit tests for ExportClient (`ExportFormat` mapping and unreachable `UNSPECIFIED`, `calculations_spec()`, the zero-source rejection, three-tier error handling) - `tests/unit/test_annotation_client.py` - Unit tests pinning the `AnnotationClient` facade wiring (every feature client present, one shared channel, one stub apiece) - `tests/integration/test_datasets_annotations_integration.py` - Live-server round trip for datasets/annotations/calculations; ingests its own samples first, because `saveDataSet` requires archived PVs +- `tests/unit/test_time_conversions.py` - Unit tests for the shared time converters (`to_timestamp()` input forms and the naive-datetime/bool/unsupported-type rejections; `to_epoch_nanos()` exactness and its round trip with `to_timestamp()`) +- `tests/unit/test_data_frame.py` - Unit tests for the data_frame builders (axis relocation, each typed column, `data_column()` bool-before-int and unset-oneof handling, provenance helpers, and every `data_frame()` shape rule incl. array dims and serialized-column name-only checks) +- `tests/unit/test_data_frame_conversions.py` - Unit tests for data_frame_conversions (nanosecond-exact expansion, per-column conversion incl. array reshaping, duplicate-name fail-loud, and — skipping cleanly without `[analysis]` — the pandas round trip, dtype mapping, and NaN fail-loud) - `tests/unit/test_query_client.py` - Unit tests for QueryClient (request building, three-tier error handling, unary paging, streaming, `PvQuery`/`ConfigQuery` helpers, `QueryParams` validation) - `tests/unit/test_query_conversions.py` - Unit tests for query_conversions (each DataValue arm, dense-alignment and duplicate-column-name fail-loud, int-gap float-upcast, timestamp columns, 1-D object arrays for complex arms, metadata in attrs, concat-by-name, Excel row-limit/stringification/native-bytes; DataFrame/NumPy/Excel tests skip cleanly when the `[analysis]` extra is absent) - `pyproject.toml` - Project metadata and dependencies @@ -525,10 +531,62 @@ Notes: ### DataSets, Annotations, and Export API (Annotation Service) -Phase 1 of issue #6 (`plan/tickets/6/plan.md`) added three feature clients on the `annotation` facade: +Issue #6 (`plan/tickets/6/plan.md`) added three feature clients on the `annotation` facade: `client.annotation.datasets` (`DataSetClient`), `client.annotation.annotations` (`AnnotationsClient`), and -`client.annotation.export` (`ExportClient`). The full usage section lands with the cookbook recipe in PR 2; the -invariants worth knowing before touching this code: +`client.annotation.export` (`ExportClient`). A DataSet names a region of the archive; an Annotation describes one +or more DataSets and may own a Calculations payload of derived values; export writes any of it to a file on the +server. Worked example: `doc/cookbook/datasets-and-annotations.md`. + +```python +from datetime import datetime, timezone +from dp_python_lib.client import ( + MldpClient, SaveDataSetRequestParams, SaveAnnotationRequestParams, ExportDataRequestParams, ExportFormat, + DataSetQuery as DS, AnnotationQuery as AQ, data_block, calculations, calculations_spec, sampling_clock, +) +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc + +client = MldpClient() +ds, an, ex = client.annotation.datasets, client.annotation.annotations, client.annotation.export +t0 = datetime(2026, 2, 2, 18, tzinfo=timezone.utc) +t1 = datetime(2026, 2, 2, 19, tzinfo=timezone.utc) + +# a region of the archive: one DataBlock per (time range, PV list). Every PV must already have +# ingested data -- see the archive-existence invariant below. +dataset_id = ds.save_dataset(SaveDataSetRequestParams( + name="CXI shift, hour 1", owner_id="cmcchesney", + data_blocks=[data_block(t0, t1, ["BPMS:GUNB:314:X"])], + tags=["cxi-3443"], attributes={"EXP": "CXI_3443"}, modified_by="cmcchesney")).dataset_id + +# derived values with column-level provenance, one frame per time axis +frame = dfb.data_frame( + sampling_clock(start_time=t0, period_nanos=1_000_000_000, count=3), + [dfb.double_column("x_rms", [0.31, 0.29, 0.33], metadata=dfb.column_metadata( + provenance=dfb.provenance(process="1 Hz RMS", + derived_from=[dfb.pv_source("BPMS:GUNB:314:X", (t0, t1))])))]) +saved = an.save_annotation(SaveAnnotationRequestParams( + name="Orbit drift", owner_id="cmcchesney", dataset_ids=[dataset_id], + tags=["reviewed"], calculations=calculations({"orbit-rms": frame}))) + +# find it again; query results carry ids, so resolve a page's datasets in ONE call +for a in an.iter_annotations([AQ.tags(["reviewed"]), AQ.attributes("EXP")]): # key-only search + print(a.name, a.dataSetIds, bool(a.calculationsId)) +datasets = ds.get_datasets([i for a in an.iter_annotations([AQ.datasets([dataset_id])]) + for i in a.dataSetIds]) + +# read the calculations back (get_annotation is the only method returning them inline) +calcs = an.get_calculations(saved.calculations_id).calculations +columns = dfc.data_frame_columns(calcs.calculationDataFrames[0].frame) # plain Python, no extras +frames = dfc.calculations_to_dataframes(calcs) # pandas, [analysis] extra + +# export; then tear down annotations first, since delete_dataset is refused while referenced +ex.export_data(ExportDataRequestParams(ExportFormat.HDF5, dataset_id=dataset_id, + calculations_spec=calculations_spec(saved.calculations_id))) +an.delete_annotation(saved.annotation_id) +ds.delete_dataset(dataset_id) +``` + +Invariants worth knowing before touching this code: - **`saveDataSet` requires every PV named in a data block to already exist in the archive** — that is, to have *ingested data*. The server's error text says `no PV metadata found for names: [...]`, but the check is a @@ -575,6 +633,10 @@ invariants worth knowing before touching this code: large `$in` and an oversized request message. A short result is logged at WARNING, since an id withheld for any other reason is indistinguishable from a dangling one. - `patchDataSet` / `patchAnnotation` are reserved "not implemented" placeholders and are not wrapped. +- Calculations are built with `data_frame.py` and read back with `data_frame_conversions.py`; both are shared with + #16/#17 rather than local to this area. `data_frame_from_pandas()` always emits a `TimestampList`, never an + inferred `SamplingClock`, and rejects `NaN` fail-loud: a dense typed column cannot express a gap, so use + `data_column()` or a separate frame. ### Sample Status API (Annotation Service) diff --git a/README.md b/README.md index 7750d67..3db05bf 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,18 @@ for their service. transparent paging (`query_samples()` / `iter_query_samples()`) and server-streaming (`iter_query_samples_stream()`). Results convert to pandas DataFrames, NumPy arrays, and Excel via the optional `[analysis]` extra. +- **DataSets** — `client.annotation.datasets`. Name a region of the archive (time ranges plus the + PVs covered over them) so it can be found, annotated, and exported later: `save_dataset()`, + `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, and a + `get_datasets(ids)` batch fetch, with the `DataSetQuery` criterion helpers. +- **Annotations and calculations** — `client.annotation.annotations`. Attach conclusions and + derived values to those datasets, with column-level provenance recording what each column came + from: `save_annotation()`, `get_annotation()`, `query_annotations()`, `iter_annotations()`, + `delete_annotation()`, `get_calculations()`, and the `AnnotationQuery` helpers. Calculations are + built with the `data_frame` builders and read back with `data_frame_conversions`, which also + bridges to pandas under the optional `[analysis]` extra. +- **Export** — `client.annotation.export`. Export a saved dataset, ad-hoc data blocks, and/or + calculations to HDF5, CSV, or XLSX. The file is written on the server; there is no retrieval RPC. - **Provider registration** — `client.ingestion_client.register_provider()`. The rest of the ingestion API is not yet implemented. @@ -97,10 +109,6 @@ Note the v2 query API comes from unreleased dp-grpc work and will not work again - `queryTable()` — PV time-series data in tabular format - `queryPvStats()` — archive ingestion statistics for PVs - `queryProviders()` / `queryProviderStats()` — provider information and ingestion statistics -- **Annotation Service** - - `saveDataSet()` / `queryDataSets()` — datasets over collections of PVs and time ranges - - `saveAnnotation()` / `queryAnnotations()` — annotations targeting a dataset - - `exportData()` — export datasets to common file formats - **Ingestion Stream Service** - `subscribeDataEvent()` — notification when a data condition in the ingestion stream triggers diff --git a/doc/cookbook/README.md b/doc/cookbook/README.md index 441c1db..9e1d983 100644 --- a/doc/cookbook/README.md +++ b/doc/cookbook/README.md @@ -26,6 +26,7 @@ client. | [Recording machine configuration](machine-configuration.md) | Defining configurations, recording when each was active, closing and opening intervals, and answering "what was the machine doing at 18:04?" | | [Querying time-series data](query.md) | Retrieving samples by PV name, by metadata, or by machine configuration, and converting results to pandas / NumPy / Excel | | [Labeling samples](sample-status.md) | Recording per-sample status codes, reading them back, and querying data with flagged samples excluded | +| [DataSets and annotations](datasets-and-annotations.md) | Naming a region of the archive, attaching analysis results with column-level provenance, round-tripping calculations through pandas, and exporting | **Not yet covered: getting data in.** `IngestionClient` currently exposes only `register_provider()`, so there is no ingestion recipe. @@ -45,6 +46,8 @@ query recipes is the data the earlier recipes create: `MODE=09`), activated over a shift with `DEST=CXI` and `EXP=CXI_3443`. - Queries that retrieve those PVs by name, by *"every monitor in GUNB"*, and by *"whatever ran during the CXI shift"*. +- A dataset naming the first hour of that shift, an annotation recording an orbit drift, and a 1 Hz + RMS calculation attached to it with provenance pointing back at the source PV. Attribute names and values are the facility's; tag values are illustrative placeholders. diff --git a/doc/cookbook/datasets-and-annotations.md b/doc/cookbook/datasets-and-annotations.md new file mode 100644 index 0000000..024cf60 --- /dev/null +++ b/doc/cookbook/datasets-and-annotations.md @@ -0,0 +1,637 @@ +# DataSets and Annotations + +Naming a region of the archive so you can find it again, attaching derived values to it with a +record of what they were computed from, and exporting the result. + +> **Target API version:** dp-grpc 1.16.0, which is **not yet released** — the newest tag is +> `rel-1.15.0`. The modernized DataSet / Annotation / Export API is new in 1.16.0 and will not work +> against a `rel-1.15.0` server, which answers these calls with `UNIMPLEMENTED`. +> +> **Verified against:** a dp-service build from `main` at commit `fddf692`, carrying the 1.16.0 API. + +See [API conventions](conventions.md) for result checking, paging, and time handling. + +Examples use `client.annotation.datasets`, `client.annotation.annotations`, and +`client.annotation.export`. Note that `client.annotation` itself is `None` unless an annotation +channel is configured, so guard on it before reaching through. + +### Imports used by the examples + +```python +# cookbook:skip +from datetime import datetime, timezone + +from dp_python_lib.client import ( + MldpClient, + SaveDataSetRequestParams, + SaveAnnotationRequestParams, + ExportDataRequestParams, + ExportFormat, + DataSetQuery as DS, + AnnotationQuery as AQ, + data_block, + calculations, + calculations_spec, + sampling_clock, +) +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc +``` + +## Contents + +- [Model](#model) — three objects and how they relate +- [Naming a region of the archive](#naming-a-region-of-the-archive) +- [Finding datasets again](#finding-datasets-again) +- [Attaching an analysis result](#attaching-an-analysis-result) +- [Recording where the numbers came from](#recording-where-the-numbers-came-from) +- [Reading calculations back](#reading-calculations-back) +- [Going through pandas](#going-through-pandas) +- [Updating without losing the calculations](#updating-without-losing-the-calculations) +- [Exporting](#exporting) +- [Tearing down](#tearing-down) +- [Also worth knowing](#also-worth-knowing) + +## Model + +Three objects, in a chain: + +A **DataSet** names a region of the archive: a list of **DataBlocks**, each one a time range plus +the PVs covered over it. It holds no data — only the coordinates of data that already exists. + +An **Annotation** describes one or more DataSets: a name, a description, tags, and optionally a +**Calculations** payload of derived values. Annotations are how analysis results get attached to +the raw data they came from. + +**Calculations** are named frames, each one a time axis plus typed columns of values, with +optional per-column **provenance** recording what each column was derived from. + +``` +DataSet ──described by──> Annotation ──owns──> Calculations +(where the (what you (the derived + data is) concluded) numbers) +``` + +Two consequences worth internalizing before you start. + +**A DataSet can only name PVs that already have archived data.** `save_dataset()` checks that +every PV in every data block exists in the archive — not merely that it has PV metadata saved. A +dataset over a PV that has never been ingested is rejected: + +``` +no PV metadata found for names: [BPMS:GUNB:314:X] +``` + +That message is misleading: the check is against ingested data, not the metadata catalogue. If you +see it for a PV you know you catalogued, the PV has no samples. + +**Calculations belong to their annotation.** They have no independent lifecycle: saving an +annotation is the only way to write them, deleting the annotation deletes them, and re-saving an +annotation without them deletes them too. See +[updating without losing the calculations](#updating-without-losing-the-calculations). + +## Naming a region of the archive + +A `DataBlock` is one time range and the PVs measured over it. Build one per (range, PV list) pair; +a dataset is a list of them, so a study spanning two disjoint windows is one dataset with two +blocks. + +```python +# cookbook:partial +t0 = datetime(2026, 2, 2, 18, 0, tzinfo=timezone.utc) +t1 = datetime(2026, 2, 2, 19, 0, tzinfo=timezone.utc) + +saved = client.annotation.datasets.save_dataset(SaveDataSetRequestParams( + name="CXI shift, hour 1", + owner_id="cmcchesney", + data_blocks=[data_block(t0, t1, ["BPMS:GUNB:314:X", "BPMS:GUNB:314:Y"])], + description="First hour of the CXI_3443 shift, both transverse BPM planes.", + tags=["cxi-3443", "shift-study"], + attributes={"EXP": "CXI_3443", "DEST": "CXI"}, + modified_by="cmcchesney", +)) +if saved.result_status.is_error: + raise RuntimeError(saved.result_status.message) + +dataset_id = saved.dataset_id +assert dataset_id is not None # guaranteed once is_error is False; accessors are Optional +print(dataset_id) # server-assigned id, e.g. '6aa1bb271a768e97db44d426' +``` + +`data_block()` requires `begin < end` and a non-empty PV list. That check exists here because the +server does not make it: it validates only that each bound is non-zero and never compares the two, +so a reversed block would otherwise be stored happily. It also rejects a bare string for the PV +list, which would otherwise be iterated into one PV name per character. + +A block's range is [half-open](conventions.md#half-open-ranges), `[begin, end)`, like every other +range in the library, so back-to-back blocks — one ending at `T`, the next starting at `T` — cover +the sample at `T` exactly once. `saveDataSet` itself never compares the bounds; the interval only +acquires meaning at export, where it reaches the same per-sample trimming that `query_samples()` +does. + +**HDF5 export is the exception.** It is bucket-granular: every bucket that *overlaps* the block is +written whole and untrimmed, so an HDF5 file can contain samples outside the range you asked for, +and back-to-back blocks sharing a straddling bucket write it twice. CSV and XLSX trim to the exact +range. Nothing client-side can change this — it is a property of the export format. + +Note the tags come back **normalized** — lowercased, deduplicated, and sorted — so a tag saved as +`CXI-3443` reads back as `cxi-3443`, and queries must match the lowercase form. + +## Finding datasets again + +Seven criteria, combined with the usual [AND across / OR within](conventions.md#query-criteria) +rules: + +```python +# cookbook:partial +# Everything from this experiment, in this area of the machine. +for dataset in client.annotation.datasets.iter_datasets([ + DS.attributes("EXP", ["CXI_3443"]), + DS.pv_names(["BPMS:GUNB:314:X"]), +]): + print(dataset.name, len(dataset.dataBlocks)) +``` + +Two criteria behave differently from the older helpers in this library: + +**`attributes()` takes an optional value list.** Omit it to ask *"does this key exist at all?"*, +regardless of value: + +```python +# cookbook:partial +# Every dataset tagged with an experiment, whichever one. +tagged = list(client.annotation.datasets.iter_datasets([DS.attributes("EXP")])) +``` + +**`criteria` itself is optional.** An empty or omitted list matches everything, so browsing the +whole collection is a legitimate call rather than something to work around: + +```python +# cookbook:partial +for dataset in client.annotation.datasets.iter_datasets(): + print(dataset.id, dataset.name) +``` + +`text()` is a full-text search across the name and description together. Only **one** text +criterion is allowed per query — two cannot be ANDed — and the client rejects a second one before +the RPC: + +```python +# cookbook:partial +# One text criterion: fine. +found = list(client.annotation.datasets.iter_datasets([DS.text("shift")])) + +# Two: ValueError, naming the rule. Combine the terms into one search instead. +``` + +### Fetching many datasets at once + +Annotations carry `dataSetIds`, not the datasets themselves. Resolving them one call at a time is +an N+1, so `get_datasets()` does it in a single paged query: + +```python +# cookbook:partial +ids = [i for a in client.annotation.annotations.iter_annotations([AQ.tags(["reviewed"])]) + for i in a.dataSetIds] +datasets = client.annotation.datasets.get_datasets(ids) # dict: id -> DataSet + +for dataset_id, dataset in datasets.items(): + print(dataset_id, dataset.name) +``` + +Ids are deduplicated, an empty list returns `{}` without an RPC, and ids that resolve to nothing +are simply **absent from the dict** rather than raising — a dangling `dataSetIds` entry is a normal +consequence of deletion, not an error. + +## Attaching an analysis result + +An annotation on its own records a conclusion: + +```python +# cookbook:partial +result = client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( + name="Orbit drift during CXI_3443", + owner_id="cmcchesney", + dataset_ids=[dataset_id], + description="Slow horizontal drift, ~0.3 mm over the hour. Suspect thermal.", + tags=["reviewed", "orbit"], + modified_by="cmcchesney", +)) +if result.result_status.is_error: + raise RuntimeError(result.result_status.message) + +annotation_id = result.annotation_id +assert annotation_id is not None +``` + +To attach *numbers*, build a `Calculations` payload. A frame is one time axis plus its columns: + +```python +# cookbook:partial +# Your analysis output: one value per sample, however you computed it. +x_rms_values = [0.31, 0.29, 0.33] # in reality, 3600 of them +y_rms_values = [0.12, 0.14, 0.11] + +# A 1 Hz RMS series: one sample per second, count matching the values above. +axis = sampling_clock(start_time=t0, period_nanos=1_000_000_000, count=len(x_rms_values)) + +frame = dfb.data_frame(axis, [ + dfb.double_column("x_rms", x_rms_values), + dfb.double_column("y_rms", y_rms_values), +]) + +result = client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( + name="1 Hz orbit RMS, CXI_3443 hour 1", + owner_id="cmcchesney", + dataset_ids=[dataset_id], + tags=["reviewed"], + calculations=calculations({"orbit-rms": frame}), + modified_by="cmcchesney", +)) +if result.result_status.is_error: + raise RuntimeError(result.result_status.message) + +# No annotation_id was passed, so this SAVED A SECOND ANNOTATION rather than adding calculations to +# the one above. Rebind both handles: the rest of this recipe works with the annotation that owns +# the calculations. To attach them to the first annotation instead, pass annotation_id= and the +# fields to carry forward -- see "Updating without losing the calculations" below. +annotation_id = result.annotation_id +calculations_id = result.calculations_id +assert annotation_id is not None and calculations_id is not None +``` + +`calculations()` takes a **dict** of frame name to frame, which makes frame-name uniqueness true by +construction — the server rejects duplicates, and a list would let you build one. + +The column builders validate the shape before anything goes over the wire: every column needs a +non-blank name and a value count matching the axis, and names must be unique **across all column +types** in a frame. A mismatch names the offending column: + +``` +data_frame() column 'x_rms' has 3599 values but the time axis has 3600 timestamps; +every column must carry exactly one value per sample +``` + +Size limits — string length, array element counts, image and struct bytes — are deliberately *not* +checked here. Those are deployment policy and can change; the server enforces them. + +### Columns with gaps + +The typed columns are dense: a `DoubleColumn` has one value per sample, with no way to mark one +absent. When a column genuinely has holes, use `data_column()`, whose values are `DataValue` +messages that can be left unset: + +```python +# cookbook:partial +# None becomes a genuinely absent value, not a fabricated zero. +sparse = dfb.data_column("beam_current", [12.7, None, 12.9]) +``` + +The alternative — and often the better one — is to give the sparse column **its own frame** over +only the timestamps where it has values, built with `timestamp_list()`. A frame per sampling rate +is the natural shape when your columns genuinely differ. + +## Recording where the numbers came from + +A calculated column without provenance is a number nobody can check later. `ColumnMetadata` records +what a column was derived from, per column: + +```python +# cookbook:partial +metadata = dfb.column_metadata( + tags=["derived"], + attributes={"unit": "mm"}, + provenance=dfb.provenance( + source="orbit-analysis-rig", + process="1 Hz RMS over 10 kHz BPM samples", + derived_from=[dfb.pv_source("BPMS:GUNB:314:X", (t0, t1))], + ), +) + +axis = sampling_clock(start_time=t0, period_nanos=1_000_000_000, count=3) +x_rms_values = [0.31, 0.29, 0.33] + +frame = dfb.data_frame(axis, [dfb.double_column("x_rms", x_rms_values, metadata=metadata)]) +``` + +`pv_source()` names an archived PV; `calculations_source()` names a column of *other* calculations, +for values derived from earlier derived values: + +```python +# cookbook:partial +chained = dfb.provenance( + process="10-minute moving average of the 1 Hz RMS", + derived_from=[dfb.calculations_source(calculations_id, "orbit-rms", "x_rms")], +) +``` + +Both accept an optional `(begin, end)` pair narrowing which part of the source was used. + +Reading it back, `column_metadata_dict()` gives you the plain-Python view — no extras needed. Each +entry carries **only the origin arm actually set**, plus the time range when there is one, reported +as epoch nanoseconds like every other instant in this library: + +```python +# cookbook:partial +provenance = dfb.provenance(derived_from=[dfb.pv_source("BPMS:GUNB:314:X", (t0, t1))]) +column = dfb.double_column("x_rms", [0.31, 0.29, 0.33], metadata=dfb.column_metadata(provenance=provenance)) + +summary = dfc.column_metadata_dict(column) +print(summary["provenance"]["derived_from"]) +# [{'pv_name': 'BPMS:GUNB:314:X', 'time_range': (1770055200000000000, 1770058800000000000)}] +``` + +A PV source has no `calculations_column` key and a calculations source has no `pv_name` key, rather +than the unset one showing up as an empty string — absence means "not this arm", the same +[absent-vs-empty](conventions.md) discipline the rest of the library follows. A source with no time +range simply has no `time_range` key, never a fabricated `(0, 0)`. + +These are **soft references**. Deleting the annotation that owns the referenced calculations leaves +the link dangling; nothing resolves or cleans it up, and readers are expected to tolerate that. + +## Reading calculations back + +`get_annotation()` is the only method that returns calculations **inline**: + +```python +# cookbook:partial +read = client.annotation.annotations.get_annotation(annotation_id) +if read.result_status.is_error: + raise RuntimeError(read.result_status.message) + +calcs = read.calculations # None if the annotation has none +``` + +`query_annotations()` deliberately does not: results carry the `calculationsId` with the content +left empty, so listing a hundred annotations does not drag a hundred payloads with it. Follow the +id when you want the content: + +```python +# cookbook:partial +for annotation in client.annotation.annotations.iter_annotations([AQ.tags(["reviewed"])]): + if not annotation.calculationsId: + continue + fetched = client.annotation.annotations.get_calculations(annotation.calculationsId) + if fetched.result_status.is_error: + raise RuntimeError(fetched.result_status.message) + payload = fetched.calculations + assert payload is not None + print(annotation.name, len(payload.calculationDataFrames)) +``` + +Without pandas, `data_frame_conversions` returns plain Python: + +```python +# cookbook:partial +calcs = client.annotation.annotations.get_annotation(annotation_id).calculations +assert calcs is not None + +entry = calcs.calculationDataFrames[0] + +epoch_nanos = dfc.data_frame_timestamps(entry.frame) # list[int], exact +columns = dfc.data_frame_columns(entry.frame) # {"x_rms": [...], "y_rms": [...]} + +print(entry.name, len(epoch_nanos), list(columns)) +``` + +Timestamps are computed in **integer nanoseconds** throughout. A `SamplingClock` expands as +`startTime + i * periodNanos` with no float step anywhere: present-day epoch nanoseconds need about +61 bits, and a float64 carries 53, so routing through float seconds would silently move every +timestamp by hundreds of nanoseconds. + +## Going through pandas + +With the [`analysis` extra](conventions.md#optional-dependencies) installed, whole frames convert +in one call: + +```python +# cookbook:partial +calcs = client.annotation.annotations.get_annotation(annotation_id).calculations + +frames = dfc.calculations_to_dataframes(calcs) # {"orbit-rms": DataFrame} +df = frames["orbit-rms"] + +print(df.index.tz) # UTC +print(df.attrs["column_metadata"]["x_rms"]["provenance"]["process"]) +``` + +The index is a UTC `DatetimeIndex` built directly from int64 nanoseconds, and per-column metadata +lands in `df.attrs["column_metadata"]` — the same convention the +[query recipe](query.md) uses. + +The reverse direction turns analysis output back into a payload: + +```python +# cookbook:partial +calcs = client.annotation.annotations.get_annotation(annotation_id).calculations +df = dfc.calculations_to_dataframes(calcs)["orbit-rms"] + +calcs_to_save = dfc.calculations_from_dataframes({"orbit-rms": df}) +``` + +Two things to know about it. + +**Missing values are rejected, loudly.** A dense typed column cannot represent a gap, so a `NaN` +anywhere raises rather than inventing a value or dropping a row: + +``` +column 'x_rms' contains missing values (first at row position(s) [17]), which a dense typed +column cannot represent. Either give the sparse column its own frame over only the timestamps +where it has values, or build it with data_frame.data_column(), whose DataValues can be left unset. +``` + +**The index always becomes an explicit `TimestampList`**, never an inferred `SamplingClock`. A +clock is only correct if the spacing is exactly uniform at nanosecond precision, and inferring that +from an index that merely *looks* regular would quietly change your timestamps. When the data +really is a regular clock, say so with `sampling_clock()` and build the frame directly. + +The dtype mapping is `float64`→Double, `float32`→Float, `int64`→Int64, `int32`→Int32, `bool`→Bool, +and object/string→String. Anything else — complex, datetime columns, categoricals — raises rather +than guessing. Duplicate column names are rejected up front: a frame stores one column per name, and +a `concat` or a `merge` with overlapping names produces duplicates easily. + +**A round trip preserves values, but not column order.** A `DataFrame` keeps each column kind in its +own repeated field, so the wire format has no single ordering across kinds. Converting out and back +returns every value, dtype, and timestamp intact, with the columns grouped by type: + +```python +# cookbook:partial +calcs = client.annotation.annotations.get_annotation(annotation_id).calculations +df = dfc.calculations_to_dataframes(calcs)["orbit-rms"] + +restored = dfc.data_frame_to_pandas(dfc.data_frame_from_pandas(df)) +restored = restored[list(df.columns)] # reindex by name to get your order back +``` + +Select by name rather than by position, and reindex when the order matters. + +## Updating without losing the calculations + +`save_annotation()` is a [full replace](conventions.md#save-semantics-full-replace), and that +**includes the calculations**. Re-saving an annotation without them does not leave them alone: it +clears the reference and deletes the stored object. + +```python +# cookbook:partial +# WRONG -- silently deletes the calculations this annotation owned +client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( + name="Orbit drift during CXI_3443 (revised)", + owner_id="cmcchesney", + dataset_ids=[dataset_id], + annotation_id=annotation_id, +)) +``` + +Read the current state and carry the calculations forward: + +```python +# cookbook:partial +read = client.annotation.annotations.get_annotation(annotation_id) +if read.result_status.is_error: + raise RuntimeError(read.result_status.message) + +existing = read.annotation +assert existing is not None # guaranteed once is_error is False + +replaced = client.annotation.annotations.save_annotation(SaveAnnotationRequestParams( + name="Orbit drift during CXI_3443 (revised)", + owner_id=existing.ownerId, + dataset_ids=list(existing.dataSetIds), + annotation_ids=list(existing.annotationIds), + description=existing.description, + tags=list(existing.tags), + attributes={a.name: a.value for a in existing.attributes}, + calculations=read.calculations, # carried forward + annotation_id=annotation_id, + modified_by="cmcchesney", +)) +if replaced.result_status.is_error: + raise RuntimeError(replaced.result_status.message) + +# Carrying calculations through a replace stores a NEW object and deletes the old one, so the id +# you were holding is now dangling. Rebind it, or a later export/read will fail. +calculations_id = replaced.calculations_id +assert calculations_id is not None +``` + +A replace that *does* carry new calculations returns a **new** `calculations_id`; the previous +object is deleted, not orphaned. An annotation saved with no calculations at all reports +`calculations_id == ""` — empty string, not `None`. `None` means the call failed. + +## Exporting + +`export_data()` writes a file **on the server**, in HDF5, CSV, or XLSX. Sources merge: a saved +dataset by id, ad-hoc data blocks, calculations, or any combination — at least one is required. + +```python +# cookbook:partial +export = client.annotation.export.export_data(ExportDataRequestParams( + ExportFormat.HDF5, + dataset_id=dataset_id, + calculations_spec=calculations_spec(calculations_id), +)) +if export.result_status.is_error: + raise RuntimeError(export.result_status.message) + +print(export.file_path) # a path on the SERVER's filesystem +print(export.file_url) # '' unless the deployment publishes exports over HTTP +``` + +`ExportFormat` accepts either the member or its string value, so `ExportFormat.CSV` and +`"csv"` both work; anything else raises a `ValueError` naming the valid formats, which is what +makes the server-rejected `EXPORT_FORMAT_UNSPECIFIED` unreachable. + +For a one-off export of a region you do not want to keep, skip the dataset entirely: + +```python +# cookbook:partial +client.annotation.export.export_data(ExportDataRequestParams( + "csv", + data_blocks=[data_block(t0, t1, ["BPMS:GUNB:314:X"])], +)) +``` + +`calculations_spec()` narrows what gets included. Omit `frame_columns` for everything; supply it to +pick specific columns of specific frames, excluding any frame you do not mention: + +```python +# cookbook:partial +spec = calculations_spec(calculations_id, {"orbit-rms": ["x_rms"]}) +``` + +**There is no download.** `file_path` is a path on the server's filesystem, and no RPC retrieves +the file — so this library offers no download convenience rather than pretending otherwise. How you +collect the file is a deployment question: a shared mount, `file_url` if the deployment publishes +over HTTP, or an out-of-band copy. + +Two rejections to expect: the tabular formats (CSV, XLSX) can only represent **scalar** columns, so +exporting array, image, or struct columns to them fails — use HDF5. And an unknown dataset or +calculations id is rejected rather than producing an empty file. + +## Tearing down + +Order matters. A dataset cannot be deleted while any annotation references it: + +```python +# cookbook:partial +refused = client.annotation.datasets.delete_dataset(dataset_id) +print(refused.result_status.is_error) # True, naming a referencing annotation +``` + +There is deliberately no `cascade=True`. Deleting a dataset and every annotation about it is two +destructive operations behind one flag; the honest shape is the two-step the server designed: + +```python +# cookbook:partial +for annotation in client.annotation.annotations.iter_annotations([AQ.datasets([dataset_id])]): + deleted = client.annotation.annotations.delete_annotation(annotation.id) + if deleted.result_status.is_error: + raise RuntimeError(deleted.result_status.message) + +client.annotation.datasets.delete_dataset(dataset_id) +``` + +Deleting an annotation **cascades to its calculations**. It does not clean up incoming references: +other annotations' `annotationIds` entries, and `derivedFrom` provenance links naming the deleted +calculations, are left dangling. + +## Also worth knowing + +- **Deleting something that does not exist is an error**, not a silent success. Both + `delete_dataset()` and `delete_annotation()` report `no ... record found for id: ...` on a second + delete, so a teardown that runs twice reports failures the second time. +- **`updatedTime` is unset on create.** It appears once a record has been replaced at least once. +- **Ids are ObjectIds.** A malformed id is rejected as malformed rather than reported as "not + found"; the distinction is useful when debugging. +- **Page tokens are opaque keyset tokens**, and a malformed or wrong-query one is **rejected**. + This differs from the older metadata queries, which silently restart from the beginning + ([dp-service #193](https://github.com/osprey-dcs/dp-service/issues/193)). Do not construct or + reuse tokens across queries. +- **`patchDataSet` and `patchAnnotation` are not wrapped.** They are reserved placeholders that + return "not implemented"; use the full-replace `save_*` methods. +- **Array, image, and struct column builders do not exist yet.** `data_frame.py` covers the typed + scalar columns and the `DataColumn` escape hatch; the rest are + [issue #17](https://github.com/osprey-dcs/dp-python-lib/issues/17)'s to design with ingestion data + in hand. Meanwhile, build those column protos directly and pass them to `data_frame()`, which + accepts pre-built columns alongside the ones its builders return. +- **Serialized columns are skipped on read.** `data_frame_columns()` ignores + `serializedDataColumns`, whose payloads this library does not decode; read the field directly if + you need it. + +### How far these examples have been verified + +The dataset and annotation lifecycle **has** been exercised against a live Annotation Service built +from dp-service `main` at `fddf692` (the 1.16.0 API, pre-release) by +`tests/integration/test_datasets_annotations_integration.py`: save/get round trips, +every criterion including the key-only attribute search, lowercase tag normalization, paging, +rejected page tokens, the `get_datasets()` batch fetch, calculations inline on `get_annotation()` +versus id-only on `query_annotations()`, the full-replace clearing behavior, the delete cascade, and +the referenced-dataset refusal. + +That test ingests its own samples first, because of the archive-existence rule described in +[Model](#model) — there is no ingestion client yet +([issue #17](https://github.com/osprey-dcs/dp-python-lib/issues/17)), so it uses the generated stub +directly. + +The `x_rms_values` in these examples stand in for real analysis output. The **numbers** are +illustrative; the calls around them are the verified part. diff --git a/plan/tickets/6/plan.md b/plan/tickets/6/plan.md index 7e48e79..bcf516d 100644 --- a/plan/tickets/6/plan.md +++ b/plan/tickets/6/plan.md @@ -36,6 +36,134 @@ dp-service `fddf692` (annotation on `localhost:50053`, ingestion on `:50051`): 18 tests, 12 subtests. Writing it surfaced one further server behavior the triage had not found — `saveDataSet` requires its PVs to exist in the archive — now recorded in section 3 and in `CLAUDE.md`. +- **Phases 2 and 3 implemented 2026-09-09.** `data_frame.py` (axis builders relocated from `sample_status_client` + and re-exported, typed scalar column builders, the `data_column()` escape hatch, provenance helpers, and + `data_frame()` assembly with the server's shape rules checked client-side) and `data_frame_conversions.py` (the + pure-Python read side plus the pandas bridges behind `[analysis]`). 87 new unit tests; 657 total with the extra + installed, 601 passing and 56 skipping cleanly without it (verified in a venv that has no pandas). One naming + collision surfaced and was resolved: the `data_frame()` *function* is deliberately not re-exported from + `dp_python_lib.client`, because binding that name would shadow the `data_frame` *module* and break the + `from dp_python_lib.client import data_frame as dfb` form this plan's own reference snippet uses. +- **Phase 4 implemented 2026-09-09.** `doc/cookbook/datasets-and-annotations.md` (continuing the shared worked + example: a dataset over the CXI shift's first hour, an orbit-drift annotation, a 1 Hz RMS calculation with + provenance, and the export), the cookbook README table and worked-example note, the `README.md` move of the three + Annotation Service bullets from TODO to Current state, and the `CLAUDE.md` usage section. The integration test + grew the Phase 4 legs — builder-made calculations read back through `data_frame_conversions` with a sub-second + axis (so the round trip exercises nanosecond arithmetic rather than whole seconds), provenance survival, and four + export cases — for 25 tests, 12 subtests, all passing against dp-service `fddf692`. The checker preamble gained + the new names plus the recipe's shared worked-example handles. The `CLAUDE.md` snippet was extracted and **run + against the live server**, not just type-checked: every documented call succeeds end to end. +- **PR #45 review fixes, 2026-09-10.** Five findings, two of them defects: + - `data_frame()` accepted an array column whose value count was not a whole multiple of `prod(dims)`, floor- + dividing it into a passing sample count — building a frame that `data_frame_conversions` then refused to read + back. The write path now applies the same whole-multiple rule the read path already did, so the two agree by + construction; a regression test asserts both reject the identical input. + - `column_metadata_dict()` dropped a provenance source's `timeRange` entirely, losing "which part of the source + was used" — the substantive half of provenance for a derived column. It is now reported as epoch nanoseconds, + and each source carries only the origin arm actually set rather than the unset arm as an empty string. The + integration test asserts the range survives a real server round trip. + - `data_frame_from_pandas()` on duplicate column labels failed deep inside as pandas' "truth value of a Series is + ambiguous"; it now rejects them up front by name. + - A pandas round trip reorders columns (grouped by proto field), which was true but undocumented and easy to read + as a bug. Documented in both docstrings and the cookbook, and pinned by a test. + - `data_column()` accepted `np.float64` (a `float` subclass) but rejected `np.int64` and `np.bool_` (subclasses of + nothing it matched). It now maps by `numbers.Integral` / `numbers.Real`, with NumPy's bool matched by type + module and name so this module keeps its no-NumPy dependency. + + Fixing the provenance finding needed a Timestamp -> epoch-nanoseconds conversion, which turned out to exist + privately in three modules already (`query_conversions`, `sample_status_conversions`, and, briefly, a fourth copy + added here). That prompted a second look at where the time converters live at all. They were defined in + `machine_config_client`, the first module to need them, and six others had grown imports of `to_timestamp()` from + there -- so datasets, queries, and DataFrames all read as though they depended on the machine configuration API. + Both directions now live in a new leaf module, `client/time_conversions.py`, which imports only stdlib and the + generated protos: `to_timestamp()`, `to_epoch_nanos()`, `TimestampInput`, and `NANOS_PER_SECOND`. Every internal + caller was repointed at it, and `machine_config_client` is now just another caller. Done inside this ticket + rather than deferred, since the feature is unreleased and a follow-up would ship the wart in the release. + +- **Copilot review of PR #45, 2026-09-10.** Seven findings against `f101ff6`; one (array floor division) duplicated a + finding already fixed, and the other six were real: + - `data_frame()` counted every column as `len(column.values)`, but an `ImageColumn` keeps its per-sample payloads + in `images` and has no `values` field, so every prebuilt image column -- a kind the module documents as + supported -- raised `AttributeError` instead of being assembled. + - `_timestamps_from_index()` read the index as a raw int64 view, which is expressed in the index's own storage + unit. pandas 2+ keeps second/millisecond/microsecond resolutions, so a `datetime64[us]` index was serialized + 1000x too early, silently. It now reads `Timestamp.value`, which is nanoseconds regardless of unit, and + rejects `NaT` (whose integer form is a valid-looking instant). A round-trip test had compared raw int64 views + on both sides, so it passed while both were equally wrong. + - Column-name validation accepted whitespace-only names, though the documented rule is non-blank. + - `data_frame_timestamps()` documented that it rejects an empty axis but returned `[]` for a set-but-empty + `timestampList`, converting a corrupt frame to a zero-row table. + - Array dims were consumed to delimit samples and then discarded, so `[2, 2]` and `[4]` were indistinguishable -- + D7 calls for the dims to travel alongside the values. Added `column_dimensions()` / + `data_frame_column_dimensions()` rather than changing `column_values()`'s one-entry-per-sample contract. + - The cookbook's first snippet bound `saved_id` while every later snippet read `dataset_id` (likewise + `saved_annotation_id` / `saved_calculations_id`). Every snippet type-checked because the checker preamble + pre-seeded those names -- the recipe was broken only when read end to end, which is how a reader reads it. + The preamble now documents that seeding a name cannot prove the recipe binds it. + +- **Copilot second-pass review, 2026-09-10.** Four findings against `86621d1`, all real: + - **`to_timestamp()` lost sub-microsecond precision on every datetime.** The datetime branch routed through + `value.timestamp()`, a float64, which cannot hold present-day epoch seconds at that resolution: 99.7% of + microsecond-precision datetimes came back with a wrong nanosecond field, by up to ~119 ns. This is the exact + contract sample-status matching and provenance ranges rest on, and it predates this ticket -- the relocation + into `time_conversions.py` is simply what put it under review. Now integer arithmetic off the timedelta from + the epoch, which is lossless because a datetime's own resolution is exactly microseconds. The float/int epoch + seconds path is unchanged. + - `timestamp_count()` accepted a hand-built `SamplingClock` with `periodNanos == 0`, which + `expand_data_timestamps()` rejects -- the same write/read asymmetry class as the array-dims finding, and one + where every sample after the first would have carried the first sample's timestamp. + - The cookbook's calculations save omitted `annotation_id`, so it created a *second* annotation while the recipe + kept reading the first (calculation-free) one; every later `get_annotation()` read the wrong record. + - The cookbook's replace discarded its result, but a replace carrying calculations stores a new object and + deletes the old, so the `calculations_id` the later export used was dangling. + + The two cookbook findings share a root cause with the earlier `saved_id` one: snippets are type-checked in + isolation against a seeded preamble, which cannot see that a *sequence* of snippets is incoherent. Verified + this time by parsing the recipe as one continuous script and checking each handle is rebound before its next + use. + +- **Copilot third-pass review, 2026-09-10.** Three findings against `70d2102`, delivered as summary-level + "previously missed" notes rather than inline comments. All three were real: + - **The pandas round trip widened dtypes.** `data_frame_to_pandas()` handed pandas untyped Python lists, so a + `FloatColumn` came back as `float64` and an `Int32Column` as `int64`; converting back then emitted + `DoubleColumn`/`Int64Column`. Each Series is now built with the dtype its column type implies. + - **The pandas round trip discarded all column metadata.** `data_frame_to_pandas()` populated + `df.attrs["column_metadata"]` and `data_frame_from_pandas()` ignored it, so tags, attributes, and provenance + were silently stripped -- losing exactly the record of where the numbers came from that makes calculations + worth storing, and which this ticket sells as a headline feature. Added `column_metadata_from_dict()`, the + inverse of `column_metadata_dict()`, and threaded it through. A frame now round-trips to **byte equality** + apart from the deliberate `SamplingClock` -> `TimestampList` axis change. + - The recipe claimed to be "Verified against dp-grpc `rel-1.16.0`", a tag that does not exist -- the newest + release is `rel-1.15.0`. It now separates the *target* API version (1.16.0, unreleased) from what was + actually tested (a dp-service build from `main` at `fddf692`). `doc/cookbook/sample-status.md` carries the + same overclaim and is left alone here: it is not this PR's file, and correcting it belongs with whoever + reconciles the cookbook's version banners at release time. + +- **Copilot fourth-pass review, 2026-09-10.** Three findings against `a29064d`, all real: + - **Enum columns lost their kind and their `enumId` through the pandas round trip.** `EnumColumn.values` is + int32, so leaving it out of the narrow-dtype mapping widened the codes to int64 and rebuilt the column as an + `Int64Column` -- and the `enumId`, which is the only thing saying what those codes mean, was dropped + entirely. The dtype mapping now covers it, and the id rides in `df.attrs["enum_ids"]`, carried even under + `exclude_column_metadata=True` because it is structural rather than descriptive. An enum id on a + non-integer column is now a named error instead of a silently wrong column kind. + - Two comments in `query_conversions` and `sample_status_conversions` still pointed at + `machine_config_client.to_epoch_nanos` after the move to `time_conversions.py` -- my own stale references + from `9e07226`, left behind by `4c6a8e1`. A sweep for the same mistake found two more: a stale module + attribution in this plan's own reference list, and a sentence in `CLAUDE.md` that an earlier edit had + mangled mid-clause. All four corrected. + +- **Copilot fifth-pass review, 2026-09-10.** Two findings against `94986fd`, both real: + - **A hand-built `TimestampList` bypassed the strict-ordering rule.** `timestamp_count()` checked only that it + was non-empty, so duplicate or decreasing timestamps -- which `timestamp_list()` rejects -- were accepted by + both `data_frame()` and `SampleStatusFrame`, producing a frame `data_frame_from_pandas()` then refuses. Two + samples claiming one instant is also inexpressible in the sample-status identity model. This is the third + instance of the same shape (after array dims and `periodNanos`): a builder enforces a rule that the shared + count/validate helper does not, so a pre-built message walks straight past it. + - **`ImageColumn` and `StructColumn` lost their structural fields on read.** `column_values()` yields the raw + payloads, but an image's `imageDescriptor` (width/height/channels/encoding) and a struct's `schemaId` are what + make those bytes interpretable, and neither was reachable from the conversion output. Added + `image_descriptor_dict()` / `column_schema_id()` and their frame-level companions, following the precedent + already set for array dims and enum ids rather than folding them into `column_values()`. ## Overview @@ -183,7 +311,7 @@ attributes a reader of this table might reach for do not exist. ### 4. What already exists in this repo to reuse -- `to_timestamp()` / `TimestampInput` (`machine_config_client.py`) — every `DataBlock` and `TimeRange` bound. +- `to_timestamp()` / `TimestampInput` (`time_conversions.py`) — every `DataBlock` and `TimeRange` bound. - `sampling_clock()` / `timestamp_list()` / `_timestamp_count()` (`sample_status_client.py`) — the `DataTimestamps` axis builders a calculations frame needs. They belong in a shared module now that a second caller exists; re-export from `sample_status_client` so nothing breaks. diff --git a/src/dp_python_lib/client/__init__.py b/src/dp_python_lib/client/__init__.py index 8c139c8..1d20093 100644 --- a/src/dp_python_lib/client/__init__.py +++ b/src/dp_python_lib/client/__init__.py @@ -10,6 +10,26 @@ SaveAnnotationRequestParams, calculations, ) + +# NOTE: the data_frame() *function* is deliberately NOT re-exported here. Binding that name in the package +# namespace would shadow the data_frame *module*, so both `from dp_python_lib.client import data_frame` and +# `import dp_python_lib.client.data_frame` would hand back the function instead of the module -- breaking the +# documented `from dp_python_lib.client import data_frame as dfb` usage. Reach it as dfb.data_frame(...). +from dp_python_lib.client.data_frame import ( + bool_column, + calculations_source, + column_metadata, + data_column, + double_column, + enum_column, + float_column, + int32_column, + int64_column, + provenance, + pv_source, + string_column, + timestamp_count, +) from dp_python_lib.client.dataset_client import ( DataSetClient, DataSetQuery, @@ -47,7 +67,6 @@ SaveConfigurationActivationRequestParams, SaveConfigurationApiResult, SaveConfigurationRequestParams, - to_timestamp, ) from dp_python_lib.client.mldp_client import MldpClient from dp_python_lib.client.pv_metadata_client import ( @@ -80,6 +99,7 @@ timestamp_list, ) from dp_python_lib.client.sample_status_conversions import SampleStatusRow +from dp_python_lib.client.time_conversions import TimestampInput, to_epoch_nanos, to_timestamp __all__ = [ "AnnotationClient", @@ -142,10 +162,25 @@ "SavePvMetadataRequestParams", "SaveSampleStatusesApiResult", "SaveSampleStatusesRequestParams", + "TimestampInput", + "bool_column", "calculations", + "calculations_source", "calculations_spec", + "column_metadata", "data_block", + "data_column", + "double_column", + "enum_column", + "float_column", + "int32_column", + "int64_column", + "provenance", + "pv_source", "sampling_clock", + "string_column", + "timestamp_count", "timestamp_list", + "to_epoch_nanos", "to_timestamp", ] diff --git a/src/dp_python_lib/client/annotations_client.py b/src/dp_python_lib/client/annotations_client.py index 9922609..eb3fd03 100644 --- a/src/dp_python_lib/client/annotations_client.py +++ b/src/dp_python_lib/client/annotations_client.py @@ -17,10 +17,10 @@ def calculations(frames: dict[str, common_pb2.DataFrame]) -> annotation_pb2.Calc Taking a dict rather than a list makes frame-name uniqueness true by construction; the server rejects duplicate frame names, and a list would let a caller build one. - Each frame is a common.DataFrame: a time axis plus the columns sampled on it. Assemble that message directly - for now -- the data_frame builders (dp_python_lib.client.data_frame), which validate a frame's internal shape - (column count against the time axis, unique column names, non-empty names and values) so an error names the - offending column, arrive in the follow-up PR for issue #6. + Each frame is a common.DataFrame: a time axis plus the columns sampled on it. Build one with the data_frame + builders (from dp_python_lib.client import data_frame as dfb; dfb.data_frame(...)), which validate the frame's + internal shape -- column count against the time axis, unique column names, non-empty names and values -- so an + error names the offending column. Note the proto's naming trap: the repeated field is 'calculationDataFrames' (singular "calculation") while the message it holds is 'CalculationsDataFrame' (plural). diff --git a/src/dp_python_lib/client/data_frame.py b/src/dp_python_lib/client/data_frame.py new file mode 100644 index 0000000..9ffb87b --- /dev/null +++ b/src/dp_python_lib/client/data_frame.py @@ -0,0 +1,714 @@ +""" +Builders for common.DataFrame -- the shared time-series payload shape (issue #6, Phase 2). + +A DataFrame is one time axis plus a set of columns sampled on it. It is the payload of an annotation's +Calculations, and it is also ingestion's `ingestionDataFrame`: the same message, so this module is the substrate +issue #17 extends rather than a parallel one to fork. + +Design decisions (see plan/tickets/6/plan.md, D6): + - The time axis builders sampling_clock() / timestamp_list() live here now that a second caller exists; they are + re-exported from sample_status_client so existing imports keep working. + - The axis carries its own sample count and every column is validated against it. That restates a number the + caller already has (sampling_clock(t0, period, count=len(values))), which is deliberate: deriving the count + from the columns would fork the axis API by caller, since sample_status_client genuinely knows its count + independently of any column. SampleStatusFrame set the same precedent -- explicit axis, per-column validation. + - Validation mirrors the server's SHAPE rules only (non-blank names, non-empty values, count match, column-name + uniqueness across types), so an error names the offending frame or column instead of bouncing the whole batch. + The server's resource caps -- 256-char strings, 10M-element arrays, 50 MB images, 1 MB structs -- stay + server-side: they are deployment policy, and duplicating numbers that can change is how clients drift. + - Array, image, struct, and serialized column builders are deliberately absent. Their ergonomics (dims, image + descriptors, schema ids) are #17's to design once with ingestion data in hand. Meanwhile they are reachable by + building the proto directly and passing it to data_frame(), which accepts pre-built column messages alongside + the ones its builders return. +""" + +from numbers import Integral, Real +from typing import Any + +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp +from dp_python_lib.grpc import common_pb2 + +# Each typed column message, paired with the DataFrame field it belongs in. data_frame() routes by exact type, so +# a pre-built column of any supported kind lands in the right repeated field without the caller naming it. +_COLUMN_FIELD_BY_TYPE = { + common_pb2.DoubleColumn: "doubleColumns", + common_pb2.FloatColumn: "floatColumns", + common_pb2.Int64Column: "int64Columns", + common_pb2.Int32Column: "int32Columns", + common_pb2.BoolColumn: "boolColumns", + common_pb2.StringColumn: "stringColumns", + common_pb2.EnumColumn: "enumColumns", + common_pb2.ImageColumn: "imageColumns", + common_pb2.StructColumn: "structColumns", + common_pb2.DoubleArrayColumn: "doubleArrayColumns", + common_pb2.FloatArrayColumn: "floatArrayColumns", + common_pb2.Int32ArrayColumn: "int32ArrayColumns", + common_pb2.Int64ArrayColumn: "int64ArrayColumns", + common_pb2.BoolArrayColumn: "boolArrayColumns", + common_pb2.DataColumn: "dataColumns", + common_pb2.SerializedDataColumn: "serializedDataColumns", +} + +# Columns whose per-sample count is len(values). The array columns are excluded: their values are flat, so the +# sample count is len(values) / prod(dims). Serialized columns carry no countable values at all. +_SCALAR_COLUMN_TYPES = ( + common_pb2.DoubleColumn, + common_pb2.FloatColumn, + common_pb2.Int64Column, + common_pb2.Int32Column, + common_pb2.BoolColumn, + common_pb2.StringColumn, + common_pb2.EnumColumn, + common_pb2.StructColumn, +) + +_ARRAY_COLUMN_TYPES = ( + common_pb2.DoubleArrayColumn, + common_pb2.FloatArrayColumn, + common_pb2.Int32ArrayColumn, + common_pb2.Int64ArrayColumn, + common_pb2.BoolArrayColumn, +) + + +# ---------------------------------------------------------------------- +# time axis +# ---------------------------------------------------------------------- + + +def sampling_clock( + start_time: TimestampInput, + period_nanos: int, + count: int, +) -> common_pb2.DataTimestamps: + """ + Builds a DataTimestamps with a SamplingClock time axis, the compact form for regularly-sampled data. + + When labeling existing archived data (sample status), the clock must match that data's clock exactly -- + matching is by exact timestamp at nanosecond precision, so an off-by-one-nanosecond period misses every sample + after the first. When describing derived values (calculations), the clock defines the axis, and count must + equal the length of every column on it. + + Note the server rejects a calculations axis whose startTime is epoch zero, so build fixtures from a real time + rather than datetime(1970, 1, 1). + + :param start_time: Time of the first sample (tz-aware datetime, epoch seconds, or common.Timestamp). + :param period_nanos: Period between samples, in nanoseconds. Must be > 0. + :param count: Number of samples in the interval. Must be >= 1. + :return: A DataTimestamps carrying a SamplingClock. + :raises ValueError: if period_nanos is not positive or count is less than 1. + """ + if period_nanos <= 0: + raise ValueError(f"sampling_clock() requires period_nanos > 0, got {period_nanos}") + if count < 1: + raise ValueError(f"sampling_clock() requires count >= 1, got {count}") + + timestamps = common_pb2.DataTimestamps() + timestamps.samplingClock.startTime.CopyFrom(to_timestamp(start_time)) + timestamps.samplingClock.periodNanos = period_nanos + timestamps.samplingClock.count = count + return timestamps + + +def timestamp_list(values: list[TimestampInput]) -> common_pb2.DataTimestamps: + """ + Builds a DataTimestamps with an explicit TimestampList time axis, the form for irregularly-spaced samples -- + and, for sample status, for sparse labeling that names only the samples being labeled. + + Timestamps must be strictly increasing, which is validated here rather than deferred to the server. When + labeling existing data, supply timestamps taken from query results (or exact SamplingClock arithmetic); a + recomputed or rounded timestamp will silently fail to match its sample. + + :param values: The timestamps of the samples (tz-aware datetimes, epoch seconds, or common.Timestamp objects). + :return: A DataTimestamps carrying a TimestampList. + :raises ValueError: if values is empty or the timestamps are not strictly increasing. + """ + if not values: + raise ValueError("timestamp_list() requires a non-empty values list") + + converted = [to_timestamp(value) for value in values] + + previous = converted[0] + for index, current in enumerate(converted[1:], start=1): + if (current.epochSeconds, current.nanoseconds) <= (previous.epochSeconds, previous.nanoseconds): + raise ValueError( + f"timestamp_list() requires strictly increasing timestamps; entry {index} " + f"({current.epochSeconds}.{current.nanoseconds:09d}) does not follow entry {index - 1} " + f"({previous.epochSeconds}.{previous.nanoseconds:09d})" + ) + previous = current + + timestamps = common_pb2.DataTimestamps() + timestamps.timestampList.timestamps.extend(converted) + return timestamps + + +def timestamp_count(timestamps: common_pb2.DataTimestamps) -> int: + """ + Returns the number of timestamps a DataTimestamps describes, for validating parallel-array lengths. + + An empty or malformed axis is rejected here rather than allowed to surface later as a confusing column-length + mismatch. The axis builders already make this unreachable -- sampling_clock() requires count >= 1 and a + positive period, and timestamp_list() requires a non-empty list -- but a hand-built DataTimestamps can still + carry a zero-count or zero-period SamplingClock, or an empty or out-of-order TimestampList, and all report + their oneof arm as set. Rejecting them keeps this in step with the read path -- expand_data_timestamps() for + the clock rules, data_frame_from_pandas() for the ordering one; anything data_frame() accepts must be readable + back. + + :param timestamps: The time axis to measure. + :return: The number of timestamps on the axis. + :raises ValueError: if neither axis form is set, if the axis describes no timestamps, if a SamplingClock's + periodNanos is not positive, or if a TimestampList is not strictly increasing. + """ + axis = timestamps.WhichOneof("value") + if axis == "samplingClock": + count = timestamps.samplingClock.count + if count < 1: + raise ValueError(f"DataTimestamps samplingClock requires count >= 1, got {count}") + period = timestamps.samplingClock.periodNanos + if period <= 0: + # Checked here as well as in sampling_clock(), which a hand-built axis bypasses. The read path + # (expand_data_timestamps) rejects a non-positive period, so accepting one here would build a frame + # the library cannot read back -- and every sample after the first would share the first's timestamp. + raise ValueError(f"DataTimestamps samplingClock requires periodNanos > 0, got {period}") + return count + if axis == "timestampList": + entries = timestamps.timestampList.timestamps + count = len(entries) + if count < 1: + raise ValueError("DataTimestamps timestampList requires at least one timestamp, got an empty list") + # Strict ordering, as timestamp_list() enforces -- a hand-built axis bypasses that builder entirely. + # Duplicate or decreasing timestamps mean two samples claim one instant, which the identity model cannot + # express, and data_frame_from_pandas() rejects the index they produce, so accepting them here would build + # a frame the library cannot round-trip. + previous = entries[0] + for index, current in enumerate(entries[1:], start=1): + if (current.epochSeconds, current.nanoseconds) <= (previous.epochSeconds, previous.nanoseconds): + raise ValueError( + f"DataTimestamps timestampList requires strictly increasing timestamps; entry {index} " + f"({current.epochSeconds}.{current.nanoseconds:09d}) does not follow entry {index - 1} " + f"({previous.epochSeconds}.{previous.nanoseconds:09d})" + ) + previous = current + return count + raise ValueError("DataTimestamps must specify either a samplingClock or a timestampList") + + +# ---------------------------------------------------------------------- +# provenance +# ---------------------------------------------------------------------- + + +def _time_range(bounds: tuple[TimestampInput, TimestampInput]) -> common_pb2.TimeRange: + """ + Builds a common.TimeRange from a (begin, end) pair. + + :param bounds: The (begin, end) pair, each a tz-aware datetime, epoch seconds, or common.Timestamp. + :return: The equivalent common.TimeRange. + :raises ValueError: if begin is not strictly before end. + """ + begin, end = (to_timestamp(bound) for bound in bounds) + if (begin.epochSeconds, begin.nanoseconds) >= (end.epochSeconds, end.nanoseconds): + raise ValueError( + f"time_range requires begin strictly before end; got begin " + f"{begin.epochSeconds}.{begin.nanoseconds:09d} and end {end.epochSeconds}.{end.nanoseconds:09d}" + ) + time_range = common_pb2.TimeRange() + time_range.beginTime.CopyFrom(begin) + time_range.endTime.CopyFrom(end) + return time_range + + +def pv_source( + pv_name: str, + time_range: tuple[TimestampInput, TimestampInput] | None = None, +) -> common_pb2.ColumnProvenance.ColumnSource: + """ + Builds a ColumnSource naming an archived PV a calculated column was derived from. + + :param pv_name: Name of the source PV. + :param time_range: Optional (begin, end) pair narrowing which part of the PV's history was used. + :return: A ColumnProvenance.ColumnSource with its pvName origin set. + :raises ValueError: if pv_name is empty, or if the time range's begin is not before its end. + """ + if not pv_name: + raise ValueError("pv_source() requires a non-empty pv_name") + + source = common_pb2.ColumnProvenance.ColumnSource() + source.pvName = pv_name + if time_range is not None: + source.timeRange.CopyFrom(_time_range(time_range)) + return source + + +def calculations_source( + calculations_id: str, + frame_name: str, + column_name: str, + time_range: tuple[TimestampInput, TimestampInput] | None = None, +) -> common_pb2.ColumnProvenance.ColumnSource: + """ + Builds a ColumnSource naming another calculations column this one was derived from. + + These are soft references: deleting the annotation that owns the referenced calculations leaves this link + dangling, which readers are expected to tolerate rather than resolve. + + :param calculations_id: Id of the calculations object holding the source column. + :param frame_name: Name of the frame within those calculations. + :param column_name: Name of the column within that frame. + :param time_range: Optional (begin, end) pair narrowing which part of the source column was used. + :return: A ColumnProvenance.ColumnSource with its calculationsColumn origin set. + :raises ValueError: if any of the three identifiers is empty, or if the time range is reversed. + """ + if not calculations_id: + raise ValueError("calculations_source() requires a non-empty calculations_id") + if not frame_name: + raise ValueError("calculations_source() requires a non-empty frame_name") + if not column_name: + raise ValueError("calculations_source() requires a non-empty column_name") + + source = common_pb2.ColumnProvenance.ColumnSource() + source.calculationsColumn.calculationsId = calculations_id + source.calculationsColumn.frameName = frame_name + source.calculationsColumn.columnName = column_name + if time_range is not None: + source.timeRange.CopyFrom(_time_range(time_range)) + return source + + +def provenance( + source: str | None = None, + process: str | None = None, + derived_from: list[common_pb2.ColumnProvenance.ColumnSource] | None = None, +) -> common_pb2.ColumnProvenance: + """ + Builds a ColumnProvenance recording where a column's values came from and how they were produced. + + :param source: Free-text name of the producing system or person. + :param process: Free-text description of the computation (e.g. "1 Hz RMS"). + :param derived_from: The inputs this column was computed from (see pv_source() / calculations_source()). + :return: A common.ColumnProvenance. + """ + result = common_pb2.ColumnProvenance() + if source: + result.source = source + if process: + result.process = process + if derived_from: + result.derivedFrom.extend(derived_from) + return result + + +def column_metadata( + tags: list[str] | None = None, + attributes: dict[str, str] | None = None, + provenance: common_pb2.ColumnProvenance | None = None, +) -> common_pb2.ColumnMetadata: + """ + Builds a ColumnMetadata: per-column tags, attributes, and provenance. + + :param tags: Tags (keywords) describing the column. + :param attributes: Map of key/value attributes describing the column. + :param provenance: Where the column's values came from (see provenance()). + :return: A common.ColumnMetadata. + """ + metadata = common_pb2.ColumnMetadata() + if provenance is not None: + metadata.provenance.CopyFrom(provenance) + if tags: + metadata.tags[:] = tags + if attributes: + for name, value in attributes.items(): + attribute = metadata.attributes.add() + attribute.name = name + attribute.value = value + return metadata + + +# ---------------------------------------------------------------------- +# typed scalar columns +# ---------------------------------------------------------------------- + + +def _build_scalar_column( + column_class: Any, + builder_name: str, + name: str, + values: list, + metadata: common_pb2.ColumnMetadata | None, +) -> Any: + """ + Builds one typed scalar column, applying the shape rules every column type shares. + + :param column_class: The typed column message class to construct. + :param builder_name: The public builder's name, used in error messages. + :param name: The column's name. + :param values: The column's values, one per sample. + :param metadata: Optional per-column metadata. + :return: The constructed typed column message. + :raises ValueError: if name is empty or values is empty. + """ + if not name or not name.strip(): + raise ValueError(f"{builder_name}() requires a non-blank name, got {name!r}") + if not values: + raise ValueError(f"{builder_name}() requires a non-empty values list for column '{name}'") + + column = column_class() + column.name = name + column.values[:] = values + if metadata is not None: + column.metadata.CopyFrom(metadata) + return column + + +def double_column( + name: str, values: list[float], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.DoubleColumn: + """ + Builds a DoubleColumn (float64) with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One float per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.DoubleColumn. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.DoubleColumn, "double_column", name, values, metadata) + + +def float_column( + name: str, values: list[float], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.FloatColumn: + """ + Builds a FloatColumn (float32) with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One float per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.FloatColumn. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.FloatColumn, "float_column", name, values, metadata) + + +def int64_column( + name: str, values: list[int], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.Int64Column: + """ + Builds an Int64Column with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One int per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.Int64Column. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.Int64Column, "int64_column", name, values, metadata) + + +def int32_column( + name: str, values: list[int], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.Int32Column: + """ + Builds an Int32Column with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One int per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.Int32Column. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.Int32Column, "int32_column", name, values, metadata) + + +def bool_column( + name: str, values: list[bool], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.BoolColumn: + """ + Builds a BoolColumn with one value per sample. + + :param name: The column's name, unique within its frame. + :param values: One bool per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.BoolColumn. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.BoolColumn, "bool_column", name, values, metadata) + + +def string_column( + name: str, values: list[str], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.StringColumn: + """ + Builds a StringColumn with one value per sample. + + The server caps individual string values (256 characters at the time of writing); that limit is deployment + policy and is deliberately not duplicated here. + + :param name: The column's name, unique within its frame. + :param values: One string per sample on the frame's time axis. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.StringColumn. + :raises ValueError: if name or values is empty. + """ + return _build_scalar_column(common_pb2.StringColumn, "string_column", name, values, metadata) + + +def enum_column( + name: str, values: list[int], enum_id: str, metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.EnumColumn: + """ + Builds an EnumColumn: integer codes plus the id of the enumeration that gives them meaning. + + :param name: The column's name, unique within its frame. + :param values: One integer code per sample on the frame's time axis. + :param enum_id: Id of the enumeration defining what the codes mean. + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.EnumColumn. + :raises ValueError: if name, values, or enum_id is empty. + """ + if not enum_id: + raise ValueError("enum_column() requires a non-empty enum_id") + column = _build_scalar_column(common_pb2.EnumColumn, "enum_column", name, values, metadata) + column.enumId = enum_id + return column + + +def _is_numpy_bool(value: Any) -> bool: + """ + True if a value is a NumPy boolean scalar. + + NumPy's boolean scalar is a subclass of neither bool nor numbers.Integral, so nothing else in data_column()'s + type mapping catches it -- while np.float64 IS a subclass of float and np.int64 IS numbers.Integral. Left + unhandled it would be the one NumPy scalar the mapping rejected. + + Identified by its type's module and name rather than by importing NumPy, which keeps this module free of the + optional [analysis] dependency. Both spellings are matched: the type is named `bool_` under NumPy 1.x and + `bool` under 2.x, and the module check is what keeps the latter from matching an unrelated class. + + :param value: The value to test. + :return: True if the value is a numpy.bool_ / numpy.bool scalar. + """ + value_type = type(value) + return value_type.__module__ == "numpy" and value_type.__name__ in ("bool_", "bool") + + +def data_column( + name: str, values: list[Any], metadata: common_pb2.ColumnMetadata | None = None +) -> common_pb2.DataColumn: + """ + Builds a legacy DataColumn of per-sample DataValues -- the escape hatch for a column that needs gaps. + + The typed columns above are dense: every sample has a value, because the repeated field carries no notion of a + missing entry. A DataColumn's values are DataValue messages, whose oneof can be left unset, so a None entry + here becomes a genuinely absent value. That makes this the only way to express a gap on a shared time axis + short of giving the sparse column its own frame. + + Values are mapped to DataValue arms by Python type: bool -> booleanValue (checked before int, since bool is a + subclass of int), int -> longValue, float -> doubleValue, str -> stringValue, bytes -> byteArrayValue, and + None -> an unset oneof. Anything else raises: silently coercing an unexpected type would store something the + caller did not mean. Pass a pre-built DataValue to use an arm this mapping does not cover. + + The integer and float arms are matched by numbers.Integral / numbers.Real, and np.bool_ by name, so NumPy + scalars work too. np.int64 is not a subclass of int and np.bool_ is not a subclass of bool (while np.float64 + IS a subclass of float), so mapping by exact Python type would accept some NumPy scalars and reject others -- + an arbitrary distinction for a caller coming from pandas or NumPy. + + :param name: The column's name, unique within its frame. + :param values: One value per sample, where None means "no value for this sample". + :param metadata: Optional per-column tags, attributes, and provenance (see column_metadata()). + :return: A common.DataColumn. + :raises ValueError: if name or values is empty, or if a value's type has no DataValue mapping. + """ + if not name or not name.strip(): + raise ValueError(f"data_column() requires a non-blank name, got {name!r}") + if not values: + raise ValueError(f"data_column() requires a non-empty values list for column '{name}'") + + column = common_pb2.DataColumn() + column.name = name + for index, value in enumerate(values): + data_value = column.dataValues.add() + if value is None: + continue + if isinstance(value, common_pb2.DataValue): + data_value.CopyFrom(value) + elif isinstance(value, bool) or _is_numpy_bool(value): + # Checked before the integer branch: bool is a subclass of int, so that branch would swallow it. + data_value.booleanValue = bool(value) + elif isinstance(value, Integral): + # Integral rather than int, so a NumPy integer scalar maps like a Python one. + data_value.longValue = int(value) + elif isinstance(value, Real): + # Real rather than float, for symmetry with the integer branch above. + data_value.doubleValue = float(value) + elif isinstance(value, str): + data_value.stringValue = value + elif isinstance(value, bytes): + data_value.byteArrayValue = value + else: + raise ValueError( + f"data_column() cannot map value at index {index} of column '{name}' " + f"(type {type(value).__name__}) to a DataValue; pass a pre-built common.DataValue instead" + ) + + if metadata is not None: + column.metadata.CopyFrom(metadata) + return column + + +# ---------------------------------------------------------------------- +# frame assembly +# ---------------------------------------------------------------------- + + +def _array_sample_size(column: Any) -> int | None: + """ + Returns an array column's per-sample size, prod(dims), or None when it cannot be derived. + + Absent dims are underivable rather than a product of 1 -- an empty product would silently treat every element + as its own sample. A zero or negative dim is equally unusable, so both report as None for the caller to + reject with a message naming the column. + + :param column: An array column (DoubleArrayColumn, Int32ArrayColumn, ...). + :return: The number of values each sample occupies, or None if dims are missing or non-positive. + """ + dims = list(column.dimensions.dims) + if not dims: + return None + product = 1 + for dim in dims: + product *= dim + if product <= 0: + return None + return product + + +def _column_sample_count(column: Any) -> int | None: + """ + Returns the number of samples a column carries, or None when the column has no countable per-sample values. + + For an array column the count is len(values) / prod(dims); a value count that is not a whole multiple of + prod(dims) has no sample count at all and reports as None, so _check_column() rejects it rather than letting + floor division round it into a passing count. The read path (data_frame_conversions._reshape_array_values) + applies the same rule, and a frame this accepted but that could not be read back would be the worst outcome. + + An ImageColumn counts its `images`, and a DataColumn its `dataValues`; only the scalar and array columns keep + their per-sample entries in `values`. + + :param column: A typed column, a legacy DataColumn, an ImageColumn, or a SerializedDataColumn. + :return: The sample count, or None for a SerializedDataColumn (whose payload is opaque) and for an array + column whose dims are missing, non-positive, or do not evenly divide its values. + """ + if isinstance(column, common_pb2.SerializedDataColumn): + return None + if isinstance(column, common_pb2.DataColumn): + return len(column.dataValues) + if isinstance(column, common_pb2.ImageColumn): + # ImageColumn holds one encoded payload per sample in `images`; it has no `values` field at all, so the + # generic path below would raise AttributeError on a column this function claims to support. + return len(column.images) + if isinstance(column, _ARRAY_COLUMN_TYPES): + product = _array_sample_size(column) + if product is None: + return None + if len(column.values) % product != 0: + return None + return len(column.values) // product + return len(column.values) + + +def _check_column(column: Any, index: int, expected_count: int, seen_names: set[str]) -> None: + """ + Applies the server's per-column shape rules to one column, with a message naming the column. + + :param column: The column to check. + :param index: The column's position in the caller's list, for messages about an unnamed column. + :param expected_count: The frame's sample count, which every counted column must match. + :param seen_names: Names already used in this frame; mutated to record this column's name. + :raises ValueError: if the column is of an unsupported type, unnamed, empty, duplicate-named, or + count-mismatched. + """ + if type(column) not in _COLUMN_FIELD_BY_TYPE: + raise ValueError( + f"data_frame() received an unsupported column type at index {index}: {type(column).__name__}. " + f"Use one of the column builders, or pass a pre-built typed column message." + ) + + name = column.name + if not name or not name.strip(): + # Blank, not merely empty: the rule is a non-blank name, and a whitespace-only one is not a name. + raise ValueError( + f"data_frame() requires a non-blank name for every column; column at index {index} has {name!r}" + ) + if name in seen_names: + raise ValueError( + f"data_frame() requires unique column names within a frame; '{name}' appears more than once " + f"(names must be unique across ALL column types, not just within one type)" + ) + seen_names.add(name) + + if isinstance(column, common_pb2.SerializedDataColumn): + # Serialized payloads are opaque; the server checks their names only. + return + + if isinstance(column, _ARRAY_COLUMN_TYPES): + product = _array_sample_size(column) + if product is None: + raise ValueError( + f"data_frame() cannot determine the sample count of array column '{name}': its dimensions are " + f"missing or zero. Set ArrayDimensions.dims so that values is samples x prod(dims)." + ) + if len(column.values) % product != 0: + raise ValueError( + f"data_frame() array column '{name}' has {len(column.values)} values, which is not a whole " + f"multiple of its per-sample size {product} (from dims {list(column.dimensions.dims)}); " + f"values must be samples x prod(dims)" + ) + + count = _column_sample_count(column) + if count == 0: + raise ValueError(f"data_frame() requires a non-empty values list for column '{name}'") + if count != expected_count: + raise ValueError( + f"data_frame() column '{name}' has {count} values but the time axis has {expected_count} timestamps; " + f"every column must carry exactly one value per sample" + ) + + +def data_frame( + data_timestamps: common_pb2.DataTimestamps, + columns: list[Any], +) -> common_pb2.DataFrame: + """ + Assembles a common.DataFrame from a time axis and a list of columns, routing each column into the repeated + field for its type. + + Accepts the columns this module builds and pre-built column messages alike, so the kinds without builders yet + (arrays, images, structs, serialized payloads -- issue #17) can be passed straight through. + + Validates the server's shape rules client-side, so a mistake names the offending column instead of bouncing the + whole save: at least one column, a non-blank name and non-empty values for each, a value count matching the + axis (for arrays, values / prod(dims)), and column names unique across ALL types within the frame. The + server's size caps are not duplicated -- see the module docstring. + + :param data_timestamps: The frame's time axis (see sampling_clock() / timestamp_list()). + :param columns: The frame's columns, in any mix of supported types. + :return: A common.DataFrame. + :raises ValueError: if the axis is empty or malformed, if columns is empty, or if any column violates a shape + rule. + """ + if not columns: + raise ValueError("data_frame() requires at least one column") + + expected_count = timestamp_count(data_timestamps) + + seen_names: set[str] = set() + for index, column in enumerate(columns): + _check_column(column, index, expected_count, seen_names) + + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(data_timestamps) + for column in columns: + getattr(frame, _COLUMN_FIELD_BY_TYPE[type(column)]).append(column) + return frame diff --git a/src/dp_python_lib/client/data_frame_conversions.py b/src/dp_python_lib/client/data_frame_conversions.py new file mode 100644 index 0000000..e921a9f --- /dev/null +++ b/src/dp_python_lib/client/data_frame_conversions.py @@ -0,0 +1,786 @@ +""" +Pythonic conversions for common.DataFrame (issue #6, Phase 2 and 3). + +Reads a DataFrame -- an annotation's calculations, and in future a bucket query's typed columns -- back into plain +Python, and, behind the optional [analysis] extra, into pandas. + +The pure-Python half has no third-party dependencies. pandas is imported lazily inside each entry point that needs +it, so importing this module never requires the extra. + +Design decisions (see plan/tickets/6/plan.md, D7): + - Timestamps are computed in INTEGER NANOSECONDS via expand_data_timestamps(), never in float seconds. A float64 + carries 53 bits of mantissa and present-day epoch nanoseconds need ~61, so a float round-trip would silently + move every timestamp. The pandas index is built from those int64 nanoseconds directly for the same reason. + - column_values() is written as a standalone per-column converter because the bucket query (#16) reuses these + same 14 typed column messages; it takes one column and needs to know nothing about the frame. + - Array columns are reshaped into one list per sample using their declared dims, rather than returned flat: a + flat list would silently lose the sample boundaries. + - Dense typed columns cannot express a gap, so the pandas->DataFrame direction rejects NaN/None fail-loud with a + message pointing at the two ways to express one (a separate frame, or data_column()). +""" + +from typing import Any + +from dp_python_lib.client.query_conversions import data_value_to_python +from dp_python_lib.client.sample_status_conversions import expand_data_timestamps +from dp_python_lib.client.time_conversions import to_epoch_nanos +from dp_python_lib.grpc import annotation_pb2, common_pb2 + +# The DataFrame fields holding typed scalar columns, in the proto's declaration order. Each carries `values` +# directly parallel to the time axis. +_SCALAR_COLUMN_FIELDS = ( + "doubleColumns", + "floatColumns", + "int64Columns", + "int32Columns", + "boolColumns", + "stringColumns", + "enumColumns", + "structColumns", +) + +# The DataFrame fields holding array columns, whose `values` are flat (samples x prod(dims)). +_ARRAY_COLUMN_FIELDS = ( + "doubleArrayColumns", + "floatArrayColumns", + "int32ArrayColumns", + "int64ArrayColumns", + "boolArrayColumns", +) + +# ImageColumn stores its per-sample payloads in `images` rather than `values`. +_IMAGE_COLUMN_FIELD = "imageColumns" + +# The array column message types, whose flat values are delimited by their declared dims. +_ARRAY_COLUMN_MESSAGE_TYPES = ( + common_pb2.DoubleArrayColumn, + common_pb2.FloatArrayColumn, + common_pb2.Int32ArrayColumn, + common_pb2.Int64ArrayColumn, + common_pb2.BoolArrayColumn, +) + + +def _require_pandas(): + """Imports and returns pandas, or raises an actionable error if the optional [analysis] extra is missing.""" + try: + import pandas + except ImportError as e: + raise ImportError( + "pandas is required for DataFrame conversions. Install the optional analysis extra: " + 'pip install "dp-python-lib[analysis]"' + ) from e + return pandas + + +def data_frame_timestamps(frame: common_pb2.DataFrame) -> list[int]: + """ + Expands a frame's time axis into one integer epoch-nanosecond value per sample. + + A SamplingClock is expanded arithmetically in integer nanoseconds, so the result reproduces the producer's + timestamps exactly; see expand_data_timestamps(). + + :param frame: The DataFrame whose axis to expand. + :return: Epoch nanoseconds for each sample, in axis order. + :raises ValueError: if the frame has no time axis, or the axis is empty or malformed. + """ + if not frame.HasField("dataTimestamps"): + raise ValueError("DataFrame has no dataTimestamps; every frame must carry a time axis") + + epoch_nanos = expand_data_timestamps(frame.dataTimestamps) + if not epoch_nanos: + # expand_data_timestamps() returns [] for a set-but-empty timestampList, which reports its oneof arm as + # set. Rejecting it here honors this function's documented contract and matches timestamp_count() on the + # write side; otherwise a corrupt frame converts to a zero-row table instead of failing loudly. + raise ValueError("DataFrame time axis describes no timestamps; every frame must cover at least one sample") + return epoch_nanos + + +def _reshape_array_values(column: Any) -> list[list]: + """ + Reshapes an array column's flat values into one list per sample, using its declared dimensions. + + :param column: An array column (DoubleArrayColumn, Int32ArrayColumn, ...). + :return: One list of values per sample; multi-dimensional arrays stay flat WITHIN a sample. + :raises ValueError: if dims are missing/zero, or the value count is not a whole multiple of prod(dims). + """ + # Absent dims are underivable rather than an empty product of 1, which would silently make every element its + # own sample. + dims = list(column.dimensions.dims) + product = 1 + for dim in dims: + product *= dim + if not dims or product <= 0: + raise ValueError( + f"array column '{column.name}' has missing or zero dimensions, so its samples cannot be delimited" + ) + + values = list(column.values) + if len(values) % product != 0: + raise ValueError( + f"array column '{column.name}' has {len(values)} values, which is not a whole multiple of its " + f"per-sample size {product} (from dims {list(column.dimensions.dims)})" + ) + return [values[i : i + product] for i in range(0, len(values), product)] + + +def column_dimensions(column: Any) -> list[int] | None: + """ + Returns an array column's declared dimensions, or None for any other column kind. + + column_values() reshapes an array column into one flat list per sample, which delimits the samples but does + not preserve the shape WITHIN one: a 2x2 sample and a 4-element sample both come back as four values. Plan + D7 calls for the dims to travel alongside those values, and this is the accessor that supplies them, kept + separate so column_values()'s "one entry per sample" contract stays uniform across all column kinds. + + :param column: Any column message. + :return: The dims as a list of ints for an array column, or None if the column is not an array column. + """ + if not isinstance(column, _ARRAY_COLUMN_MESSAGE_TYPES): + return None + return list(column.dimensions.dims) + + +def data_frame_column_dimensions(frame: common_pb2.DataFrame) -> dict[str, list[int]]: + """ + Returns the declared dimensions of every array column in a frame, keyed by column name. + + Pairs with data_frame_columns(): that gives one flat list per sample, this gives the shape those values have. + Non-array columns are absent from the result rather than mapped to None, so a caller can test membership. + + :param frame: The DataFrame to inspect. + :return: A dict mapping each array column's name to its dims; empty when the frame has no array columns. + """ + dimensions: dict[str, list[int]] = {} + for column in iter_frame_columns(frame): + dims = column_dimensions(column) + if dims is not None: + dimensions[column.name] = dims + return dimensions + + +def image_descriptor_dict(column: Any) -> dict[str, Any] | None: + """ + Returns an ImageColumn's descriptor -- width, height, channels, encoding -- as a plain dict, or None for any + other column kind. + + column_values() yields one encoded payload per sample, which nothing can decode on its own: the descriptor is + what says how to interpret those bytes. It is kept separate for the same reason column_dimensions() is, so + that column_values() stays "exactly one entry per sample" across every column kind. + + :param column: Any column message. + :return: A dict with 'width', 'height', 'channels', and 'encoding' for an ImageColumn, else None. + """ + if not isinstance(column, common_pb2.ImageColumn): + return None + descriptor = column.imageDescriptor + return { + "width": descriptor.width, + "height": descriptor.height, + "channels": descriptor.channels, + "encoding": descriptor.encoding, + } + + +def column_schema_id(column: Any) -> str | None: + """ + Returns a StructColumn's schemaId, or None for any other column kind. + + A struct column's values are opaque serialized payloads; the schema id is what names the schema needed to + interpret them, so the values alone are not usable without it. + + :param column: Any column message. + :return: The schemaId for a StructColumn, else None. + """ + if not isinstance(column, common_pb2.StructColumn): + return None + return column.schemaId + + +def data_frame_image_descriptors(frame: common_pb2.DataFrame) -> dict[str, dict[str, Any]]: + """ + Returns every ImageColumn's descriptor in a frame, keyed by column name. + + Pairs with data_frame_columns(), which gives the payloads these describe. Non-image columns are absent. + + :param frame: The frame to inspect. + :return: A dict of column name -> descriptor dict; empty when the frame has no image columns. + """ + return {column.name: image_descriptor_dict(column) for column in frame.imageColumns} + + +def data_frame_schema_ids(frame: common_pb2.DataFrame) -> dict[str, str]: + """ + Returns every StructColumn's schemaId in a frame, keyed by column name. + + Pairs with data_frame_columns(), which gives the payloads the schema describes. Non-struct columns are absent. + + :param frame: The frame to inspect. + :return: A dict of column name -> schemaId; empty when the frame has no struct columns. + """ + return {column.name: column.schemaId for column in frame.structColumns} + + +def column_values(column: Any) -> list: + """ + Extracts one Python value per sample from any supported column message. + + Written as a standalone converter because the bucket query (#16) carries the same 14 typed column messages and + can reuse this without going through a DataFrame. + + Mapping: typed scalar columns yield their native values; an EnumColumn yields its integer codes (the enumeration + naming them is `enumId` on the column); an array column yields one list per sample; an ImageColumn yields one + bytes payload per sample; and a legacy DataColumn is converted per value by data_value_to_python(), so an unset + oneof becomes None -- the only representation of a gap in this API. + + Every column kind here yields exactly one entry per sample. The structural information some kinds carry + alongside their values lives in companion accessors rather than being folded into the result: an array + column's dims in column_dimensions(), an image column's descriptor in image_descriptor_dict(), and a struct + column's schema id in column_schema_id(). Those payloads cannot be interpreted without them. + + :param column: A typed column, a legacy DataColumn, or an ImageColumn. + :return: One value per sample, in axis order. + :raises ValueError: if the column type is unsupported, or an array column's dims do not divide its values. + """ + if isinstance(column, common_pb2.DataColumn): + return [data_value_to_python(value) for value in column.dataValues] + if isinstance(column, common_pb2.ImageColumn): + return list(column.images) + if isinstance(column, _ARRAY_COLUMN_MESSAGE_TYPES): + return _reshape_array_values(column) + if hasattr(column, "values"): + return list(column.values) + raise ValueError(f"unsupported column type for value extraction: {type(column).__name__}") + + +def iter_frame_columns(frame: common_pb2.DataFrame): + """ + Yields every column in a frame, across all of its repeated fields, in a stable order. + + Serialized columns are skipped: their payloads are opaque encoded blobs that this library does not decode + (deferred, as in query_conversions). A caller needing them can read frame.serializedDataColumns directly. + + :param frame: The DataFrame to walk. + :return: An iterator over the frame's column messages. + """ + for field in (*_SCALAR_COLUMN_FIELDS, _IMAGE_COLUMN_FIELD, *_ARRAY_COLUMN_FIELDS): + yield from getattr(frame, field) + yield from frame.dataColumns + + +def data_frame_columns(frame: common_pb2.DataFrame) -> dict[str, list]: + """ + Converts every column in a frame into a dict of column name -> one Python value per sample. + + Column names are unique across all types within a frame (the server enforces it on save, and data_frame() + checks it on the way out), so keying by name is lossless. A frame that violates it anyway -- hand-built, or + from an older server -- raises rather than silently dropping the earlier column. + + :param frame: The DataFrame to convert. + :return: A dict mapping column name to its per-sample values. + :raises ValueError: if two columns share a name, or a column cannot be converted. + """ + columns: dict[str, list] = {} + for column in iter_frame_columns(frame): + if column.name in columns: + raise ValueError( + f"DataFrame carries more than one column named '{column.name}'; names must be unique across all " + f"column types within a frame" + ) + columns[column.name] = column_values(column) + return columns + + +def _column_source_dict(entry: common_pb2.ColumnProvenance.ColumnSource) -> dict[str, Any]: + """ + Summarizes one ColumnProvenance.ColumnSource -- where a column's values came from -- as a plain dict. + + Only the origin arm actually set is present: a pvName source has no 'calculations_column' key and vice versa, + rather than the unset arm standing in as an empty string. Absence means "not this arm", the same absent-vs- + empty discipline the sample status conversions apply to confidence and reason. + + The optional timeRange -- which part of the source was used -- is reported as epoch nanoseconds, matching this + module's integer-nanosecond convention. It is the substantive half of provenance for a derived column: the + source PV without its window says much less than the pair does. + + :param entry: The ColumnSource to summarize. + :return: A dict carrying the set origin arm and, when present, 'time_range' as (begin_nanos, end_nanos). + """ + result: dict[str, Any] = {} + + origin = entry.WhichOneof("origin") + if origin == "pvName": + result["pv_name"] = entry.pvName + elif origin == "calculationsColumn": + result["calculations_column"] = { + "calculations_id": entry.calculationsColumn.calculationsId, + "frame_name": entry.calculationsColumn.frameName, + "column_name": entry.calculationsColumn.columnName, + } + + if entry.HasField("timeRange"): + result["time_range"] = ( + to_epoch_nanos(entry.timeRange.beginTime), + to_epoch_nanos(entry.timeRange.endTime), + ) + return result + + +def column_metadata_dict(column: Any) -> dict[str, Any]: + """ + Summarizes a column's ColumnMetadata as a plain dict, for carrying alongside converted values. + + :param column: A column message that may have a `metadata` field. + :return: A dict with 'tags', 'attributes', and 'provenance' keys; empty containers when unset. + """ + metadata = getattr(column, "metadata", None) + if metadata is None: + return {"tags": [], "attributes": {}, "provenance": None} + + provenance = None + if metadata.HasField("provenance"): + source = metadata.provenance + provenance = { + "source": source.source, + "process": source.process, + "derived_from": [_column_source_dict(entry) for entry in source.derivedFrom], + } + + return { + "tags": list(metadata.tags), + "attributes": {attribute.name: attribute.value for attribute in metadata.attributes}, + "provenance": provenance, + } + + +def _check_frame_alignment(frame: common_pb2.DataFrame, columns: dict[str, list], n_rows: int) -> None: + """ + Rejects a frame whose columns do not all match its time axis. + + The server validates this on save, so a mismatch here means a hand-built frame or a corrupt record; failing + loudly beats producing a silently truncated or NaN-padded table. + + :param frame: The frame being converted, named in the error message. + :param columns: The already-converted columns. + :param n_rows: The number of timestamps on the axis. + :raises ValueError: if any column's length differs from n_rows. + """ + for name, values in columns.items(): + if len(values) != n_rows: + raise ValueError( + f"DataFrame column '{name}' has {len(values)} values but the time axis has {n_rows} timestamps; " + f"columns must be dense and index-aligned with the axis" + ) + + +# The protobuf column types whose pandas dtype is narrower than what inference from a plain Python list yields. +# Double/Int64/Bool are listed too so the mapping is explicit rather than half-stated. +_NARROW_DTYPE_BY_COLUMN_TYPE = { + common_pb2.FloatColumn: "float32", + common_pb2.Int32Column: "int32", + common_pb2.DoubleColumn: "float64", + common_pb2.Int64Column: "int64", + common_pb2.BoolColumn: "bool", + # EnumColumn's values field is int32 too; without this its codes widen to int64 and it rebuilds as an + # Int64Column. The enumId that gives those codes meaning travels separately -- see _enum_ids(). + common_pb2.EnumColumn: "int32", +} + + +def _narrow_column_dtypes(frame: common_pb2.DataFrame) -> dict[str, str]: + """ + Maps each scalar column's name to the pandas dtype its protobuf type implies. + + Only the columns whose type pins a dtype appear; strings, arrays, images, and DataColumns are absent, so the + caller leaves those to pandas' inference. + + :param frame: The frame whose columns to inspect. + :return: A dict of column name -> pandas dtype string. + """ + return { + column.name: _NARROW_DTYPE_BY_COLUMN_TYPE[type(column)] + for column in iter_frame_columns(frame) + if type(column) in _NARROW_DTYPE_BY_COLUMN_TYPE + } + + +def _enum_ids(frame: common_pb2.DataFrame) -> dict[str, str]: + """ + Maps each EnumColumn's name to its enumId. + + An enum column's values are bare integer codes; the enumId is what says which enumeration names them, so + dropping it would leave the codes meaningless. It has nowhere to live in a pandas Series, so it rides in + df.attrs["enum_ids"] and is consulted when rebuilding the column. + + :param frame: The frame whose enum columns to inspect. + :return: A dict of column name -> enumId; empty when the frame has no enum columns. + """ + return {column.name: column.enumId for column in frame.enumColumns} + + +def data_frame_to_pandas(frame: common_pb2.DataFrame, exclude_column_metadata: bool = False) -> Any: + """ + Converts a DataFrame into a pandas DataFrame with a UTC DatetimeIndex. + + Requires the optional [analysis] extra. + + The index is built from int64 epoch nanoseconds directly, so a SamplingClock axis stays exact -- routing + through float seconds would move present-day timestamps by hundreds of nanoseconds. Per-column ColumnMetadata + lands in df.attrs["column_metadata"] (a dict keyed by column name), matching query_conversions' convention. + + Note columns come back GROUPED BY TYPE, not in their original order: a DataFrame stores each column kind in its + own repeated field, so the wire format does not preserve a single ordering across kinds. A round trip through + data_frame_from_pandas() therefore preserves every value, dtype, and timestamp, but can reorder the columns + (float64, then int, then bool, then string, matching the proto's field order). Select by name, or reindex to + the order you want. + + :param frame: The DataFrame to convert. + :param exclude_column_metadata: When True, skip populating df.attrs["column_metadata"]. + :return: A pandas.DataFrame indexed by UTC timestamp. + :raises ImportError: if the [analysis] extra is not installed. + :raises ValueError: if the frame's axis is missing/empty, or a column is not aligned with it. + """ + pd = _require_pandas() + + epoch_nanos = data_frame_timestamps(frame) + columns = data_frame_columns(frame) + _check_frame_alignment(frame, columns, len(epoch_nanos)) + + index = pd.DatetimeIndex(pd.to_datetime(pd.Series(epoch_nanos, dtype="int64"), unit="ns", utc=True)) + + # Build each Series with the dtype its protobuf column type implies. Handing pandas an untyped list widens + # FloatColumn to float64 and Int32Column to int64, so a frame -> pandas -> frame round trip would come back + # as DoubleColumn/Int64Column. Columns with no narrow equivalent (strings, arrays, images, DataColumn) are + # left to pandas' own inference. + narrow_dtypes = _narrow_column_dtypes(frame) + df = pd.DataFrame( + { + name: pd.Series(values, index=index, dtype=narrow_dtypes[name]) if name in narrow_dtypes else values + for name, values in columns.items() + }, + index=index, + ) + + # enum_ids is structural, not metadata: without it an enum column cannot be rebuilt as one at all, so it is + # carried even when exclude_column_metadata drops the descriptive attrs. + enum_ids = _enum_ids(frame) + if enum_ids: + df.attrs["enum_ids"] = enum_ids + + if not exclude_column_metadata: + df.attrs["column_metadata"] = { + column.name: column_metadata_dict(column) for column in iter_frame_columns(frame) + } + return df + + +def _timestamps_from_index(index: Any) -> common_pb2.DataTimestamps: + """ + Builds a DataTimestamps from a pandas DatetimeIndex, in integer nanoseconds throughout. + + Always emits a TimestampList rather than trying to detect a regular interval and emit a SamplingClock. A clock + is only correct if the spacing is exactly uniform at nanosecond precision, and inferring that from an index + that merely looks regular would quietly change the timestamps -- the one thing this API cannot tolerate. Build + a clock explicitly with sampling_clock() when that is what the data is. + + Each instant is read as Timestamp.value, which is nanoseconds whatever the index's own storage unit; a raw + int64 view is in that unit, so a datetime64[us] index would land 1000x too early. NaT is rejected rather than + passed through, since its integer form is a valid-looking instant. + + :param index: A pandas DatetimeIndex. + :return: A DataTimestamps carrying a TimestampList. + :raises ValueError: if the index is empty, not a DatetimeIndex, or not strictly increasing. + """ + pd = _require_pandas() + + if not isinstance(index, pd.DatetimeIndex): + raise ValueError( + f"DataFrame must be indexed by a pandas DatetimeIndex to become a time axis, got {type(index).__name__}" + ) + if len(index) == 0: + raise ValueError("DataFrame must have at least one row to become a DataFrame with a time axis") + + # tz-naive input is ambiguous: it could be UTC or local. Require the caller to say, as to_timestamp() does. + if index.tz is None: + raise ValueError( + "DataFrame index must be timezone-aware so its instants are unambiguous; localize it first, " + "e.g. df.index = df.index.tz_localize('UTC')" + ) + + # NaT has an integer representation (the int64 minimum), so a raw integer view would serialize it as a real + # instant. Reject it before converting: an absent timestamp is not a time axis position. + if index.isna().any(): + missing = [int(position) for position in index.isna().nonzero()[0][:5]] + raise ValueError( + f"DataFrame index contains NaT at row position(s) {missing}; every row must have a real timestamp to " + f"become a time axis. Drop those rows, or supply the timestamps they should carry." + ) + + # Timestamp.value is always nanoseconds regardless of the index's own storage unit. A raw int64 view is NOT: + # pandas 2+ keeps second, millisecond, and microsecond resolutions, and viewing a datetime64[us] index as + # int64 yields microseconds -- silently placing every instant 1000x too early. + epoch_nanos = [entry.value for entry in index] + + timestamps = common_pb2.DataTimestamps() + previous = None + for position, nanos in enumerate(epoch_nanos): + if previous is not None and nanos <= previous: + raise ValueError( + f"DataFrame index must be strictly increasing to become a time axis; entry {position} " + f"({nanos} ns) does not follow entry {position - 1} ({previous} ns)" + ) + previous = nanos + timestamp = timestamps.timestampList.timestamps.add() + timestamp.epochSeconds, timestamp.nanoseconds = divmod(nanos, 1_000_000_000) + return timestamps + + +def _timestamp_from_nanos(epoch_nanos: int) -> common_pb2.Timestamp: + """ + Builds a common.Timestamp from integer epoch nanoseconds -- the inverse of to_epoch_nanos(). + + :param epoch_nanos: Epoch nanoseconds. + :return: The equivalent common.Timestamp. + """ + timestamp = common_pb2.Timestamp() + timestamp.epochSeconds, timestamp.nanoseconds = divmod(epoch_nanos, 1_000_000_000) + return timestamp + + +def column_metadata_from_dict(summary: dict[str, Any] | None) -> common_pb2.ColumnMetadata | None: + """ + Rebuilds a ColumnMetadata from the dict column_metadata_dict() produced -- the inverse of that function. + + This is what lets a frame survive a pandas round trip with its provenance intact: data_frame_to_pandas() parks + each column's metadata in df.attrs["column_metadata"], and data_frame_from_pandas() feeds it back through + here. Without it the attrs were carried and then silently dropped, losing exactly the record of where the + numbers came from that makes calculations worth storing. + + :param summary: A dict as returned by column_metadata_dict(), or None. + :return: The equivalent ColumnMetadata, or None when the summary is absent or carries nothing. + """ + if not summary: + return None + + tags = summary.get("tags") or [] + attributes = summary.get("attributes") or {} + provenance_summary = summary.get("provenance") + if not tags and not attributes and not provenance_summary: + return None + + metadata = common_pb2.ColumnMetadata() + if tags: + metadata.tags[:] = list(tags) + for name, value in attributes.items(): + attribute = metadata.attributes.add() + attribute.name = name + attribute.value = value + + if provenance_summary: + provenance = metadata.provenance + if provenance_summary.get("source"): + provenance.source = provenance_summary["source"] + if provenance_summary.get("process"): + provenance.process = provenance_summary["process"] + for entry in provenance_summary.get("derived_from") or []: + source = provenance.derivedFrom.add() + if "pv_name" in entry: + source.pvName = entry["pv_name"] + elif "calculations_column" in entry: + column = entry["calculations_column"] + source.calculationsColumn.calculationsId = column["calculations_id"] + source.calculationsColumn.frameName = column["frame_name"] + source.calculationsColumn.columnName = column["column_name"] + time_range = entry.get("time_range") + if time_range is not None: + begin_nanos, end_nanos = time_range + source.timeRange.beginTime.CopyFrom(_timestamp_from_nanos(begin_nanos)) + source.timeRange.endTime.CopyFrom(_timestamp_from_nanos(end_nanos)) + + return metadata + + +def _column_from_series( + name: str, series: Any, metadata: common_pb2.ColumnMetadata | None = None, enum_id: str | None = None +) -> Any: + """ + Builds the typed column matching a pandas Series' dtype. + + Mapping: float64 -> DoubleColumn, float32 -> FloatColumn, int64 -> Int64Column, int32 -> Int32Column, + bool -> BoolColumn, and object/string -> StringColumn. Anything else raises rather than guessing. + + An enum_id routes an integer column to EnumColumn instead, since an enum's values are indistinguishable from + plain int32 codes by dtype alone -- the id is what makes it an enumeration. + + A missing value anywhere is a fail-loud error: the typed columns are dense repeated fields with no way to mark + an absent entry, so a NaN would have to be either invented as a real value or silently dropped. + + :param name: The column's name. + :param series: The pandas Series holding its values. + :param metadata: Optional ColumnMetadata to attach (see column_metadata_from_dict()). + :param enum_id: When set, build an EnumColumn carrying this enumeration id rather than an integer column. + :return: The matching typed column message. + :raises ValueError: if the dtype has no mapping, or the series contains a missing value. + """ + from dp_python_lib.client import data_frame as builders + + if series.isna().any(): + missing_positions = [int(position) for position in series.isna().to_numpy().nonzero()[0][:5]] + raise ValueError( + f"column '{name}' contains missing values (first at row position(s) {missing_positions}), which a " + f"dense typed column cannot represent. Either give the sparse column its own frame over only the " + f"timestamps where it has values, or build it with data_frame.data_column(), whose DataValues can be " + f"left unset." + ) + + dtype = series.dtype + dtype_name = str(dtype) + + if enum_id is not None: + # Checked before EVERY dtype branch, not just the integer ones: an EnumColumn's values are int32 codes, so + # dtype alone cannot distinguish it from a plain Int32Column, and a non-integer dtype paired with an enum + # id is a caller error worth naming rather than quietly building the wrong column kind. + if dtype_name not in ("int64", "Int64", "int32", "Int32"): + raise ValueError( + f"column '{name}' carries an enum id ({enum_id!r}) but has dtype {dtype_name}; an EnumColumn's " + f"values must be integer codes" + ) + return builders.enum_column(name, [int(v) for v in series], enum_id, metadata=metadata) + + if dtype_name == "float64": + return builders.double_column(name, [float(v) for v in series], metadata=metadata) + if dtype_name == "float32": + return builders.float_column(name, [float(v) for v in series], metadata=metadata) + if dtype_name in ("int64", "Int64"): + return builders.int64_column(name, [int(v) for v in series], metadata=metadata) + if dtype_name in ("int32", "Int32"): + return builders.int32_column(name, [int(v) for v in series], metadata=metadata) + if dtype_name in ("bool", "boolean"): + return builders.bool_column(name, [bool(v) for v in series], metadata=metadata) + if dtype_name in ("object", "string", "str") or dtype_name.startswith("string"): + values = list(series) + if not all(isinstance(value, str) for value in values): + offending = next( + (type(value).__name__ for value in values if not isinstance(value, str)), + "unknown", + ) + raise ValueError( + f"column '{name}' has dtype {dtype_name} but holds a non-string value (type {offending}); " + f"object columns are mapped to StringColumn, so convert the values first or build the column " + f"explicitly with a data_frame builder" + ) + return builders.string_column(name, values, metadata=metadata) + + raise ValueError( + f"column '{name}' has dtype {dtype_name}, which has no typed-column mapping. Supported dtypes are " + f"float64, float32, int64, int32, bool, and object/string; build other kinds explicitly with the " + f"data_frame builders and pass them to data_frame()." + ) + + +def data_frame_from_pandas(df: Any) -> common_pb2.DataFrame: + """ + Converts a pandas DataFrame into a common.DataFrame, mapping each column's dtype to a typed column. + + Requires the optional [analysis] extra. + + The index becomes an explicit TimestampList; see _timestamps_from_index() for why a SamplingClock is never + inferred. Missing values are rejected fail-loud, since a dense typed column cannot express a gap. + + Per-column metadata is read back from df.attrs["column_metadata"] when present, so a frame that went out + through data_frame_to_pandas() returns with its tags, attributes, and provenance intact. Enum columns are + rebuilt as enum columns via df.attrs["enum_ids"]; without that id an integer column is just an integer column. + + Converting back with data_frame_to_pandas() preserves every value, dtype, and timestamp, but can return the + columns grouped by type rather than in their original order -- see that function's docstring. + + :param df: The pandas DataFrame to convert. Must have a tz-aware, strictly increasing DatetimeIndex. + :return: A common.DataFrame. + :raises ImportError: if the [analysis] extra is not installed. + :raises ValueError: if the index is unusable, the frame has no columns, a dtype has no mapping, or any column + contains a missing value. + """ + from dp_python_lib.client import data_frame as builders + + _require_pandas() + + if len(df.columns) == 0: + raise ValueError("DataFrame must have at least one column") + + # Duplicate labels make df[name] a DataFrame rather than a Series, which would otherwise surface far downstream + # as pandas' "truth value of a Series is ambiguous". Column names must be unique within a frame anyway, so + # reject it here with a message naming the offender -- concat() and merge() produce this easily by accident. + names = [str(name) for name in df.columns] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ValueError( + f"DataFrame has more than one column named {', '.join(repr(name) for name in duplicates)}; column " + f"names must be unique within a frame. Rename or drop the duplicates first (e.g. after a concat or " + f"a merge with overlapping names)." + ) + + # data_frame_to_pandas() parks each column's ColumnMetadata here; carrying it back is what keeps provenance + # alive across a round trip. A frame built by hand simply has no attrs, and every column gets None. + metadata_by_column = df.attrs.get("column_metadata") or {} + enum_ids = df.attrs.get("enum_ids") or {} + + data_timestamps = _timestamps_from_index(df.index) + columns = [ + _column_from_series( + str(name), + df[name], + metadata=column_metadata_from_dict(metadata_by_column.get(str(name))), + enum_id=enum_ids.get(str(name)), + ) + for name in df.columns + ] + return builders.data_frame(data_timestamps, columns) + + +def calculations_from_dataframes(frames: dict[str, Any]) -> annotation_pb2.Calculations: + """ + Converts a mapping of frame name -> pandas DataFrame into a Calculations payload. + + Requires the optional [analysis] extra. + + :param frames: Mapping of frame name to the pandas DataFrame holding that frame's columns. + :return: An annotation.Calculations. + :raises ImportError: if the [analysis] extra is not installed. + :raises ValueError: if frames is empty, a frame name is empty, or any frame cannot be converted. + """ + from dp_python_lib.client.annotations_client import calculations as build_calculations + + _require_pandas() + + if not frames: + raise ValueError("calculations_from_dataframes() requires at least one frame") + + return build_calculations({name: data_frame_from_pandas(df) for name, df in frames.items()}) + + +def calculations_to_dataframes( + calculations: annotation_pb2.Calculations | None, + exclude_column_metadata: bool = False, +) -> dict[str, Any]: + """ + Converts every frame of a Calculations into a pandas DataFrame, keyed by frame name. + + Requires the optional [analysis] extra. + + :param calculations: The Calculations to convert, or None. + :param exclude_column_metadata: When True, skip populating each frame's df.attrs["column_metadata"]. + :return: A dict mapping frame name to its pandas DataFrame; empty when calculations is None or has no frames. + :raises ImportError: if the [analysis] extra is not installed. + :raises ValueError: if two frames share a name, or a frame cannot be converted. + """ + _require_pandas() + + if calculations is None: + return {} + + frames: dict[str, Any] = {} + for entry in calculations.calculationDataFrames: + if entry.name in frames: + raise ValueError(f"Calculations carries more than one frame named '{entry.name}'; frame names are unique") + frames[entry.name] = data_frame_to_pandas(entry.frame, exclude_column_metadata=exclude_column_metadata) + return frames diff --git a/src/dp_python_lib/client/dataset_client.py b/src/dp_python_lib/client/dataset_client.py index 372bb23..c62f468 100644 --- a/src/dp_python_lib/client/dataset_client.py +++ b/src/dp_python_lib/client/dataset_client.py @@ -3,10 +3,10 @@ import grpc -from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp from dp_python_lib.client.query_support import check_at_most_one_text_criterion from dp_python_lib.client.result import ApiResultBase from dp_python_lib.client.service_api_client_base import ServiceApiClientBase +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp from dp_python_lib.grpc import annotation_pb2, annotation_pb2_grpc, common_pb2 ID_QUERY_CHUNK_SIZE = 100 diff --git a/src/dp_python_lib/client/machine_config_client.py b/src/dp_python_lib/client/machine_config_client.py index aa82b8b..eb8bee1 100644 --- a/src/dp_python_lib/client/machine_config_client.py +++ b/src/dp_python_lib/client/machine_config_client.py @@ -1,5 +1,4 @@ import logging -import math from collections.abc import Iterator from datetime import datetime, timezone @@ -7,61 +6,12 @@ from dp_python_lib.client.result import ApiResultBase from dp_python_lib.client.service_api_client_base import ServiceApiClientBase -from dp_python_lib.grpc import annotation_pb2, annotation_pb2_grpc, common_pb2 - -# Accepted input types for API parameters that map to a common.Timestamp: -# a timezone-aware datetime, epoch seconds (int or float), or an already-built Timestamp. -TimestampInput = datetime | int | float | common_pb2.Timestamp - -def to_timestamp(value: TimestampInput) -> common_pb2.Timestamp: - """ - Converts a user-supplied time value into a common.Timestamp{epochSeconds, nanoseconds}. - - Accepts: - - a timezone-aware datetime (naive datetimes are rejected to avoid silent local-timezone bugs), - - epoch seconds as an int or float (float fractional part becomes nanoseconds), - - an already-built common.Timestamp (returned as-is). - - :param value: The time value to convert. - :return: An equivalent common.Timestamp. - :raises ValueError: if a datetime is naive (has no tzinfo), or if the resulting epoch seconds are negative - (pre-1970) -- common.Timestamp.epochSeconds is an unsigned (uint64) field and cannot represent them. - :raises TypeError: if value is not one of the supported types. - """ - if isinstance(value, common_pb2.Timestamp): - return value - - if isinstance(value, datetime): - if value.tzinfo is None or value.tzinfo.utcoffset(value) is None: - raise ValueError( - "to_timestamp() requires a timezone-aware datetime; naive datetimes are rejected " - "to avoid silent local-timezone bugs. Use datetime.now(timezone.utc) or attach tzinfo." - ) - epoch = value.timestamp() - return to_timestamp(epoch) - - if isinstance(value, bool): - # bool is a subclass of int; reject it explicitly as it is virtually always a mistake. - raise TypeError("to_timestamp() does not accept bool") - - if isinstance(value, (int, float)): - # Floor the seconds (not truncate toward zero) so the fractional remainder, and therefore - # nanoseconds, is always in [0, 1_000_000_000) even for negative epoch inputs. - epoch_seconds = math.floor(value) - nanoseconds = int(round((float(value) - epoch_seconds) * 1_000_000_000)) - # Guard against float rounding pushing nanoseconds up to a full second. - if nanoseconds >= 1_000_000_000: - epoch_seconds += 1 - nanoseconds -= 1_000_000_000 - timestamp = common_pb2.Timestamp() - timestamp.epochSeconds = epoch_seconds - timestamp.nanoseconds = nanoseconds - return timestamp - - raise TypeError( - f"to_timestamp() expects datetime, int/float epoch seconds, or common.Timestamp, got {type(value).__name__}" - ) +# The shared time converters moved to time_conversions.py in issue #6. They were defined here, as the first +# module to need them, and six others grew imports of them from here -- which read as though datasets, queries, +# and DataFrames depended on the machine configuration API. This module is now just another caller. +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp +from dp_python_lib.grpc import annotation_pb2, annotation_pb2_grpc, common_pb2 class ConfigurationQuery: diff --git a/src/dp_python_lib/client/query_client.py b/src/dp_python_lib/client/query_client.py index 2580d92..fda07ec 100644 --- a/src/dp_python_lib/client/query_client.py +++ b/src/dp_python_lib/client/query_client.py @@ -4,9 +4,9 @@ import grpc -from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp from dp_python_lib.client.result import ApiResultBase from dp_python_lib.client.service_api_client_base import ServiceApiClientBase +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp from dp_python_lib.grpc import common_pb2, query_pb2, query_pb2_grpc diff --git a/src/dp_python_lib/client/query_conversions.py b/src/dp_python_lib/client/query_conversions.py index dc2ab05..6d51d91 100644 --- a/src/dp_python_lib/client/query_conversions.py +++ b/src/dp_python_lib/client/query_conversions.py @@ -22,6 +22,7 @@ from collections.abc import Iterator from typing import Any +from dp_python_lib.client.time_conversions import to_epoch_nanos from dp_python_lib.grpc import common_pb2, query_pb2 # Excel's hard row ceiling (1,048,576 rows including a header row). @@ -98,9 +99,9 @@ def _require_numpy(): ) -def _timestamp_to_epoch_nanos(ts: common_pb2.Timestamp) -> int: - """Converts a common.Timestamp to an integer count of nanoseconds since the Unix epoch.""" - return ts.epochSeconds * 1_000_000_000 + ts.nanoseconds +# The Timestamp -> epoch-nanoseconds conversion is shared (see time_conversions.to_epoch_nanos); this private +# alias is the name this module has always used internally. +_timestamp_to_epoch_nanos = to_epoch_nanos def data_value_to_python(value: common_pb2.DataValue) -> Any: diff --git a/src/dp_python_lib/client/sample_status_client.py b/src/dp_python_lib/client/sample_status_client.py index 25b0d75..59a2515 100644 --- a/src/dp_python_lib/client/sample_status_client.py +++ b/src/dp_python_lib/client/sample_status_client.py @@ -3,99 +3,29 @@ import grpc -from dp_python_lib.client.machine_config_client import TimestampInput, to_timestamp +from dp_python_lib.client.data_frame import sampling_clock, timestamp_list +from dp_python_lib.client.data_frame import timestamp_count as _timestamp_count from dp_python_lib.client.result import ApiResultBase from dp_python_lib.client.service_api_client_base import ServiceApiClientBase +from dp_python_lib.client.time_conversions import TimestampInput, to_timestamp from dp_python_lib.grpc import annotation_pb2, annotation_pb2_grpc, common_pb2 - -def sampling_clock( - start_time: TimestampInput, - period_nanos: int, - count: int, -) -> common_pb2.DataTimestamps: - """ - Builds a DataTimestamps with a SamplingClock time axis, the compact form for dense labeling of regularly-sampled - data. The clock must match the archived data's clock exactly -- status-to-sample matching is by exact timestamp - at nanosecond precision, so an off-by-one-nanosecond period misses every sample after the first. - - :param start_time: Time of the first sample (tz-aware datetime, epoch seconds, or common.Timestamp). - :param period_nanos: Period between samples, in nanoseconds. Must be > 0. - :param count: Number of samples in the interval. Must be >= 1. - :return: A DataTimestamps carrying a SamplingClock. - :raises ValueError: if period_nanos is not positive or count is less than 1. - """ - if period_nanos <= 0: - raise ValueError(f"sampling_clock() requires period_nanos > 0, got {period_nanos}") - if count < 1: - raise ValueError(f"sampling_clock() requires count >= 1, got {count}") - - timestamps = common_pb2.DataTimestamps() - timestamps.samplingClock.startTime.CopyFrom(to_timestamp(start_time)) - timestamps.samplingClock.periodNanos = period_nanos - timestamps.samplingClock.count = count - return timestamps - - -def timestamp_list(values: list[TimestampInput]) -> common_pb2.DataTimestamps: - """ - Builds a DataTimestamps with an explicit TimestampList time axis, the form for sparse labeling -- naming only the - samples being labeled. The unlabeled samples carry no assertion; there is no need to mark the rest "good". - - Timestamps must be strictly increasing, which is validated here rather than deferred to the server. Supply - timestamps taken from data query results (or exact SamplingClock arithmetic); a recomputed or rounded timestamp - will silently fail to match its sample. - - :param values: The timestamps to label (tz-aware datetimes, epoch seconds, or common.Timestamp objects). - :return: A DataTimestamps carrying a TimestampList. - :raises ValueError: if values is empty or the timestamps are not strictly increasing. - """ - if not values: - raise ValueError("timestamp_list() requires a non-empty values list") - - converted = [to_timestamp(value) for value in values] - - previous = converted[0] - for index, current in enumerate(converted[1:], start=1): - if (current.epochSeconds, current.nanoseconds) <= (previous.epochSeconds, previous.nanoseconds): - raise ValueError( - f"timestamp_list() requires strictly increasing timestamps; entry {index} " - f"({current.epochSeconds}.{current.nanoseconds:09d}) does not follow entry {index - 1} " - f"({previous.epochSeconds}.{previous.nanoseconds:09d})" - ) - previous = current - - timestamps = common_pb2.DataTimestamps() - timestamps.timestampList.timestamps.extend(converted) - return timestamps - - -def _timestamp_count(timestamps: common_pb2.DataTimestamps) -> int: - """ - Returns the number of timestamps a DataTimestamps describes, for validating parallel-array lengths. - - An empty axis is rejected here rather than allowed to surface later as a confusing column-length mismatch. - The axis builders already make this unreachable -- sampling_clock() requires count >= 1 and timestamp_list() - requires a non-empty list -- but a hand-built DataTimestamps can still carry a zero-count SamplingClock or an - empty TimestampList, and both report their oneof arm as set. Rejecting them keeps this in step with - expand_data_timestamps(), which applies the same rule on the read path. - - :param timestamps: The time axis to measure. - :return: The number of timestamps on the axis. - :raises ValueError: if neither axis form is set, or if the axis describes no timestamps. - """ - axis = timestamps.WhichOneof("value") - if axis == "samplingClock": - count = timestamps.samplingClock.count - if count < 1: - raise ValueError(f"DataTimestamps samplingClock requires count >= 1, got {count}") - return count - if axis == "timestampList": - count = len(timestamps.timestampList.timestamps) - if count < 1: - raise ValueError("DataTimestamps timestampList requires at least one timestamp, got an empty list") - return count - raise ValueError("DataTimestamps must specify either a samplingClock or a timestampList") +# sampling_clock(), timestamp_list(), and timestamp_count() moved to data_frame.py in issue #6 Phase 2, once +# calculations frames became a second caller. They are re-exported here (and from dp_python_lib.client) so existing +# imports keep working; __all__ names them explicitly so the re-export reads as deliberate rather than as an unused +# import. _timestamp_count is the private alias this module has always used. +__all__ = [ + "DeleteSampleStatusesApiResult", + "QuerySampleStatusesApiResult", + "QuerySampleStatusesRequestParams", + "SampleStatusClient", + "SampleStatusColumn", + "SampleStatusFrame", + "SaveSampleStatusesApiResult", + "SaveSampleStatusesRequestParams", + "sampling_clock", + "timestamp_list", +] class SampleStatusColumn: diff --git a/src/dp_python_lib/client/sample_status_conversions.py b/src/dp_python_lib/client/sample_status_conversions.py index 36959d3..dee8fef 100644 --- a/src/dp_python_lib/client/sample_status_conversions.py +++ b/src/dp_python_lib/client/sample_status_conversions.py @@ -21,18 +21,12 @@ from collections.abc import Iterator +from dp_python_lib.client.time_conversions import NANOS_PER_SECOND, to_epoch_nanos from dp_python_lib.grpc import common_pb2 -_NANOS_PER_SECOND = 1_000_000_000 - - -def _timestamp_to_nanos(timestamp: common_pb2.Timestamp) -> int: - """ - Converts a common.Timestamp into a single integer of epoch nanoseconds. - :param timestamp: The timestamp to convert. - :return: Epoch nanoseconds as a Python int (arbitrary precision, so no overflow). - """ - return timestamp.epochSeconds * _NANOS_PER_SECOND + timestamp.nanoseconds +# The Timestamp -> epoch-nanoseconds conversion is shared (see time_conversions.to_epoch_nanos); this private +# alias is the name this module has always used internally. +_timestamp_to_nanos = to_epoch_nanos def _nanos_to_timestamp(epoch_nanos: int) -> common_pb2.Timestamp: @@ -42,7 +36,7 @@ def _nanos_to_timestamp(epoch_nanos: int) -> common_pb2.Timestamp: :return: The equivalent common.Timestamp. """ timestamp = common_pb2.Timestamp() - timestamp.epochSeconds, timestamp.nanoseconds = divmod(epoch_nanos, _NANOS_PER_SECOND) + timestamp.epochSeconds, timestamp.nanoseconds = divmod(epoch_nanos, NANOS_PER_SECOND) return timestamp diff --git a/src/dp_python_lib/client/time_conversions.py b/src/dp_python_lib/client/time_conversions.py new file mode 100644 index 0000000..249ed75 --- /dev/null +++ b/src/dp_python_lib/client/time_conversions.py @@ -0,0 +1,115 @@ +""" +Time conversions shared across the client library. + +Every API that takes an instant accepts the same three spellings -- a timezone-aware datetime, epoch seconds, or an +already-built common.Timestamp -- and every conversions module that reads one back wants integer nanoseconds. Those +two directions are `to_timestamp()` and `to_epoch_nanos()`, and they live here rather than in a feature client. + +They were originally defined in machine_config_client, the first module to need them, and four other modules grew +imports of `to_timestamp()` from there -- which read as though datasets, queries, and DataFrames depended on the +machine configuration API. Meanwhile the reverse conversion had been written out privately three separate times. +This module is a leaf: it imports nothing from the client package, so any client module can use it freely. + +Design decisions: + - Naive datetimes are rejected rather than assumed to be UTC or local. A silent local-timezone interpretation is + the kind of bug that surfaces months later as data attributed to the wrong shift. + - `to_epoch_nanos()` is integer arithmetic on a Python int end to end. Present-day epoch nanoseconds need about + 61 bits and a float64 carries 53, so routing the conversion through float seconds would move the instant -- + and sample-status matching is by exact timestamp at nanosecond precision. +""" + +import math +from datetime import datetime, timezone + +from dp_python_lib.grpc import common_pb2 + +# Accepted input types for API parameters that map to a common.Timestamp: +# a timezone-aware datetime, epoch seconds (int or float), or an already-built Timestamp. +TimestampInput = datetime | int | float | common_pb2.Timestamp + + +NANOS_PER_SECOND = 1_000_000_000 + +_NANOS_PER_MICROSECOND = 1_000 +_SECONDS_PER_DAY = 86_400 +# The reference instant for the datetime -> Timestamp conversion. Subtracting two aware datetimes yields a +# timedelta whose days/seconds/microseconds are exact integers, which is what keeps that path lossless. +_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) + + +def to_epoch_nanos(timestamp: common_pb2.Timestamp) -> int: + """ + Converts a common.Timestamp into a single integer of epoch nanoseconds -- the inverse of to_timestamp(). + + Integer arithmetic throughout, on a Python int, so there is no overflow and no precision loss. Every + conversions module in the library reads instants back through this; see the module docstring for why the + integer path is a correctness requirement rather than an optimization. + + :param timestamp: The timestamp to convert. + :return: Epoch nanoseconds as a Python int. + """ + return timestamp.epochSeconds * NANOS_PER_SECOND + timestamp.nanoseconds + + +def to_timestamp(value: TimestampInput) -> common_pb2.Timestamp: + """ + Converts a user-supplied time value into a common.Timestamp{epochSeconds, nanoseconds}. + + Accepts: + - a timezone-aware datetime (naive datetimes are rejected to avoid silent local-timezone bugs), + - epoch seconds as an int or float (float fractional part becomes nanoseconds), + - an already-built common.Timestamp (returned as-is). + + :param value: The time value to convert. + :return: An equivalent common.Timestamp. + :raises ValueError: if a datetime is naive (has no tzinfo), or if the resulting epoch seconds are negative + (pre-1970) -- common.Timestamp.epochSeconds is an unsigned (uint64) field and cannot represent them. + :raises TypeError: if value is not one of the supported types. + """ + if isinstance(value, common_pb2.Timestamp): + return value + + if isinstance(value, datetime): + if value.tzinfo is None or value.tzinfo.utcoffset(value) is None: + raise ValueError( + "to_timestamp() requires a timezone-aware datetime; naive datetimes are rejected " + "to avoid silent local-timezone bugs. Use datetime.now(timezone.utc) or attach tzinfo." + ) + # Integer arithmetic, NOT value.timestamp(): that returns a float64, which cannot hold present-day epoch + # seconds at sub-microsecond resolution, so the fractional part comes back slightly wrong. It moved 99.7% + # of microsecond-precision datetimes by up to ~119 ns -- fatal for an API whose sample-status matching and + # provenance ranges are exact at nanosecond precision. A datetime's own resolution is exactly + # microseconds, so converting through its integer fields is lossless. + delta = value - _EPOCH + epoch_seconds = delta.days * _SECONDS_PER_DAY + delta.seconds + if epoch_seconds < 0: + raise ValueError( + f"to_timestamp() cannot represent {value.isoformat()}: common.Timestamp.epochSeconds is unsigned, " + f"so instants before 1970-01-01T00:00:00Z have no representation." + ) + timestamp = common_pb2.Timestamp() + timestamp.epochSeconds = epoch_seconds + timestamp.nanoseconds = delta.microseconds * _NANOS_PER_MICROSECOND + return timestamp + + if isinstance(value, bool): + # bool is a subclass of int; reject it explicitly as it is virtually always a mistake. + raise TypeError("to_timestamp() does not accept bool") + + if isinstance(value, (int, float)): + # Floor the seconds (not truncate toward zero) so the fractional remainder, and therefore + # nanoseconds, is always in [0, 1_000_000_000) even for negative epoch inputs. + epoch_seconds = math.floor(value) + nanoseconds = int(round((float(value) - epoch_seconds) * 1_000_000_000)) + # Guard against float rounding pushing nanoseconds up to a full second. + if nanoseconds >= 1_000_000_000: + epoch_seconds += 1 + nanoseconds -= 1_000_000_000 + timestamp = common_pb2.Timestamp() + timestamp.epochSeconds = epoch_seconds + timestamp.nanoseconds = nanoseconds + return timestamp + + raise TypeError( + f"to_timestamp() expects datetime, int/float epoch seconds, or common.Timestamp, got {type(value).__name__}" + ) diff --git a/tests/integration/test_datasets_annotations_integration.py b/tests/integration/test_datasets_annotations_integration.py index 888edfe..22b1e8c 100644 --- a/tests/integration/test_datasets_annotations_integration.py +++ b/tests/integration/test_datasets_annotations_integration.py @@ -10,6 +10,8 @@ # Add src directory to path for imports sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src")) +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc from dp_python_lib.client.annotations_client import ( AnnotationQuery, SaveAnnotationRequestParams, @@ -20,10 +22,16 @@ SaveDataSetRequestParams, data_block, ) +from dp_python_lib.client.export_client import ExportDataRequestParams, ExportFormat, calculations_spec from dp_python_lib.client.mldp_client import MldpClient from dp_python_lib.grpc import common_pb2, ingestion_pb2, ingestion_pb2_grpc +def _epoch_nanos(when: datetime) -> int: + """Epoch nanoseconds for a tz-aware datetime, for comparing against the conversions' integer-nanos output.""" + return int(when.timestamp()) * 1_000_000_000 + when.microsecond * 1_000 + + class TestDataSetsAnnotationsIntegration(unittest.TestCase): """ Integration tests for DataSetClient and AnnotationsClient that require a running MLDP ecosystem. @@ -68,6 +76,7 @@ def setUpClass(cls): cls.client = MldpClient() cls.datasets = cls.client.annotation.datasets cls.annotations = cls.client.annotation.annotations + cls.export = cls.client.annotation.export cls._verify_modernized_api_available() @@ -529,6 +538,149 @@ def test_delete_annotation_twice_is_a_business_error(self): self.assertTrue(second.result_status.is_error) self.assertIsNone(second.annotation_id) + # ------------------------------------------------------------------ + # Builder-made calculations, read back through the conversions (Phase 2) + # ------------------------------------------------------------------ + + CALC_START = datetime(2024, 2, 2, 18, 0, 0, tzinfo=timezone.utc) + CALC_PERIOD_NANOS = 250_000_000 + CALC_VALUES = (12.7, 12.8, 12.9) + + def _builder_calculations(self): + """ + A Calculations payload built entirely through data_frame.py, with column-level provenance. + + Deliberately uses a sub-second period, so the axis round trip exercises the nanosecond arithmetic rather + than whole seconds a float could also represent. + """ + axis = dfb.sampling_clock(self.CALC_START, self.CALC_PERIOD_NANOS, len(self.CALC_VALUES)) + column = dfb.double_column( + "x_rms", + list(self.CALC_VALUES), + metadata=dfb.column_metadata( + tags=["derived"], + attributes={"unit": "mm"}, + provenance=dfb.provenance( + source="itest-rig", + process="1 Hz RMS", + derived_from=[dfb.pv_source(self.pv_name, (self.begin_time, self.end_time))], + ), + ), + ) + return calculations({"orbit-rms": dfb.data_frame(axis, [column])}) + + def test_builder_calculations_round_trip_is_nanosecond_exact(self): + """ + Save calculations built by data_frame.py, read them back, and check the axis reproduces exactly. + + This is the leg that would fail silently if any part of the path routed timestamps through float seconds: + a float64 cannot represent present-day epoch nanoseconds, so the expanded positions would drift. + """ + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + fetched = self.annotations.get_calculations(saved.calculations_id) + self.assertFalse(fetched.result_status.is_error, fetched.result_status.message) + + frames = fetched.calculations.calculationDataFrames + self.assertEqual([f.name for f in frames], ["orbit-rms"]) + frame = frames[0].frame + + start_nanos = int(self.CALC_START.timestamp()) * 1_000_000_000 + expected = [start_nanos + i * self.CALC_PERIOD_NANOS for i in range(len(self.CALC_VALUES))] + self.assertEqual(dfc.data_frame_timestamps(frame), expected) + + self.assertEqual(dfc.data_frame_columns(frame), {"x_rms": list(self.CALC_VALUES)}) + + def test_builder_provenance_survives_the_round_trip(self): + """Column-level provenance is the reason calculations are worth storing; it must come back intact.""" + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + frame = self.annotations.get_calculations(saved.calculations_id).calculations.calculationDataFrames[0].frame + metadata = dfc.column_metadata_dict(frame.doubleColumns[0]) + + self.assertEqual(metadata["tags"], ["derived"]) + self.assertEqual(metadata["attributes"], {"unit": "mm"}) + self.assertEqual(metadata["provenance"]["source"], "itest-rig") + self.assertEqual(metadata["provenance"]["process"], "1 Hz RMS") + source = metadata["provenance"]["derived_from"][0] + self.assertEqual(source["pv_name"], self.pv_name) + # The source's time range is the substantive half of provenance: which part of the PV's history was used. + # It must survive the server round trip, exact to the nanosecond, like every other instant in this library. + self.assertEqual( + source["time_range"], + (_epoch_nanos(self.begin_time), _epoch_nanos(self.end_time)), + ) + # A pvName source carries no calculations_column key at all, rather than an empty placeholder. + self.assertNotIn("calculations_column", source) + + # ------------------------------------------------------------------ + # Export (Phase 4) + # ------------------------------------------------------------------ + + def test_calculations_only_csv_export(self): + """ + A calculations-only export needs no ingested data of its own, which makes it the cheapest end-to-end + check that exportData() works. + """ + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + result = self.export.export_data( + ExportDataRequestParams( + ExportFormat.CSV, + calculations_spec=calculations_spec(saved.calculations_id), + ) + ) + + self.assertFalse(result.result_status.is_error, result.result_status.message) + self.assertTrue(result.file_path, "a successful export must report a server-side file path") + # file_url is empty unless the deployment publishes over HTTP; empty is normal, not a failure. + self.assertIsNotNone(result.file_url) + + def test_export_accepts_a_bare_format_string(self): + """ExportFormat coercion is part of the params contract, so exercise it against the real server too.""" + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + result = self.export.export_data( + ExportDataRequestParams("csv", calculations_spec=calculations_spec(saved.calculations_id)) + ) + + self.assertFalse(result.result_status.is_error, result.result_status.message) + + def test_export_with_a_column_filter(self): + """calculations_spec() narrows the export to named columns of named frames.""" + dataset_id = self._save_dataset() + saved = self._save_annotation([dataset_id], calculations=self._builder_calculations()) + + result = self.export.export_data( + ExportDataRequestParams( + ExportFormat.CSV, + calculations_spec=calculations_spec(saved.calculations_id, {"orbit-rms": ["x_rms"]}), + ) + ) + + self.assertFalse(result.result_status.is_error, result.result_status.message) + + def test_export_of_an_unknown_calculations_id_is_rejected(self): + result = self.export.export_data( + ExportDataRequestParams(ExportFormat.CSV, calculations_spec=calculations_spec("000000000000000000000000")) + ) + + self.assertTrue(result.result_status.is_error) + self.assertIsNone(result.file_path) + + def test_dataset_export_to_hdf5(self): + """The dataset path exports the archived samples the run ingested.""" + dataset_id = self._save_dataset() + + result = self.export.export_data(ExportDataRequestParams(ExportFormat.HDF5, dataset_id=dataset_id)) + + self.assertFalse(result.result_status.is_error, result.result_status.message) + self.assertTrue(result.file_path) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_data_frame.py b/tests/unit/test_data_frame.py new file mode 100644 index 0000000..e888475 --- /dev/null +++ b/tests/unit/test_data_frame.py @@ -0,0 +1,478 @@ +import os +import sys +import unittest +from datetime import datetime, timezone + +# Add src directory to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src")) + +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.grpc import common_pb2 + +try: + import numpy as np + + _HAVE_NUMPY = True +except ImportError: + _HAVE_NUMPY = False + +T0 = datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc) +T1 = datetime(2026, 7, 14, 19, 0, 0, tzinfo=timezone.utc) + + +def _axis(count=3): + """A SamplingClock axis of the given length, at 1 Hz.""" + return dfb.sampling_clock(T0, 1_000_000_000, count) + + +class TestSamplingClock(unittest.TestCase): + """The axis builders relocated here from sample_status_client in Phase 2; behavior is unchanged.""" + + def test_builds_clock(self): + axis = dfb.sampling_clock(T0, 1_000_000, 3) + self.assertEqual(axis.samplingClock.startTime.epochSeconds, int(T0.timestamp())) + self.assertEqual(axis.samplingClock.periodNanos, 1_000_000) + self.assertEqual(axis.samplingClock.count, 3) + + def test_rejects_non_positive_period(self): + for period in (0, -1): + with self.assertRaises(ValueError): + dfb.sampling_clock(T0, period, 3) + + def test_rejects_count_below_one(self): + with self.assertRaises(ValueError): + dfb.sampling_clock(T0, 1_000_000, 0) + + def test_still_importable_from_sample_status_client(self): + # The relocation must not break existing imports. + from dp_python_lib.client.sample_status_client import sampling_clock, timestamp_list + + self.assertIs(sampling_clock, dfb.sampling_clock) + self.assertIs(timestamp_list, dfb.timestamp_list) + + +class TestTimestampList(unittest.TestCase): + def test_builds_list(self): + axis = dfb.timestamp_list([100, 200, 300]) + self.assertEqual([t.epochSeconds for t in axis.timestampList.timestamps], [100, 200, 300]) + + def test_rejects_empty(self): + with self.assertRaises(ValueError): + dfb.timestamp_list([]) + + def test_rejects_non_increasing(self): + for values in ([100, 300, 200], [100, 100]): + with self.assertRaises(ValueError): + dfb.timestamp_list(values) + + def test_rejects_non_increasing_at_nanosecond_precision(self): + with self.assertRaises(ValueError): + dfb.timestamp_list([100.000_002, 100.000_001]) + + +class TestTimestampCount(unittest.TestCase): + def test_counts_both_axis_forms(self): + self.assertEqual(dfb.timestamp_count(dfb.sampling_clock(T0, 1_000, 7)), 7) + self.assertEqual(dfb.timestamp_count(dfb.timestamp_list([1, 2])), 2) + + def test_rejects_unset_axis(self): + with self.assertRaises(ValueError): + dfb.timestamp_count(common_pb2.DataTimestamps()) + + def test_rejects_hand_built_empty_axes(self): + # The builders make these unreachable, but a hand-built axis can still carry them. + empty_clock = common_pb2.DataTimestamps() + empty_clock.samplingClock.count = 0 + empty_clock.samplingClock.periodNanos = 1 + with self.assertRaises(ValueError): + dfb.timestamp_count(empty_clock) + + empty_list = common_pb2.DataTimestamps() + empty_list.timestampList.SetInParent() + with self.assertRaises(ValueError): + dfb.timestamp_count(empty_list) + + +class TestScalarColumnBuilders(unittest.TestCase): + """Each typed scalar builder produces the right message with the right values.""" + + def test_each_builder_and_field(self): + cases = [ + (dfb.double_column("d", [1.0, 2.0]), common_pb2.DoubleColumn, [1.0, 2.0]), + (dfb.int64_column("i64", [1, 2]), common_pb2.Int64Column, [1, 2]), + (dfb.int32_column("i32", [1, 2]), common_pb2.Int32Column, [1, 2]), + (dfb.bool_column("b", [True, False]), common_pb2.BoolColumn, [True, False]), + (dfb.string_column("s", ["a", "b"]), common_pb2.StringColumn, ["a", "b"]), + ] + for column, expected_type, expected_values in cases: + with self.subTest(column=column.name): + self.assertIsInstance(column, expected_type) + self.assertEqual(list(column.values), expected_values) + + def test_float_column_is_float32(self): + column = dfb.float_column("f", [1.5, 2.5]) + self.assertIsInstance(column, common_pb2.FloatColumn) + self.assertEqual(list(column.values), [1.5, 2.5]) + + def test_enum_column_carries_enum_id(self): + column = dfb.enum_column("state", [0, 1, 2], enum_id="beam-state") + self.assertEqual(list(column.values), [0, 1, 2]) + self.assertEqual(column.enumId, "beam-state") + + def test_enum_column_requires_enum_id(self): + with self.assertRaises(ValueError) as ctx: + dfb.enum_column("state", [0], enum_id="") + self.assertIn("enum_id", str(ctx.exception)) + + def test_metadata_is_attached(self): + metadata = dfb.column_metadata(tags=["derived"]) + column = dfb.double_column("d", [1.0], metadata=metadata) + self.assertEqual(list(column.metadata.tags), ["derived"]) + + def test_rejects_empty_name_and_values(self): + for builder in (dfb.double_column, dfb.float_column, dfb.int64_column, dfb.int32_column, dfb.string_column): + with self.subTest(builder=builder.__name__): + with self.assertRaises(ValueError): + builder("", [1]) + with self.assertRaises(ValueError): + builder("name", []) + + def test_empty_values_error_names_the_column(self): + with self.assertRaises(ValueError) as ctx: + dfb.double_column("x_rms", []) + self.assertIn("x_rms", str(ctx.exception)) + + +class TestDataColumnEscapeHatch(unittest.TestCase): + """The legacy DataColumn is the only way to express a gap on a shared axis.""" + + def test_none_becomes_an_unset_oneof(self): + column = dfb.data_column("sparse", [1.0, None, 3.0]) + arms = [v.WhichOneof("value") for v in column.dataValues] + self.assertEqual(arms, ["doubleValue", None, "doubleValue"]) + + def test_maps_each_python_type(self): + column = dfb.data_column("mixed", [True, 7, 1.5, "s", b"bytes"]) + arms = [v.WhichOneof("value") for v in column.dataValues] + self.assertEqual(arms, ["booleanValue", "longValue", "doubleValue", "stringValue", "byteArrayValue"]) + + def test_bool_is_checked_before_int(self): + # bool is a subclass of int, so an int-first check would store True as longValue 1. + column = dfb.data_column("flags", [True, False]) + self.assertEqual([v.WhichOneof("value") for v in column.dataValues], ["booleanValue", "booleanValue"]) + self.assertEqual([v.booleanValue for v in column.dataValues], [True, False]) + + def test_accepts_prebuilt_data_value(self): + prebuilt = common_pb2.DataValue() + prebuilt.uintValue = 42 + column = dfb.data_column("u", [prebuilt]) + self.assertEqual(column.dataValues[0].WhichOneof("value"), "uintValue") + self.assertEqual(column.dataValues[0].uintValue, 42) + + @unittest.skipUnless(_HAVE_NUMPY, "requires the [analysis] extra (numpy)") + def test_maps_numpy_scalars_like_their_python_counterparts(self): + # np.float64 subclasses float, but np.int64 does not subclass int and np.bool_ does not subclass bool. + # Mapping by exact Python type would accept the first and reject the other two -- an arbitrary split for a + # caller coming from pandas or NumPy. + column = dfb.data_column("np", [np.float64(1.5), np.int64(2), np.int32(3), np.bool_(True), np.float32(0.5)]) + arms = [v.WhichOneof("value") for v in column.dataValues] + self.assertEqual(arms, ["doubleValue", "longValue", "longValue", "booleanValue", "doubleValue"]) + self.assertEqual(column.dataValues[1].longValue, 2) + self.assertIs(column.dataValues[3].booleanValue, True) + + @unittest.skipUnless(_HAVE_NUMPY, "requires the [analysis] extra (numpy)") + def test_numpy_bool_is_checked_before_the_integer_branch(self): + # np.bool_ is not Integral, but this pins the ordering the way test_bool_is_checked_before_int does. + column = dfb.data_column("flags", [np.bool_(True), np.bool_(False)]) + self.assertEqual([v.WhichOneof("value") for v in column.dataValues], ["booleanValue", "booleanValue"]) + self.assertEqual([v.booleanValue for v in column.dataValues], [True, False]) + + def test_rejects_unmappable_type(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_column("c", [1 + 2j]) + message = str(ctx.exception) + self.assertIn("complex", message) + self.assertIn("index 0", message) + + def test_rejects_whitespace_only_name(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_column(" ", [1]) + self.assertIn("non-blank", str(ctx.exception)) + + def test_rejects_empty_name_and_values(self): + with self.assertRaises(ValueError): + dfb.data_column("", [1]) + with self.assertRaises(ValueError): + dfb.data_column("name", []) + + +class TestProvenanceHelpers(unittest.TestCase): + def test_pv_source_sets_the_pv_name_arm(self): + source = dfb.pv_source("BPMS:GUNB:314:X") + self.assertEqual(source.WhichOneof("origin"), "pvName") + self.assertEqual(source.pvName, "BPMS:GUNB:314:X") + self.assertFalse(source.HasField("timeRange")) + + def test_pv_source_with_time_range(self): + source = dfb.pv_source("A:1", (T0, T1)) + self.assertTrue(source.HasField("timeRange")) + self.assertEqual(source.timeRange.beginTime.epochSeconds, int(T0.timestamp())) + self.assertEqual(source.timeRange.endTime.epochSeconds, int(T1.timestamp())) + + def test_calculations_source_sets_the_calculations_arm(self): + source = dfb.calculations_source("calc-1", "f1", "x_rms") + self.assertEqual(source.WhichOneof("origin"), "calculationsColumn") + self.assertEqual(source.calculationsColumn.calculationsId, "calc-1") + self.assertEqual(source.calculationsColumn.frameName, "f1") + self.assertEqual(source.calculationsColumn.columnName, "x_rms") + + def test_sources_reject_empty_identifiers(self): + with self.assertRaises(ValueError): + dfb.pv_source("") + with self.assertRaises(ValueError): + dfb.calculations_source("", "f", "c") + with self.assertRaises(ValueError): + dfb.calculations_source("calc", "", "c") + with self.assertRaises(ValueError): + dfb.calculations_source("calc", "f", "") + + def test_sources_reject_reversed_time_range(self): + with self.assertRaises(ValueError) as ctx: + dfb.pv_source("A:1", (T1, T0)) + self.assertIn("strictly before", str(ctx.exception)) + + def test_provenance_fields(self): + result = dfb.provenance( + source="analysis-rig", + process="1 Hz RMS", + derived_from=[dfb.pv_source("A:1"), dfb.calculations_source("c", "f", "x")], + ) + self.assertEqual(result.source, "analysis-rig") + self.assertEqual(result.process, "1 Hz RMS") + self.assertEqual(len(result.derivedFrom), 2) + + def test_provenance_omits_unset_fields(self): + result = dfb.provenance() + self.assertEqual(result.source, "") + self.assertEqual(result.process, "") + self.assertEqual(len(result.derivedFrom), 0) + + def test_column_metadata_fields(self): + metadata = dfb.column_metadata( + tags=["derived", "reviewed"], + attributes={"unit": "mm"}, + provenance=dfb.provenance(process="p"), + ) + self.assertEqual(list(metadata.tags), ["derived", "reviewed"]) + self.assertEqual({(a.name, a.value) for a in metadata.attributes}, {("unit", "mm")}) + self.assertEqual(metadata.provenance.process, "p") + + def test_column_metadata_empty(self): + metadata = dfb.column_metadata() + self.assertEqual(list(metadata.tags), []) + self.assertEqual(len(metadata.attributes), 0) + self.assertFalse(metadata.HasField("provenance")) + + +class TestDataFrameAssembly(unittest.TestCase): + """data_frame() routes columns by type and enforces the server's shape rules client-side.""" + + def test_routes_each_column_type_to_its_field(self): + frame = dfb.data_frame( + _axis(2), + [ + dfb.double_column("d", [1.0, 2.0]), + dfb.float_column("f", [1.0, 2.0]), + dfb.int64_column("i64", [1, 2]), + dfb.int32_column("i32", [1, 2]), + dfb.bool_column("b", [True, False]), + dfb.string_column("s", ["a", "b"]), + dfb.enum_column("e", [0, 1], enum_id="enum-1"), + dfb.data_column("legacy", [1.0, None]), + ], + ) + self.assertEqual([c.name for c in frame.doubleColumns], ["d"]) + self.assertEqual([c.name for c in frame.floatColumns], ["f"]) + self.assertEqual([c.name for c in frame.int64Columns], ["i64"]) + self.assertEqual([c.name for c in frame.int32Columns], ["i32"]) + self.assertEqual([c.name for c in frame.boolColumns], ["b"]) + self.assertEqual([c.name for c in frame.stringColumns], ["s"]) + self.assertEqual([c.name for c in frame.enumColumns], ["e"]) + self.assertEqual([c.name for c in frame.dataColumns], ["legacy"]) + + def test_copies_the_axis(self): + axis = _axis(3) + frame = dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0, 3.0])]) + self.assertEqual(frame.dataTimestamps.samplingClock.count, 3) + + def test_rejects_empty_column_list(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), []) + self.assertIn("at least one column", str(ctx.exception)) + + def test_rejects_count_mismatch_naming_the_column(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(3), [dfb.double_column("x_rms", [1.0, 2.0])]) + message = str(ctx.exception) + self.assertIn("x_rms", message) + self.assertIn("2 values", message) + self.assertIn("3 timestamps", message) + + def test_rejects_duplicate_names_across_types(self): + # Uniqueness is across ALL column types, not just within one. + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), [dfb.double_column("dup", [1.0]), dfb.string_column("dup", ["a"])]) + self.assertIn("dup", str(ctx.exception)) + + def test_rejects_unnamed_prebuilt_column(self): + unnamed = common_pb2.DoubleColumn() + unnamed.values[:] = [1.0] + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), [unnamed]) + self.assertIn("index 0", str(ctx.exception)) + + def test_rejects_unsupported_column_type(self): + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), ["not a column"]) + self.assertIn("unsupported column type", str(ctx.exception)) + + def test_rejects_out_of_order_prebuilt_timestamp_list(self): + # timestamp_list() enforces strict ordering, but a hand-built axis bypasses that builder. Two samples + # claiming one instant is inexpressible in the identity model, and data_frame_from_pandas() rejects the + # index such an axis produces -- so accepting it here would build an un-round-trippable frame. + for label, seconds in (("decreasing", [2, 1]), ("duplicate", [1, 1])): + with self.subTest(case=label): + axis = common_pb2.DataTimestamps() + for second in seconds: + entry = axis.timestampList.timestamps.add() + entry.epochSeconds = 1_700_000_000 + second + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0])]) + self.assertIn("strictly increasing", str(ctx.exception)) + + def test_rejects_prebuilt_timestamp_list_duplicated_at_nanosecond_precision(self): + axis = common_pb2.DataTimestamps() + for _ in range(2): + entry = axis.timestampList.timestamps.add() + entry.epochSeconds = 1_700_000_000 + entry.nanoseconds = 5 + with self.assertRaises(ValueError): + dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0])]) + + def test_accepts_strictly_increasing_prebuilt_timestamp_list(self): + axis = common_pb2.DataTimestamps() + for second in (1, 2): + entry = axis.timestampList.timestamps.add() + entry.epochSeconds = 1_700_000_000 + second + frame = dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0])]) + self.assertEqual(len(frame.dataTimestamps.timestampList.timestamps), 2) + + def test_rejects_zero_period_sampling_clock(self): + # sampling_clock() makes this unreachable, but a hand-built axis bypasses it. expand_data_timestamps() + # rejects a non-positive period on the read side, so accepting it here would build an unreadable frame -- + # and every sample after the first would carry the first one's timestamp. + axis = common_pb2.DataTimestamps() + axis.samplingClock.startTime.epochSeconds = 1_700_000_000 + axis.samplingClock.periodNanos = 0 + axis.samplingClock.count = 3 + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0, 3.0])]) + self.assertIn("periodNanos > 0", str(ctx.exception)) + + def test_rejects_empty_axis(self): + with self.assertRaises(ValueError): + dfb.data_frame(common_pb2.DataTimestamps(), [dfb.double_column("d", [1.0])]) + + def test_accepts_prebuilt_array_column(self): + # Array builders are #17's; a hand-built one passes through, with dims giving the sample count. + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([4]) + column.values[:] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] # 2 samples x 4 + frame = dfb.data_frame(_axis(2), [column]) + self.assertEqual([c.name for c in frame.doubleArrayColumns], ["waveform"]) + + def test_array_column_count_uses_dims_product(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([4]) + column.values[:] = [1.0] * 8 # 2 samples, but the axis says 3 + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(3), [column]) + self.assertIn("2 values", str(ctx.exception)) + + def test_accepts_prebuilt_image_column(self): + # ImageColumn keeps one payload per sample in `images`; it has no `values` field at all, so a generic + # len(column.values) count raises AttributeError on a column type data_frame() claims to support. + column = common_pb2.ImageColumn() + column.name = "camera" + column.images.extend([b"frame-0", b"frame-1"]) + frame = dfb.data_frame(_axis(2), [column]) + self.assertEqual([c.name for c in frame.imageColumns], ["camera"]) + + def test_image_column_count_is_validated_against_the_axis(self): + column = common_pb2.ImageColumn() + column.name = "camera" + column.images.extend([b"only-one"]) + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(3), [column]) + self.assertIn("1 values", str(ctx.exception)) + + def test_rejects_whitespace_only_column_name(self): + # The rule is a non-blank name; " " is empty of content while passing a bare falsiness check. + for blank in (" ", "\t", "\n"): + with self.subTest(name=blank): + column = common_pb2.DoubleColumn() + column.name = blank + column.values[:] = [1.0] + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(1), [column]) + self.assertIn("non-blank", str(ctx.exception)) + + def test_array_column_with_ragged_values_is_rejected(self): + # 5 values with a per-sample size of 2 is not a whole number of samples. Floor division would round it to + # a passing count of 2 and build a frame that data_frame_conversions then refuses to read back. + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([2]) + column.values[:] = [1.0, 2.0, 3.0, 4.0, 5.0] + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(2), [column]) + message = str(ctx.exception) + self.assertIn("whole multiple", message) + self.assertIn("waveform", message) + + def test_array_column_accepts_multidimensional_dims(self): + column = common_pb2.DoubleArrayColumn() + column.name = "image" + column.dimensions.dims.extend([2, 3]) + column.values[:] = [float(i) for i in range(12)] # 2 samples x (2 x 3) + frame = dfb.data_frame(_axis(2), [column]) + self.assertEqual([c.name for c in frame.doubleArrayColumns], ["image"]) + + def test_array_column_without_dims_is_rejected(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.values[:] = [1.0, 2.0] + with self.assertRaises(ValueError) as ctx: + dfb.data_frame(_axis(2), [column]) + self.assertIn("dimensions", str(ctx.exception)) + + def test_serialized_column_gets_name_checks_only(self): + # Serialized payloads are opaque, so the server checks names only; this must not demand a count match. + column = common_pb2.SerializedDataColumn() + column.name = "serialized" + column.encoding = "arrow" + column.payload = b"opaque" + frame = dfb.data_frame(_axis(3), [column]) + self.assertEqual([c.name for c in frame.serializedDataColumns], ["serialized"]) + + def test_serialized_column_still_needs_a_unique_name(self): + column = common_pb2.SerializedDataColumn() + column.name = "dup" + with self.assertRaises(ValueError): + dfb.data_frame(_axis(1), [dfb.double_column("dup", [1.0]), column]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_data_frame_conversions.py b/tests/unit/test_data_frame_conversions.py new file mode 100644 index 0000000..e1f21c2 --- /dev/null +++ b/tests/unit/test_data_frame_conversions.py @@ -0,0 +1,828 @@ +import os +import sys +import unittest +from datetime import datetime, timezone + +# Add src directory to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src")) + +from dp_python_lib.client import data_frame as dfb +from dp_python_lib.client import data_frame_conversions as dfc +from dp_python_lib.client.annotations_client import calculations +from dp_python_lib.client.query_conversions import Image +from dp_python_lib.grpc import annotation_pb2, common_pb2 + +# The pandas half depends on the optional [analysis] extra; skip those tests cleanly when it is absent. The +# pure-Python read side needs no optional deps and always runs. +try: + import pandas as pd + + _HAVE_ANALYSIS = True +except ImportError: + _HAVE_ANALYSIS = False + +T0 = datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc) +T0_NANOS = int(T0.timestamp()) * 1_000_000_000 +T1 = datetime(2026, 7, 14, 19, 0, 0, tzinfo=timezone.utc) +T1_NANOS = int(T1.timestamp()) * 1_000_000_000 + + +def _axis(count=3, period_nanos=1_000_000_000): + return dfb.sampling_clock(T0, period_nanos, count) + + +# ---------------------------------------------------------------------- +# pure Python read side (no optional dependencies) +# ---------------------------------------------------------------------- + + +class TestDataFrameTimestamps(unittest.TestCase): + def test_sampling_clock_expands_in_exact_integer_nanoseconds(self): + # The whole point of the integer path: a float64 cannot hold present-day epoch nanos, so a float + # round-trip would move every one of these. + frame = dfb.data_frame(_axis(3, period_nanos=1), [dfb.double_column("d", [1.0, 2.0, 3.0])]) + self.assertEqual(dfc.data_frame_timestamps(frame), [T0_NANOS, T0_NANOS + 1, T0_NANOS + 2]) + + def test_timestamp_list_is_returned_in_order(self): + axis = dfb.timestamp_list([100, 200, 300]) + frame = dfb.data_frame(axis, [dfb.double_column("d", [1.0, 2.0, 3.0])]) + self.assertEqual( + dfc.data_frame_timestamps(frame), + [100_000_000_000, 200_000_000_000, 300_000_000_000], + ) + + def test_frame_without_axis_raises(self): + frame = common_pb2.DataFrame() + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_timestamps(frame) + self.assertIn("dataTimestamps", str(ctx.exception)) + + +class TestColumnValues(unittest.TestCase): + """column_values() is a standalone per-column converter, reusable by the bucket query (#16).""" + + def test_each_scalar_column_type(self): + cases = [ + (dfb.double_column("d", [1.5, 2.5]), [1.5, 2.5]), + (dfb.int64_column("i", [1, 2]), [1, 2]), + (dfb.int32_column("i32", [3, 4]), [3, 4]), + (dfb.bool_column("b", [True, False]), [True, False]), + (dfb.string_column("s", ["a", "b"]), ["a", "b"]), + (dfb.enum_column("e", [0, 1], enum_id="x"), [0, 1]), + ] + for column, expected in cases: + with self.subTest(column=column.name): + self.assertEqual(dfc.column_values(column), expected) + + def test_float_column_values_round_to_float32(self): + # float32 storage is lossy by nature; assert the round trip rather than exact float64 equality. + column = dfb.float_column("f", [1.5, 2.25]) + self.assertEqual(dfc.column_values(column), [1.5, 2.25]) + + def test_data_column_unset_value_becomes_none(self): + # The one representation of a gap in this API. + column = dfb.data_column("sparse", [1.0, None, 3.0]) + self.assertEqual(dfc.column_values(column), [1.0, None, 3.0]) + + def test_data_column_complex_arms(self): + column = common_pb2.DataColumn() + column.name = "mixed" + column.dataValues.add().stringValue = "s" + image = column.dataValues.add() + image.imageValue.image = b"png-bytes" + image.imageValue.fileType = common_pb2.Image.FileType.PNG + + values = dfc.column_values(column) + self.assertEqual(values[0], "s") + self.assertEqual(values[1], Image(b"png-bytes", "PNG")) + + def test_array_column_is_reshaped_per_sample(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([3]) + column.values[:] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] + self.assertEqual(dfc.column_values(column), [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) + + def test_multidimensional_array_stays_flat_within_a_sample(self): + column = common_pb2.Int32ArrayColumn() + column.name = "grid" + column.dimensions.dims.extend([2, 2]) + column.values[:] = [1, 2, 3, 4, 5, 6, 7, 8] + self.assertEqual(dfc.column_values(column), [[1, 2, 3, 4], [5, 6, 7, 8]]) + + def test_array_column_without_dims_raises(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.values[:] = [1.0, 2.0] + with self.assertRaises(ValueError) as ctx: + dfc.column_values(column) + self.assertIn("dimensions", str(ctx.exception)) + + def test_array_column_with_indivisible_values_raises(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([3]) + column.values[:] = [1.0, 2.0, 3.0, 4.0] # not a whole multiple of 3 + with self.assertRaises(ValueError) as ctx: + dfc.column_values(column) + self.assertIn("not a whole multiple", str(ctx.exception)) + + def test_image_column_yields_bytes_per_sample(self): + column = common_pb2.ImageColumn() + column.name = "frames" + column.images.extend([b"a", b"b"]) + self.assertEqual(dfc.column_values(column), [b"a", b"b"]) + + def test_unsupported_type_raises(self): + with self.assertRaises(ValueError): + dfc.column_values("not a column") + + +class TestDataFrameColumns(unittest.TestCase): + def test_collects_every_column_type_by_name(self): + frame = dfb.data_frame( + _axis(2), + [ + dfb.double_column("d", [1.0, 2.0]), + dfb.string_column("s", ["a", "b"]), + dfb.data_column("legacy", [1.0, None]), + ], + ) + self.assertEqual(dfc.data_frame_columns(frame), {"d": [1.0, 2.0], "s": ["a", "b"], "legacy": [1.0, None]}) + + def test_duplicate_names_raise_rather_than_dropping(self): + # data_frame() prevents this, but a hand-built or older-server frame can still carry it. + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(_axis(1)) + for field in (frame.doubleColumns, frame.stringColumns): + column = field.add() + column.name = "dup" + frame.doubleColumns[0].values[:] = [1.0] + frame.stringColumns[0].values[:] = ["a"] + + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_columns(frame) + self.assertIn("dup", str(ctx.exception)) + + def test_serialized_columns_are_skipped(self): + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(_axis(1)) + double = frame.doubleColumns.add() + double.name = "d" + double.values[:] = [1.0] + serialized = frame.serializedDataColumns.add() + serialized.name = "opaque" + serialized.payload = b"blob" + + self.assertEqual(dfc.data_frame_columns(frame), {"d": [1.0]}) + + +class TestColumnMetadataDict(unittest.TestCase): + def test_full_metadata(self): + column = dfb.double_column( + "d", + [1.0], + metadata=dfb.column_metadata( + tags=["derived"], + attributes={"unit": "mm"}, + provenance=dfb.provenance( + source="rig", + process="RMS", + derived_from=[ + dfb.pv_source("A:1"), + dfb.calculations_source("c", "f", "x"), + dfb.pv_source("A:2", (T0, T1)), + ], + ), + ), + ) + result = dfc.column_metadata_dict(column) + + self.assertEqual(result["tags"], ["derived"]) + self.assertEqual(result["attributes"], {"unit": "mm"}) + self.assertEqual(result["provenance"]["source"], "rig") + self.assertEqual(result["provenance"]["process"], "RMS") + # Only the origin arm actually set is present -- the unset arm is absent, not an empty placeholder. + self.assertEqual(result["provenance"]["derived_from"][0], {"pv_name": "A:1"}) + self.assertEqual( + result["provenance"]["derived_from"][1], + {"calculations_column": {"calculations_id": "c", "frame_name": "f", "column_name": "x"}}, + ) + + def test_provenance_preserves_source_time_range(self): + """A source's timeRange -- which part of it was used -- survives read-back, as epoch nanoseconds.""" + column = dfb.double_column( + "d", + [1.0], + metadata=dfb.column_metadata( + provenance=dfb.provenance( + derived_from=[ + dfb.pv_source("A:1", (T0, T1)), + dfb.calculations_source("c", "f", "x", (T0, T1)), + dfb.pv_source("A:2"), + ] + ) + ), + ) + + sources = dfc.column_metadata_dict(column)["provenance"]["derived_from"] + self.assertEqual(sources[0]["time_range"], (T0_NANOS, T1_NANOS)) + self.assertEqual(sources[1]["time_range"], (T0_NANOS, T1_NANOS)) + # An absent range is absent, not a fabricated (0, 0) pair. + self.assertNotIn("time_range", sources[2]) + + def test_absent_metadata(self): + result = dfc.column_metadata_dict(dfb.double_column("d", [1.0])) + self.assertEqual(result, {"tags": [], "attributes": {}, "provenance": None}) + + +# ---------------------------------------------------------------------- +# pandas bridges ([analysis] extra) +# ---------------------------------------------------------------------- + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestDataFrameToPandas(unittest.TestCase): + def test_basic_conversion(self): + frame = dfb.data_frame( + _axis(3), + [dfb.double_column("x_rms", [12.7, 12.8, 12.9]), dfb.string_column("label", ["a", "b", "c"])], + ) + df = dfc.data_frame_to_pandas(frame) + + self.assertEqual(list(df.columns), ["x_rms", "label"]) + self.assertEqual(df["x_rms"].tolist(), [12.7, 12.8, 12.9]) + self.assertEqual(len(df), 3) + + def test_index_is_utc_and_nanosecond_exact(self): + # A SamplingClock at 1 ns spacing: only an integer path reproduces these instants. + frame = dfb.data_frame(_axis(3, period_nanos=1), [dfb.double_column("d", [1.0, 2.0, 3.0])]) + df = dfc.data_frame_to_pandas(frame) + + self.assertEqual(str(df.index.tz), "UTC") + self.assertEqual( + [int(v) for v in df.index.view("int64")], + [T0_NANOS, T0_NANOS + 1, T0_NANOS + 2], + ) + + def test_metadata_lands_in_attrs(self): + frame = dfb.data_frame( + _axis(1), + [dfb.double_column("d", [1.0], metadata=dfb.column_metadata(tags=["derived"]))], + ) + df = dfc.data_frame_to_pandas(frame) + + self.assertEqual(df.attrs["column_metadata"]["d"]["tags"], ["derived"]) + + def test_metadata_can_be_excluded(self): + frame = dfb.data_frame(_axis(1), [dfb.double_column("d", [1.0])]) + df = dfc.data_frame_to_pandas(frame, exclude_column_metadata=True) + self.assertNotIn("column_metadata", df.attrs) + + def test_gap_becomes_nan(self): + frame = dfb.data_frame(_axis(3), [dfb.data_column("sparse", [1.0, None, 3.0])]) + df = dfc.data_frame_to_pandas(frame) + self.assertTrue(df["sparse"].isna().tolist() == [False, True, False]) + + def test_misaligned_column_raises(self): + # Hand-built: data_frame() would have caught this on the way out. + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(_axis(3)) + column = frame.doubleColumns.add() + column.name = "short" + column.values[:] = [1.0, 2.0] + + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_to_pandas(frame) + message = str(ctx.exception) + self.assertIn("short", message) + self.assertIn("index-aligned", message) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestDataFrameFromPandas(unittest.TestCase): + def _index(self, periods=3): + return pd.date_range("2026-07-14 18:00", periods=periods, freq="s", tz="UTC") + + def test_dtype_mapping(self): + import numpy as np + + df = pd.DataFrame( + { + "d": [1.0, 2.0], + "f": np.array([1.5, 2.5], dtype="float32"), + "i64": np.array([1, 2], dtype="int64"), + "i32": np.array([1, 2], dtype="int32"), + "b": [True, False], + "s": ["a", "b"], + }, + index=self._index(2), + ) + frame = dfc.data_frame_from_pandas(df) + + self.assertEqual([c.name for c in frame.doubleColumns], ["d"]) + self.assertEqual([c.name for c in frame.floatColumns], ["f"]) + self.assertEqual([c.name for c in frame.int64Columns], ["i64"]) + self.assertEqual([c.name for c in frame.int32Columns], ["i32"]) + self.assertEqual([c.name for c in frame.boolColumns], ["b"]) + self.assertEqual([c.name for c in frame.stringColumns], ["s"]) + + def test_index_becomes_a_timestamp_list(self): + # Never a SamplingClock: inferring uniformity from an index that merely looks regular would move + # timestamps, which this API cannot tolerate. + df = pd.DataFrame({"d": [1.0, 2.0, 3.0]}, index=self._index()) + frame = dfc.data_frame_from_pandas(df) + + self.assertEqual(frame.dataTimestamps.WhichOneof("value"), "timestampList") + self.assertEqual(len(frame.dataTimestamps.timestampList.timestamps), 3) + + def test_round_trip_preserves_values_and_instants(self): + index = self._index() + df = pd.DataFrame({"d": [1.0, 2.0, 3.0], "s": ["a", "b", "c"]}, index=index) + + back = dfc.data_frame_to_pandas(dfc.data_frame_from_pandas(df)) + + self.assertEqual(back["d"].tolist(), [1.0, 2.0, 3.0]) + self.assertEqual(back["s"].tolist(), ["a", "b", "c"]) + # Compare the instants themselves, not raw int64 views: a view is in the index's own storage unit, and + # date_range() yields a microsecond-unit index here while the converter always emits nanoseconds. The + # earlier form compared two different units and only passed because both sides were equally wrong. + self.assertEqual( + [entry.value for entry in back.index], + [entry.value for entry in index], + ) + + def test_nan_is_fail_loud_with_sparsity_guidance(self): + df = pd.DataFrame({"x": [1.0, None, 3.0]}, index=self._index()) + + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + message = str(ctx.exception) + self.assertIn("x", message) + self.assertIn("missing values", message) + self.assertIn("data_column", message) + + def test_naive_index_is_rejected(self): + df = pd.DataFrame({"x": [1.0]}, index=pd.date_range("2026-07-14", periods=1, freq="s")) + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + self.assertIn("timezone-aware", str(ctx.exception)) + + def test_non_datetime_index_is_rejected(self): + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(pd.DataFrame({"x": [1.0]})) + self.assertIn("DatetimeIndex", str(ctx.exception)) + + def test_non_increasing_index_is_rejected(self): + index = pd.DatetimeIndex(["2026-07-14 18:00:01", "2026-07-14 18:00:00"], tz="UTC") + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(pd.DataFrame({"x": [1.0, 2.0]}, index=index)) + self.assertIn("strictly increasing", str(ctx.exception)) + + def test_no_columns_is_rejected(self): + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(pd.DataFrame(index=self._index())) + self.assertIn("at least one column", str(ctx.exception)) + + def test_unmappable_dtype_is_rejected(self): + df = pd.DataFrame({"c": [1 + 2j, 2 + 0j, 3 + 0j]}, index=self._index()) + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + message = str(ctx.exception) + self.assertIn("complex128", message) + self.assertIn("no typed-column mapping", message) + + def test_object_column_holding_non_strings_is_rejected(self): + df = pd.DataFrame({"o": [{"a": 1}, {"b": 2}, {"c": 3}]}, index=self._index()) + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + self.assertIn("non-string value", str(ctx.exception)) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestCalculationsBridges(unittest.TestCase): + def _index(self, periods=3): + return pd.date_range("2026-07-14 18:00", periods=periods, freq="s", tz="UTC") + + def test_round_trip_multiple_frames(self): + frames = { + "bpm-statistics": pd.DataFrame({"x_rms": [1.0, 2.0, 3.0]}, index=self._index()), + "rf-statistics": pd.DataFrame({"y_rms": [4.0, 5.0, 6.0]}, index=self._index()), + } + calcs = dfc.calculations_from_dataframes(frames) + + self.assertEqual( + {f.name for f in calcs.calculationDataFrames}, + {"bpm-statistics", "rf-statistics"}, + ) + + back = dfc.calculations_to_dataframes(calcs) + self.assertEqual(set(back), set(frames)) + self.assertEqual(back["bpm-statistics"]["x_rms"].tolist(), [1.0, 2.0, 3.0]) + self.assertEqual(back["rf-statistics"]["y_rms"].tolist(), [4.0, 5.0, 6.0]) + + def test_to_dataframes_none_is_empty(self): + self.assertEqual(dfc.calculations_to_dataframes(None), {}) + + def test_to_dataframes_no_frames_is_empty(self): + self.assertEqual(dfc.calculations_to_dataframes(annotation_pb2.Calculations()), {}) + + def test_to_dataframes_rejects_duplicate_frame_names(self): + calcs = annotation_pb2.Calculations() + for _ in range(2): + entry = calcs.calculationDataFrames.add() + entry.name = "dup" + entry.frame.CopyFrom(dfb.data_frame(_axis(1), [dfb.double_column("d", [1.0])])) + + with self.assertRaises(ValueError) as ctx: + dfc.calculations_to_dataframes(calcs) + self.assertIn("dup", str(ctx.exception)) + + def test_from_dataframes_rejects_empty(self): + with self.assertRaises(ValueError) as ctx: + dfc.calculations_from_dataframes({}) + self.assertIn("at least one frame", str(ctx.exception)) + + def test_from_dataframes_rejects_empty_frame_name(self): + with self.assertRaises(ValueError): + dfc.calculations_from_dataframes({"": pd.DataFrame({"x": [1.0]}, index=self._index(1))}) + + def test_metadata_survives_to_pandas(self): + frame = dfb.data_frame( + _axis(1), + [dfb.double_column("d", [1.0], metadata=dfb.column_metadata(tags=["derived"]))], + ) + back = dfc.calculations_to_dataframes(calculations({"f1": frame})) + self.assertEqual(back["f1"].attrs["column_metadata"]["d"]["tags"], ["derived"]) + + +class TestWriteAndReadPathsAgree(unittest.TestCase): + """Anything data_frame() accepts must be readable by this module -- the two paths share one shape contract.""" + + def test_ragged_array_rejected_by_both_paths(self): + # Regression: the write path used floor division, so it accepted a value count that was not a whole + # multiple of prod(dims) and produced a frame data_frame_columns() then refused to read. + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([2]) + column.values[:] = [1.0, 2.0, 3.0, 4.0, 5.0] + + with self.assertRaises(ValueError) as write_error: + dfb.data_frame(_axis(2), [column]) + + frame = common_pb2.DataFrame() + frame.dataTimestamps.CopyFrom(_axis(2)) + frame.doubleArrayColumns.append(column) + with self.assertRaises(ValueError) as read_error: + dfc.data_frame_columns(frame) + + for message in (str(write_error.exception), str(read_error.exception)): + self.assertIn("whole multiple", message) + self.assertIn("waveform", message) + + def test_well_formed_array_survives_the_round_trip(self): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend([2]) + column.values[:] = [1.0, 2.0, 3.0, 4.0] + frame = dfb.data_frame(_axis(2), [column]) + self.assertEqual(dfc.data_frame_columns(frame), {"waveform": [[1.0, 2.0], [3.0, 4.0]]}) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestPandasDuplicateColumnNames(unittest.TestCase): + def test_duplicate_column_names_are_rejected_by_name(self): + # df[name] on a duplicated label returns a DataFrame, not a Series, which would otherwise surface deep + # inside the converter as pandas' "truth value of a Series is ambiguous". + import pandas as pd + + index = pd.DatetimeIndex(pd.to_datetime([T0_NANOS, T0_NANOS + 1], unit="ns", utc=True)) + df = pd.DataFrame([[1.0, 2.0], [3.0, 4.0]], columns=["a", "a"], index=index) + + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + message = str(ctx.exception) + self.assertIn("'a'", message) + self.assertIn("unique", message) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestPandasRoundTripColumnOrder(unittest.TestCase): + """A DataFrame stores each column kind in its own repeated field, so ordering across kinds is not preserved.""" + + def _round_trip(self): + import pandas as pd + + index = pd.DatetimeIndex( + pd.to_datetime([T0_NANOS, T0_NANOS + 1_000_000_000, T0_NANOS + 2_000_000_000], unit="ns", utc=True) + ) + df = pd.DataFrame( + {"a": [1.0, 2.0, 3.0], "b": ["x", "y", "z"], "c": [True, False, True]}, + index=index, + ) + return df, dfc.data_frame_to_pandas(dfc.data_frame_from_pandas(df)) + + def test_columns_come_back_grouped_by_type(self): + # Pinned rather than merely documented: the order is a consequence of the proto's field order, and a + # caller who reindexes by name needs it to be stable. + _, restored = self._round_trip() + self.assertEqual(list(restored.columns), ["a", "c", "b"]) + + def test_values_index_and_dtypes_survive_the_reordering(self): + original, restored = self._round_trip() + realigned = restored[list(original.columns)] + self.assertTrue(original.index.equals(realigned.index)) + self.assertEqual(list(original.dtypes), list(realigned.dtypes)) + for name in original.columns: + self.assertEqual(list(original[name]), list(realigned[name])) + + +class TestEmptyAxisIsRejected(unittest.TestCase): + def test_set_but_empty_timestamp_list_is_rejected(self): + # expand_data_timestamps() returns [] for this, and the oneof still reports as set, so without an explicit + # check a corrupt frame converts to a zero-row table instead of failing loudly. + frame = common_pb2.DataFrame() + frame.dataTimestamps.timestampList.SetInParent() + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_timestamps(frame) + self.assertIn("no timestamps", str(ctx.exception)) + + +class TestArrayColumnDimensions(unittest.TestCase): + """column_values() flattens each sample; the dims that give it shape are recoverable separately (plan D7).""" + + def _frame(self, dims, values, count): + column = common_pb2.DoubleArrayColumn() + column.name = "waveform" + column.dimensions.dims.extend(dims) + column.values[:] = values + return dfb.data_frame(_axis(count), [column]) + + def test_dimensions_distinguish_shapes_with_equal_sample_size(self): + # [2, 2] and [4] both yield four values per sample; without the dims they are indistinguishable. + square = self._frame([2, 2], [float(i) for i in range(8)], 2) + flat = self._frame([4], [float(i) for i in range(8)], 2) + + self.assertEqual(dfc.data_frame_columns(square), dfc.data_frame_columns(flat)) + self.assertEqual(dfc.data_frame_column_dimensions(square), {"waveform": [2, 2]}) + self.assertEqual(dfc.data_frame_column_dimensions(flat), {"waveform": [4]}) + + def test_non_array_columns_have_no_dimensions(self): + self.assertIsNone(dfc.column_dimensions(dfb.double_column("d", [1.0]))) + frame = dfb.data_frame(_axis(1), [dfb.double_column("d", [1.0])]) + self.assertEqual(dfc.data_frame_column_dimensions(frame), {}) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestIndexUnitAndNaT(unittest.TestCase): + """The index's storage unit is not guaranteed to be nanoseconds, and NaT has an integer representation.""" + + def test_non_nanosecond_index_units_convert_exactly(self): + import pandas as pd + + # A raw int64 view is in the index's OWN unit, so a datetime64[us] index would land 1000x too early. + for unit, expected_nanos in ( + ("s", 1_700_000_000_000_000_000), + ("ms", 1_700_000_000_123_000_000), + ("us", 1_700_000_000_123_456_000), + ("ns", 1_700_000_000_123_456_789), + ): + with self.subTest(unit=unit): + index = pd.DatetimeIndex(pd.to_datetime([1_700_000_000_123_456_789], unit="ns", utc=True)).as_unit(unit) + timestamp = dfc._timestamps_from_index(index).timestampList.timestamps[0] + actual = timestamp.epochSeconds * 1_000_000_000 + timestamp.nanoseconds + self.assertEqual(actual, expected_nanos) + + def test_nat_in_the_index_is_rejected(self): + import pandas as pd + + index = pd.DatetimeIndex([pd.NaT, pd.Timestamp("2026-01-01", tz="UTC")]) + with self.assertRaises(ValueError) as ctx: + dfc._timestamps_from_index(index) + self.assertIn("NaT", str(ctx.exception)) + + +def _columns_of(frame): + """Yields a frame's columns, for rebuilding it on a different time axis.""" + return dfc.iter_frame_columns(frame) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestPandasRoundTripPreservesTypesAndMetadata(unittest.TestCase): + """A frame that goes out through pandas must come back as the same frame, not a widened, stripped one.""" + + def _metadata(self): + return dfb.column_metadata( + tags=["derived"], + attributes={"unit": "mm"}, + provenance=dfb.provenance( + source="rig", + process="1 Hz RMS", + derived_from=[ + dfb.pv_source("BPMS:GUNB:314:X", (T0, T1)), + dfb.calculations_source("cid", "orbit-rms", "x_rms"), + ], + ), + ) + + def _frame(self): + return dfb.data_frame( + _axis(2), + [ + dfb.float_column("f", [1.5, 2.5], metadata=self._metadata()), + dfb.int32_column("i32", [1, 2]), + dfb.int64_column("i64", [3, 4]), + dfb.double_column("d", [1.0, 2.0]), + dfb.bool_column("b", [True, False]), + dfb.string_column("s", ["a", "b"]), + ], + ) + + def test_narrow_dtypes_reach_pandas(self): + # An untyped list would let pandas widen float32 to float64 and int32 to int64. + df = dfc.data_frame_to_pandas(self._frame()) + self.assertEqual(str(df["f"].dtype), "float32") + self.assertEqual(str(df["i32"].dtype), "int32") + self.assertEqual(str(df["i64"].dtype), "int64") + self.assertEqual(str(df["d"].dtype), "float64") + self.assertEqual(str(df["b"].dtype), "bool") + + def test_column_types_survive_the_round_trip(self): + original = self._frame() + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + + def kinds(frame): + return { + column.name: field + for field in ( + "doubleColumns", + "floatColumns", + "int64Columns", + "int32Columns", + "boolColumns", + "stringColumns", + ) + for column in getattr(frame, field) + } + + self.assertEqual(kinds(restored), kinds(original)) + + def test_metadata_and_provenance_survive_the_round_trip(self): + original = self._frame() + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + self.assertEqual(restored.floatColumns[0].metadata, original.floatColumns[0].metadata) + + def test_whole_frame_round_trips_apart_from_the_deliberate_axis_change(self): + # Everything except the axis form comes back byte for byte. The axis is deliberately NOT preserved: a + # SamplingClock becomes an explicit TimestampList, because inferring a clock back from an index that + # merely looks regular would quietly move timestamps. The instants themselves are identical. + original = dfb.data_frame(dfb.timestamp_list([T0, T1]), list(_columns_of(self._frame()))) + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + self.assertEqual(restored, original) + + def test_sampling_clock_axis_becomes_a_timestamp_list_with_the_same_instants(self): + original = self._frame() # a SamplingClock axis + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + + self.assertEqual(original.dataTimestamps.WhichOneof("value"), "samplingClock") + self.assertEqual(restored.dataTimestamps.WhichOneof("value"), "timestampList") + self.assertEqual(dfc.data_frame_timestamps(restored), dfc.data_frame_timestamps(original)) + + def test_excluded_metadata_is_simply_absent(self): + # With exclude_column_metadata=True there are no attrs to carry, and the columns come back bare. + original = self._frame() + df = dfc.data_frame_to_pandas(original, exclude_column_metadata=True) + restored = dfc.data_frame_from_pandas(df) + self.assertFalse(restored.floatColumns[0].HasField("metadata")) + + def test_hand_built_frame_without_attrs_still_converts(self): + import pandas as pd + + index = pd.DatetimeIndex(pd.to_datetime([T0_NANOS, T0_NANOS + 1_000_000_000], unit="ns", utc=True)) + df = pd.DataFrame({"d": [1.0, 2.0]}, index=index) + frame = dfc.data_frame_from_pandas(df) + self.assertFalse(frame.doubleColumns[0].HasField("metadata")) + + +class TestColumnMetadataFromDict(unittest.TestCase): + def test_none_and_empty_summaries_yield_none(self): + self.assertIsNone(dfc.column_metadata_from_dict(None)) + self.assertIsNone(dfc.column_metadata_from_dict({})) + self.assertIsNone(dfc.column_metadata_from_dict({"tags": [], "attributes": {}, "provenance": None})) + + def test_inverts_column_metadata_dict(self): + original = dfb.column_metadata( + tags=["a"], + attributes={"k": "v"}, + provenance=dfb.provenance(process="p", derived_from=[dfb.pv_source("PV", (T0, T1))]), + ) + column = dfb.double_column("d", [1.0], metadata=original) + self.assertEqual(dfc.column_metadata_from_dict(dfc.column_metadata_dict(column)), original) + + +@unittest.skipUnless(_HAVE_ANALYSIS, "requires the [analysis] extra (pandas)") +class TestEnumColumnRoundTrip(unittest.TestCase): + """ + An EnumColumn's values are int32 codes, so dtype alone cannot tell it from an Int32Column. The enumId that + gives those codes meaning has nowhere to live in a Series, so it rides in df.attrs["enum_ids"]. + """ + + def _frame(self): + return dfb.data_frame( + dfb.timestamp_list([T0, T1]), + [ + dfb.enum_column("state", [0, 1], enum_id="beam-state", metadata=dfb.column_metadata(tags=["t"])), + dfb.int32_column("plain", [7, 8]), + ], + ) + + def test_enum_codes_are_not_widened(self): + df = dfc.data_frame_to_pandas(self._frame()) + self.assertEqual(str(df["state"].dtype), "int32") + + def test_enum_id_travels_in_attrs(self): + df = dfc.data_frame_to_pandas(self._frame()) + self.assertEqual(df.attrs["enum_ids"], {"state": "beam-state"}) + + def test_enum_column_round_trips_whole(self): + original = self._frame() + restored = dfc.data_frame_from_pandas(dfc.data_frame_to_pandas(original)) + + self.assertEqual([c.name for c in restored.enumColumns], ["state"]) + self.assertEqual(restored.enumColumns[0].enumId, "beam-state") + # The plain int32 column beside it must NOT become an enum. + self.assertEqual([c.name for c in restored.int32Columns], ["plain"]) + self.assertEqual(restored, original) + + def test_enum_id_survives_excluded_metadata(self): + # enum_ids is structural: without it the column cannot be rebuilt as an enum at all, so it is carried even + # when the descriptive metadata is dropped. + original = self._frame() + df = dfc.data_frame_to_pandas(original, exclude_column_metadata=True) + restored = dfc.data_frame_from_pandas(df) + + self.assertEqual(restored.enumColumns[0].enumId, "beam-state") + self.assertFalse(restored.enumColumns[0].HasField("metadata")) + + def test_enum_id_on_a_non_integer_column_is_rejected(self): + import pandas as pd + + index = pd.DatetimeIndex(pd.to_datetime([T0_NANOS, T0_NANOS + 1], unit="ns", utc=True)) + for dtype, values in (("float64", [1.5, 2.5]), ("bool", [True, False]), ("object", ["a", "b"])): + with self.subTest(dtype=dtype): + df = pd.DataFrame({"x": pd.Series(values, index=index)}, index=index) + df.attrs["enum_ids"] = {"x": "beam-state"} + with self.assertRaises(ValueError) as ctx: + dfc.data_frame_from_pandas(df) + self.assertIn("enum id", str(ctx.exception)) + + +class TestStructuralColumnFields(unittest.TestCase): + """ + Some column kinds carry information their values cannot be interpreted without. column_values() keeps its + one-entry-per-sample shape, so those fields travel in companion accessors -- the same split as array dims. + """ + + def _frame(self): + image = common_pb2.ImageColumn() + image.name = "cam" + image.images.extend([b"frame-0", b"frame-1"]) + image.imageDescriptor.width = 640 + image.imageDescriptor.height = 480 + image.imageDescriptor.channels = 3 + image.imageDescriptor.encoding = "rgb8" + + struct = common_pb2.StructColumn() + struct.name = "readings" + struct.schemaId = "schema-1" + struct.values.extend([b"payload-0", b"payload-1"]) + + return dfb.data_frame(dfb.timestamp_list([T0, T1]), [image, struct]) + + def test_image_descriptor_is_recoverable(self): + # Without width/height/channels/encoding the payload bytes cannot be decoded at all. + self.assertEqual( + dfc.data_frame_image_descriptors(self._frame()), + {"cam": {"width": 640, "height": 480, "channels": 3, "encoding": "rgb8"}}, + ) + + def test_struct_schema_id_is_recoverable(self): + self.assertEqual(dfc.data_frame_schema_ids(self._frame()), {"readings": "schema-1"}) + + def test_values_still_yield_one_entry_per_sample(self): + self.assertEqual( + dfc.data_frame_columns(self._frame()), + {"cam": [b"frame-0", b"frame-1"], "readings": [b"payload-0", b"payload-1"]}, + ) + + def test_other_column_kinds_have_neither(self): + plain = dfb.double_column("d", [1.0]) + self.assertIsNone(dfc.image_descriptor_dict(plain)) + self.assertIsNone(dfc.column_schema_id(plain)) + frame = dfb.data_frame(_axis(1), [plain]) + self.assertEqual(dfc.data_frame_image_descriptors(frame), {}) + self.assertEqual(dfc.data_frame_schema_ids(frame), {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_machine_config_client.py b/tests/unit/test_machine_config_client.py index 0804c02..95a0a4a 100644 --- a/tests/unit/test_machine_config_client.py +++ b/tests/unit/test_machine_config_client.py @@ -1,7 +1,6 @@ import os import sys import unittest -from datetime import datetime, timedelta, timezone from unittest.mock import Mock import grpc @@ -17,7 +16,6 @@ QueryConfigurationsApiResult, SaveConfigurationApiResult, SaveConfigurationRequestParams, - to_timestamp, ) from dp_python_lib.grpc import annotation_pb2, common_pb2 @@ -32,61 +30,6 @@ def _response_with_field(field_name): return response -class TestToTimestamp(unittest.TestCase): - """Unit tests for the to_timestamp() conversion helper.""" - - def test_passthrough_timestamp(self): - ts = common_pb2.Timestamp() - ts.epochSeconds = 123 - ts.nanoseconds = 456 - self.assertIs(to_timestamp(ts), ts) - - def test_int_epoch_seconds(self): - ts = to_timestamp(1_700_000_000) - self.assertEqual(ts.epochSeconds, 1_700_000_000) - self.assertEqual(ts.nanoseconds, 0) - - def test_float_epoch_seconds_with_fraction(self): - ts = to_timestamp(1_700_000_000.25) - self.assertEqual(ts.epochSeconds, 1_700_000_000) - self.assertEqual(ts.nanoseconds, 250_000_000) - - def test_aware_datetime_utc(self): - dt = datetime(2023, 11, 14, 22, 13, 20, tzinfo=timezone.utc) - expected_epoch = int(dt.timestamp()) - ts = to_timestamp(dt) - self.assertEqual(ts.epochSeconds, expected_epoch) - self.assertEqual(ts.nanoseconds, 0) - - def test_aware_datetime_nonzero_offset(self): - tz = timezone(timedelta(hours=-5)) - dt = datetime(2023, 11, 14, 17, 13, 20, tzinfo=tz) # same instant as the UTC test above - ts = to_timestamp(dt) - self.assertEqual(ts.epochSeconds, int(dt.timestamp())) - - def test_negative_epoch_rejected_by_uint64_field(self): - # common.Timestamp.epochSeconds is uint64, so pre-1970 (negative) epochs cannot be represented. - # Flooring keeps nanoseconds normalized, and the negative seconds are cleanly rejected at assignment - # (ValueError: out of range) rather than silently producing an incorrect (seconds, nanos) pair. - with self.assertRaises(ValueError): - to_timestamp(-1.25) - with self.assertRaises(ValueError): - to_timestamp(-5) - - def test_naive_datetime_raises(self): - with self.assertRaises(ValueError): - # DTZ001: the naive datetime is the point of this test -- it must be rejected. - to_timestamp(datetime(2023, 11, 14, 22, 13, 20)) # noqa: DTZ001 - - def test_bool_raises(self): - with self.assertRaises(TypeError): - to_timestamp(True) - - def test_unsupported_type_raises(self): - with self.assertRaises(TypeError): - to_timestamp("2023-11-14") - - class TestConfigurationQuery(unittest.TestCase): """Unit tests for the ConfigurationQuery criterion helpers.""" diff --git a/tests/unit/test_time_conversions.py b/tests/unit/test_time_conversions.py new file mode 100644 index 0000000..ef89788 --- /dev/null +++ b/tests/unit/test_time_conversions.py @@ -0,0 +1,132 @@ +import os +import sys +import unittest +from datetime import datetime, timedelta, timezone + +# Add src directory to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src")) + +from dp_python_lib.client.time_conversions import to_epoch_nanos, to_timestamp +from dp_python_lib.grpc import common_pb2 + + +class TestToEpochNanos(unittest.TestCase): + """to_epoch_nanos() is to_timestamp()'s inverse, shared by the three conversions modules.""" + + def test_combines_seconds_and_nanoseconds(self): + ts = common_pb2.Timestamp() + ts.epochSeconds = 1_700_000_000 + ts.nanoseconds = 123_456_789 + self.assertEqual(to_epoch_nanos(ts), 1_700_000_000_123_456_789) + + def test_round_trips_with_to_timestamp_exactly(self): + # The exactness is the point: present-day epoch nanoseconds need ~61 bits and a float64 carries 53, so a + # conversion routed through float seconds would move the instant. + original = 1_770_055_200_123_456_789 + ts = common_pb2.Timestamp() + ts.epochSeconds, ts.nanoseconds = divmod(original, 1_000_000_000) + self.assertEqual(to_epoch_nanos(ts), original) + + def test_zero_timestamp_is_zero(self): + self.assertEqual(to_epoch_nanos(common_pb2.Timestamp()), 0) + + +class TestDatetimeConversionIsExact(unittest.TestCase): + """ + A datetime must not route through datetime.timestamp(). + + That returns a float64, which cannot hold present-day epoch seconds at sub-microsecond resolution, so the + fractional part comes back slightly wrong -- it moved 99.7% of microsecond-precision datetimes by up to ~119ns. + Sample-status matching and provenance ranges are exact at nanosecond precision, so that is a correctness bug, + not a rounding nicety. + """ + + def test_microsecond_datetimes_convert_exactly(self): + for microsecond in (1, 999, 123_456, 250_000, 999_999): + with self.subTest(microsecond=microsecond): + dt = datetime(2026, 7, 14, 18, 0, 0, microsecond, tzinfo=timezone.utc) + self.assertEqual(to_timestamp(dt).nanoseconds, microsecond * 1_000) + + def test_every_microsecond_of_a_second_is_exact(self): + # Exhaustive over the fractional field at a present-day instant: the float path failed almost all of these. + base = datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc) + wrong = [ + microsecond + for microsecond in range(0, 1_000_000, 977) # a prime stride, ~1024 samples across the range + if to_timestamp(base.replace(microsecond=microsecond)).nanoseconds != microsecond * 1_000 + ] + self.assertEqual(wrong, []) + + def test_round_trips_through_to_epoch_nanos(self): + dt = datetime(2026, 7, 14, 18, 0, 0, 123_456, tzinfo=timezone.utc) + expected = int(datetime(2026, 7, 14, 18, 0, 0, tzinfo=timezone.utc).timestamp()) * 1_000_000_000 + self.assertEqual(to_epoch_nanos(to_timestamp(dt)), expected + 123_456_000) + + def test_non_utc_offset_yields_the_same_instant(self): + aware = datetime(2026, 7, 14, 13, 0, 0, 123_456, tzinfo=timezone(timedelta(hours=-5))) + utc = datetime(2026, 7, 14, 18, 0, 0, 123_456, tzinfo=timezone.utc) + self.assertEqual(to_epoch_nanos(to_timestamp(aware)), to_epoch_nanos(to_timestamp(utc))) + + def test_epoch_and_pre_epoch(self): + self.assertEqual(to_epoch_nanos(to_timestamp(datetime(1970, 1, 1, tzinfo=timezone.utc))), 0) + with self.assertRaises(ValueError): + to_timestamp(datetime(1969, 12, 31, 23, 59, 59, tzinfo=timezone.utc)) + + +class TestToTimestamp(unittest.TestCase): + """Unit tests for the to_timestamp() conversion helper.""" + + def test_passthrough_timestamp(self): + ts = common_pb2.Timestamp() + ts.epochSeconds = 123 + ts.nanoseconds = 456 + self.assertIs(to_timestamp(ts), ts) + + def test_int_epoch_seconds(self): + ts = to_timestamp(1_700_000_000) + self.assertEqual(ts.epochSeconds, 1_700_000_000) + self.assertEqual(ts.nanoseconds, 0) + + def test_float_epoch_seconds_with_fraction(self): + ts = to_timestamp(1_700_000_000.25) + self.assertEqual(ts.epochSeconds, 1_700_000_000) + self.assertEqual(ts.nanoseconds, 250_000_000) + + def test_aware_datetime_utc(self): + dt = datetime(2023, 11, 14, 22, 13, 20, tzinfo=timezone.utc) + expected_epoch = int(dt.timestamp()) + ts = to_timestamp(dt) + self.assertEqual(ts.epochSeconds, expected_epoch) + self.assertEqual(ts.nanoseconds, 0) + + def test_aware_datetime_nonzero_offset(self): + tz = timezone(timedelta(hours=-5)) + dt = datetime(2023, 11, 14, 17, 13, 20, tzinfo=tz) # same instant as the UTC test above + ts = to_timestamp(dt) + self.assertEqual(ts.epochSeconds, int(dt.timestamp())) + + def test_negative_epoch_rejected_by_uint64_field(self): + # common.Timestamp.epochSeconds is uint64, so pre-1970 (negative) epochs cannot be represented. + # Flooring keeps nanoseconds normalized, and the negative seconds are cleanly rejected at assignment + # (ValueError: out of range) rather than silently producing an incorrect (seconds, nanos) pair. + with self.assertRaises(ValueError): + to_timestamp(-1.25) + with self.assertRaises(ValueError): + to_timestamp(-5) + + def test_naive_datetime_raises(self): + with self.assertRaises(ValueError): + # DTZ001: the naive datetime is the point of this test -- it must be rejected. + to_timestamp(datetime(2023, 11, 14, 22, 13, 20)) # noqa: DTZ001 + + def test_bool_raises(self): + with self.assertRaises(TypeError): + to_timestamp(True) + + def test_unsupported_type_raises(self): + with self.assertRaises(TypeError): + to_timestamp("2023-11-14") + + +if __name__ == "__main__": + unittest.main()