Reconcile the data reader parameters

docs/examples/read_dataset/read_dataset.md

@@ -4,7 +4,7 @@ A Harp acquisition is usually saved as a **de-multiplexed dataset folder**: one
 
 This is the recommended entry point for a recorded session on disk. To decode a single loose `.bin` file instead, see [Reading Data into a DataFrame](../read_data_to_dataframe/read_data_to_dataframe.md).
 
-The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. The Harp time becomes the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed.
+The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. The Harp time becomes the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when the dataset is opened with an `epoch`.
 
 <!--codeinclude-->
 ```python

docs/examples/read_dataset/read_dataset.py

@@ -34,9 +34,10 @@
 frames = {name: reader.read(name) for name in reader.contents}
 print(list(frames))
 
-# Pass an epoch to turn the "Time" index into an absolute `DatetimeIndex` instead
-# of float seconds. `REFERENCE_EPOCH` is time zero of the Harp clock in UTC.
-absolute = reader.read(44, epoch=data.REFERENCE_EPOCH)
+# Opening with an epoch turns the "Time" index into an absolute `DatetimeIndex`
+# instead of float seconds, for every register of the dataset. `REFERENCE_EPOCH` is
+# time zero of the Harp clock in UTC.
+absolute = data.open_dataset("session.harp", epoch=data.REFERENCE_EPOCH).read(44)
 print(absolute.index[:3])
 
 # --- Working from a device module already in hand ----------------------------

src/packages/harp-data/README.md

@@ -53,7 +53,7 @@ df = reader.read(behavior.AnalogData)  # by register class
 
 Prefer the register class where a generated package supplies one, since it is the only form that type-checks and a misspelling is caught before the folder is read. A module built by `create_device_module` resolves its registers as `Any`, so there the class verifies no more than the name does.
 
-The Harp time becomes the DataFrame index named `"Time"`, as float seconds by default or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Data carrying no timestamp raise unless `timestamp=False` is passed. Multi-chunk registers logged as `<DeviceName>_<address>_<suffix>.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked.
+The Harp time becomes the DataFrame index named `"Time"`, as float seconds by default or an absolute `DatetimeIndex` when the dataset is opened with `epoch=REFERENCE_EPOCH`. The anchor is set once for the dataset, since it describes how the recording was made rather than how one register is read. Data carrying no timestamp raise unless `time_index=False` is passed. Multi-chunk registers logged as `<DeviceName>_<address>_<suffix>.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked.
 
 The `<DeviceName>` prefix comes from the `DEVICE_NAME` declared by the device module. Pass `name=` to override it, or to supply one when the module declares an empty name.
 
@@ -69,11 +69,11 @@ from my_device import AnalogData
 
 df = data.parse_to_dataframe(AnalogData, "AnalogData.bin")
 df = data.parse_to_dataframe(
-    AnalogData, raw, timestamp=True, message_type=False, decode_enums=True
+    AnalogData, raw, time_index=True, epoch=None, keep_type=False, decode_enums=True
 )
 ```
 
-With `timestamp=True`, the default, the Harp time becomes the DataFrame index named `"Time"`, as float seconds, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is also passed. Enum fields decode to `pd.Categorical`, and `decode_enums=False` keeps raw codes.
+`time_index` decides the index: `True`, the default, gives the Harp time named `"Time"`, and `False` gives a `RangeIndex`. `epoch` anchors that index, giving float seconds when omitted and an absolute `DatetimeIndex` when set to a datetime such as `REFERENCE_EPOCH`. This function reads one file rather than a dataset, so it takes the anchor directly. Enum fields decode to `pd.Categorical`, and `decode_enums=False` keeps raw codes.
 
 ## From an already-parsed payload
 

src/packages/harp-data/src/harp/data/_dataset.py

@@ -72,6 +72,10 @@ class DatasetReader(Generic[M]):
     when a register was logged as several ``<name>_<address>_<suffix>.bin`` chunks,
     they are concatenated in filename order. Pass ``resolver`` (a :data:`FileResolver`)
     to support an alternative on-disk layout.
+
+    ``epoch`` anchors the time index of every read to absolute time, so one dataset is
+    read on one clock rather than the choice being made per register. It describes how
+    the recording was anchored. The reference Harp clock starts at :data:`REFERENCE_EPOCH`.
     """
 
     def __init__(
@@ -81,12 +85,14 @@ def __init__(
         *,
         name: str | None = None,
         resolver: FileNameResolver = default_file_resolver,
+        epoch: datetime | None = None,
         validate: bool = True,
     ) -> None:
         self._device_module = device_module
         self._root = Path(root)
         self._name_override = name
         self._resolver = resolver
+        self._epoch = epoch
         self._name = self._resolve_name()
         self._paths = dict(self._resolver(self._root, self._name))
         registers = device_module.REGISTER_MAP
@@ -158,9 +164,8 @@ def read(
         register: RegisterKey,
         *,
         suffix: str | None = None,
-        timestamp: bool = True,
-        epoch: datetime | None = None,
-        message_type: bool = False,
+        time_index: bool = True,
+        keep_type: bool = False,
         decode_enums: bool = True,
         demux_bit_masks: bool = False,
     ) -> pd.DataFrame:
@@ -182,17 +187,18 @@ def read(
         ``suffix`` selects a single ``<name>_<address>_<suffix>.bin`` chunk, and naming
         one that is absent raises ``FileNotFoundError`` (default: concatenate every
         chunk for the address). The remaining options match
-        :func:`~harp.data.parse_to_dataframe`.
+        :func:`~harp.data.parse_to_dataframe`, except that the epoch is the one the
+        reader was opened with.
         """
         cls, address = self._resolve(register)
         paths = self._resolve_paths(address, suffix)
         raw = b"".join(p.read_bytes() for p in paths)
         return parse_to_dataframe(
             cls,
             raw,
-            timestamp=timestamp,
-            epoch=epoch,
-            message_type=message_type,
+            time_index=time_index,
+            epoch=self._epoch,
+            keep_type=keep_type,
             decode_enums=decode_enums,
             demux_bit_masks=demux_bit_masks,
         )
@@ -229,6 +235,7 @@ def open_dataset(
     *,
     name: str | None = ...,
     resolver: FileNameResolver = ...,
+    epoch: datetime | None = ...,
     validate: bool = ...,
 ) -> DatasetReader[M]: ...
 
@@ -243,6 +250,7 @@ def open_dataset(
     resolver: FileNameResolver = ...,
     converters: Mapping[str, Any] | None = ...,
     require_converters: bool = ...,
+    epoch: datetime | None = ...,
     validate: bool = ...,
 ) -> DatasetReader[DeviceModule]: ...
 
@@ -256,6 +264,7 @@ def open_dataset(
     resolver: FileNameResolver = default_file_resolver,
     converters: Mapping[str, Any] | None = None,
     require_converters: bool = True,
+    epoch: datetime | None = None,
     validate: bool = True,
 ) -> DatasetReader:
     """Open a de-multiplexed Harp dataset folder and return a :class:`DatasetReader`.
@@ -274,6 +283,10 @@ def open_dataset(
     decoding. These three parameters describe alternative ways to supply a module, so
     they are mutually exclusive, and will raise when more than one is specified.
 
+    ``epoch`` anchors the time index of every read to absolute time, since the anchor
+    describes the recording rather than one register. The reference Harp clock starts
+    at :data:`REFERENCE_EPOCH`, and the default of ``None`` gives float seconds.
+
     ``validate`` cannot rescue a corrupt ``device.yml`` if that schema file is also
     used to build the module. Reading such a folder always requires supplying a module
     obtained elsewhere.
@@ -286,7 +299,12 @@ def open_dataset(
                 "do not apply when one is given. Drop them, or drop the device module."
             )
         return DatasetReader(
-            device_module, root_path, name=name, resolver=resolver, validate=validate
+            device_module,
+            root_path,
+            name=name,
+            resolver=resolver,
+            epoch=epoch,
+            validate=validate,
         )
     schema_path = Path(schema) if schema is not None else root_path / DEVICE_SCHEMA_FILENAME
     if not schema_path.is_file():
@@ -302,5 +320,6 @@ def open_dataset(
         root_path,
         name=name,
         resolver=resolver,
+        epoch=epoch,
         validate=validate and schema is not None,
     )

src/packages/harp-data/src/harp/data/_read.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from typing import Any
 
 import pandas as pd
@@ -76,17 +77,21 @@ def _infer_native_register(raw: bytes) -> type[RegisterBase[Any]]:
 def read(
     source: Source,
     *,
-    timestamp: bool = True,
-    message_type: bool = False,
+    time_index: bool = True,
+    epoch: datetime | None = None,
+    keep_type: bool = False,
 ) -> pd.DataFrame:
     """Read the binary data of a single register, inferring its native layout.
 
     ``source`` may be a file path, raw bytes, or an open binary file. The element
     type, length and timestamp presence are read from the first frame; values
     decode to the matching native numpy type (no enum or bit-mask decoding).
+    The remaining options match :func:`~harp.data.parse_to_dataframe`.
     """
     raw = _read_bytes(source)
     if len(raw) == 0:
         return pd.DataFrame()
     register = _infer_native_register(raw)
-    return parse_to_dataframe(register, raw, timestamp=timestamp, message_type=message_type)
+    return parse_to_dataframe(
+        register, raw, time_index=time_index, epoch=epoch, keep_type=keep_type
+    )

src/packages/harp-data/src/harp/data/_reader.py

@@ -2,14 +2,14 @@
 
 from datetime import datetime
 from pathlib import Path
-from typing import Any, BinaryIO, Union
+from typing import Any, BinaryIO
 
 import numpy as np
 import pandas as pd
 from harp.protocol import RegisterBase
 from numpy.typing import NDArray
 
-Source = Union[str, Path, bytes, bytearray, memoryview, BinaryIO]
+Source = str | Path | bytes | bytearray | memoryview | BinaryIO
 
 _MSG_NAMES = np.array(["_NONE", "Read", "Write", "Event"])
 
@@ -68,38 +68,39 @@ def parse_to_dataframe(
     register: type[RegisterBase[Any]],
     source: Source,
     *,
-    timestamp: bool = True,
-    epoch: Union[datetime, None] = None,
-    message_type: bool = False,
+    time_index: bool = True,
+    epoch: datetime | None = None,
+    keep_type: bool = False,
     decode_enums: bool = True,
     demux_bit_masks: bool = False,
 ) -> pd.DataFrame:
     """Parse all frames of ``register`` from ``source`` into a DataFrame.
 
-    ``source`` may be a file path, raw bytes, or an open binary file object. When
-    ``timestamp`` is set, the Harp time becomes the DataFrame index (named
-    ``"Time"``): float seconds by default, or an absolute ``DatetimeIndex`` when
-    ``epoch`` is given (e.g. :data:`REFERENCE_EPOCH`). ``message_type`` inserts a
-    leading column; ``decode_enums`` controls whether enum fields become
+    ``source`` may be a file path, raw bytes, or an open binary file object.
+    ``time_index`` makes the Harp time the DataFrame index, named ``"Time"``, and
+    ``False`` leaves a ``RangeIndex``. ``epoch`` anchors that index to absolute time,
+    giving a ``DatetimeIndex`` measured from it, where the default of ``None`` gives
+    float seconds; the Harp clock starts at :data:`REFERENCE_EPOCH`. ``keep_type``
+    inserts a leading column; ``decode_enums`` controls whether enum fields become
     ``pd.Categorical`` (True) or raw codes; ``demux_bit_masks`` expands each flag
     (``BitMask``) field into one boolean column per flag member (True) or keeps it
     as a single raw-integer column.
     """
     raw = _read_bytes(source)
-    _data, timestamps, msg_view, payload = register.parse_bulk(raw, parse_timestamp=timestamp)
+    _data, timestamps, msg_view, payload = register.parse_bulk(raw, parse_timestamp=time_index)
     df = payload_to_dataframe(payload, decode_enums=decode_enums, demux_bit_masks=demux_bit_masks)
 
-    if message_type and msg_view is not None:
+    if keep_type and msg_view is not None:
         df.insert(
             0,
             "message_type",
             pd.Categorical(_MSG_NAMES[msg_view & 0x03], categories=_MSG_NAMES[1:]),
         )
-    if timestamp:
+    if time_index:
         if timestamps is None:
             if len(df) > 0:
                 raise ValueError(
-                    "Buffer contains no timestamp data; pass timestamp=False to suppress "
+                    "Buffer contains no timestamp data; pass time_index=False to suppress "
                     "the time index."
                 )
             seconds = np.empty(0, dtype=np.float64)  # empty buffer: empty Time index

tests/data/test_dataset.py

@@ -111,7 +111,7 @@ def test_timestamp_false_gives_range_index(dataset):
     # The diagnostic escape hatch, and the only way to read frames carrying no timestamp.
     mod, _name, root, specs = dataset
     reader = DatasetReader(mod, root)
-    df = reader.read(next(iter(specs)), timestamp=False)
+    df = reader.read(next(iter(specs)), time_index=False)
     assert df.index.name is None
 
 
@@ -129,7 +129,7 @@ def test_untimestamped_frames_raise_value_error(emitted_module, tmp_path):
 
     with pytest.raises(ValueError, match="no timestamp data"):
         reader.read(address)
-    assert len(reader.read(address, timestamp=False)) == 3
+    assert len(reader.read(address, time_index=False)) == 3
 
 
 def test_time_index_is_float_seconds_without_epoch(dataset):
@@ -141,18 +141,36 @@ def test_time_index_is_float_seconds_without_epoch(dataset):
     assert list(df.index) == [0.0, 1.0, 2.0, 3.0, 4.0]  # arange(5) seconds from the fixture
 
 
-def test_epoch_gives_absolute_datetime_index(dataset):
+def test_dataset_epoch_gives_absolute_datetime_index(dataset):
     mod, _name, root, specs = dataset
-    reader = DatasetReader(mod, root)
+    reader = DatasetReader(mod, root, epoch=REFERENCE_EPOCH)
     address = next(iter(specs))
-    df = reader.read(address, epoch=REFERENCE_EPOCH)
+    df = reader.read(address)
     assert isinstance(df.index, pd.DatetimeIndex)
     assert df.index.name == "Time"
     # Harp seconds are measured from the reference epoch (timestamps were arange(5)).
     assert df.index[0] == pd.Timestamp(REFERENCE_EPOCH)
     assert df.index[2] == pd.Timestamp(REFERENCE_EPOCH) + pd.Timedelta(seconds=2)
 
 
+def test_dataset_epoch_applies_to_every_register(dataset):
+    # The anchor is set once for the dataset, so no read can index on a different clock
+    # from its siblings and frames from several registers share one index type.
+    mod, _name, root, specs = dataset
+    reader = DatasetReader(mod, root, epoch=REFERENCE_EPOCH)
+    for address in specs:
+        assert isinstance(reader.read(address).index, pd.DatetimeIndex)
+
+
+def test_parse_takes_epoch_without_dataset(dataset):
+    # parse_to_dataframe has no dataset to take the anchor from, so it accepts one.
+    _mod, _name, _root, specs = dataset
+    cls, buf = specs[next(iter(specs))]
+    df = parse_to_dataframe(cls, buf, epoch=REFERENCE_EPOCH)
+    assert isinstance(df.index, pd.DatetimeIndex)
+    assert df.index[0] == pd.Timestamp(REFERENCE_EPOCH)
+
+
 def test_suffix_chunks_are_concatenated(emitted_module, tmp_path):
     # Chunk suffixes in this test are ISO 8601 UTC timestamps in basic format, so
     # filename order is chronological order. Written newest first to test the sorting.

tests/device/test_emit.py

@@ -535,7 +535,7 @@ def test_emitted_register_bulk_matches_oracle(name, device_registers):
 
     # Cross-read via harp.data: the shared bytes decode to equal frames through
     # either class, including column names and decoded enum labels, which now agree.
-    df_emitted = parse_to_dataframe(emitted, buf, timestamp=False)
-    df_oracle = parse_to_dataframe(oracle, buf, timestamp=False)
+    df_emitted = parse_to_dataframe(emitted, buf, time_index=False)
+    df_oracle = parse_to_dataframe(oracle, buf, time_index=False)
     assert list(df_emitted.columns) == list(df_oracle.columns)
     assert df_emitted.equals(df_oracle)

tests/protocol/test_register.py

@@ -237,8 +237,8 @@ def test_empty_buffer_keeps_columns():
     # so a buffer carrying no frames still renders all of them.
     reg = RegisterU32Array(0x28, length=3)
     records = np.arange(6, dtype=np.uint32).reshape(2, 3)
-    populated = parse_to_dataframe(reg, bytes(reg.format_bulk(records)), timestamp=False)
-    empty = parse_to_dataframe(reg, b"", timestamp=False)
+    populated = parse_to_dataframe(reg, bytes(reg.format_bulk(records)), time_index=False)
+    empty = parse_to_dataframe(reg, b"", time_index=False)
     assert list(empty.columns) == list(populated.columns)
     assert empty.dtypes.equals(populated.dtypes)
     assert len(empty) == 0
@@ -594,7 +594,7 @@ def test_format_bulk_parse_bulk_roundtrip():
     reg = RegisterU16(0x20)
     values = np.array([1, 2, 3], dtype="<u2")
     buf = reg.format_bulk(values, timestamps=[1.0, 2.0, 3.0])
-    df = parse_to_dataframe(reg, bytes(buf), timestamp=False)
+    df = parse_to_dataframe(reg, bytes(buf), time_index=False)
     assert df["value"].tolist() == [1, 2, 3]