From 514c4d946cadaa177da244391a32c70546e0f904 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sat, 22 Aug 2026 02:28:50 +0100 Subject: [PATCH] Add ArrayConverter for array payload members A payload member spanning several elements had no expressible type. The sub-array dtype IdentityConverter needs carries np.void as its scalar type, so the member resolved as void, and pinning the type parameter instead resolves it as the scalar. ArrayConverter takes the element type and a length and resolves as an NDArray of that element. It is typing-only and builds an IdentityConverter over the same sub-array dtype, so the runtime object is the passthrough converter every code path and isinstance already expects. The runtime emitter uses it for a member longer than one element, so a schema-built module and a generated package agree. harp-benchmarks was the one package src tree outside the pyright include, and no test imports its benchmark module, so three parse_to_dataframe calls still passed timestamp and raised TypeError. They now pass time_index, a masked np.int32 member no longer takes a Python int, and the StartPulseTrain default check supplies the two members that dataclass_transform makes required. --- pyproject.toml | 1 + .../src/harp/benchmarks/benchmark.py | 6 ++-- .../src/harp/benchmarks/register_models.py | 16 +++++----- .../src/harp/device/schema/_emit.py | 5 +++- .../src/harp/protocol/__init__.py | 2 ++ .../src/harp/protocol/_payload_converters.py | 29 ++++++++++++++++++- tests/conformance.py | 27 ++++++++++++++++- 7 files changed, 72 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1f3ce03..53677af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,6 +118,7 @@ include = [ "src/packages/harp-device/src", "src/packages/harp-serial/src", "src/packages/harp-data/src", + "src/packages/harp-benchmarks/src", "docs/examples", "tests/conformance.py", ] diff --git a/src/packages/harp-benchmarks/src/harp/benchmarks/benchmark.py b/src/packages/harp-benchmarks/src/harp/benchmarks/benchmark.py index 196e9e3..d324ab0 100644 --- a/src/packages/harp-benchmarks/src/harp/benchmarks/benchmark.py +++ b/src/packages/harp-benchmarks/src/harp/benchmarks/benchmark.py @@ -110,13 +110,13 @@ def benchmark_register(reg: BenchmarkedRegister, path: Path, *, runs: int) -> Re file_bytes=file_bytes, ) df_pre = _time( - lambda: parse_to_dataframe(register, raw, timestamp=reg.timestamped), + lambda: parse_to_dataframe(register, raw, time_index=reg.timestamped), runs=runs, frames=frames, file_bytes=file_bytes, ) df_re = _time( - lambda: parse_to_dataframe(register, path.read_bytes(), timestamp=reg.timestamped), + lambda: parse_to_dataframe(register, path.read_bytes(), time_index=reg.timestamped), runs=runs, frames=frames, file_bytes=file_bytes, @@ -326,7 +326,7 @@ def main() -> None: f"df={_fmt_ms(res.df_preread.mean):>9s}ms" ) if args.head: - df = parse_to_dataframe(reg.register, path.read_bytes(), timestamp=True) + df = parse_to_dataframe(reg.register, path.read_bytes(), time_index=True) print(df.head(5)) print() diff --git a/src/packages/harp-benchmarks/src/harp/benchmarks/register_models.py b/src/packages/harp-benchmarks/src/harp/benchmarks/register_models.py index f8c1c2e..0df07c0 100644 --- a/src/packages/harp-benchmarks/src/harp/benchmarks/register_models.py +++ b/src/packages/harp-benchmarks/src/harp/benchmarks/register_models.py @@ -6,6 +6,7 @@ from harp.protocol import ( AnonymousPayload, + ArrayConverter, BitMask, BoolConverter, Converter, @@ -109,9 +110,7 @@ class AnalogDataPayload(StructPayload[np.float32], length=6): analog0: np.float32 = Field(IdentityConverter(np.float32), offset=0) analog1: np.float32 = Field(IdentityConverter(np.float32), offset=1) analog2: np.float32 = Field(IdentityConverter(np.float32), offset=2) - accelerometer: NDArray[np.float32] = Field( - IdentityConverter(np.dtype((np.float32, (3,)))), offset=3 - ) + accelerometer: NDArray[np.float32] = Field(ArrayConverter(np.float32, 3), offset=3) class AnalogData(RegisterBase[AnalogDataPayload]): @@ -149,9 +148,7 @@ class VersionPayload(StructPayload[np.uint8], length=32): firmware_version: HarpVersion = Field(HarpVersionConverter(np.uint8), offset=3) hardware_version: HarpVersion = Field(HarpVersionConverter(np.uint8), offset=6) core_id: str = Field(StringConverter(3), offset=9) - interface_hash: NDArray[np.uint8] = Field( - IdentityConverter(np.dtype((np.uint8, (20,)))), offset=12 - ) + interface_hash: NDArray[np.uint8] = Field(ArrayConverter(np.uint8, 20), offset=12) class Version(RegisterBase[VersionPayload]): @@ -375,7 +372,7 @@ def main() -> None: # pragma: no cover - manual exploration entry point assert int(p.header) == 7 and int(p.data) == -1234 print("CustomMemberConverter OK") - p = _roundtrip(BitmaskSplitter, BitmaskSplitterPayload(low=0xA, high=0x5)) + p = _roundtrip(BitmaskSplitter, BitmaskSplitterPayload(low=np.int32(0xA), high=np.int32(0x5))) assert int(p.low) == 0xA and int(p.high) == 0x5 assert p.payload_array.tobytes() == bytes([0x5A]) print("BitmaskSplitter OK") @@ -411,7 +408,10 @@ def main() -> None: # pragma: no cover - manual exploration entry point assert p.digital_output == PwmPort.PWM1 and int(p.pulse_width) == 300 assert int(p.frequency) == 200 and int(p.pulse_count) == 50 assert StartPulseTrainPayload.payload_dtype.itemsize == 4 - assert int(StartPulseTrainPayload(pulse_count=np.uint8(3)).frequency) == 1 # defaultValue + partial = StartPulseTrainPayload( + digital_output=PwmPort.PWM0, pulse_width=np.uint16(0), pulse_count=np.uint8(3) + ) + assert int(partial.frequency) == 1 # defaultValue print("StartPulseTrain OK (4 masked members, 2 words, default frequency=1)") p = _roundtrip(EncoderMode, EncoderModeMask.DISPLACEMENT) diff --git a/src/packages/harp-device/src/harp/device/schema/_emit.py b/src/packages/harp-device/src/harp/device/schema/_emit.py index 1dfb3eb..4923e5c 100644 --- a/src/packages/harp-device/src/harp/device/schema/_emit.py +++ b/src/packages/harp-device/src/harp/device/schema/_emit.py @@ -15,6 +15,7 @@ Field, GroupMask, HarpVersionConverter, + ArrayConverter, IdentityConverter, RegisterBase, RegisterFloat, @@ -318,7 +319,9 @@ def _resolve_converter(self, ctx: ConverterContext) -> Converter[Any]: if ctx.mask is not None: return IdentityConverter(ctx.member_dtype) # bit-field: native slice of the element if it is None: - return IdentityConverter(ctx.raw_dtype) # raw passthrough / sub-array + if ctx.length > 1: + return ArrayConverter(ctx.element, ctx.length) # raw passthrough, sub-array + return IdentityConverter(ctx.element) # raw passthrough, single element # A known primitive that didn't fit is re-interpreted per field (``{Name}Converter``); # an unknown interfaceType is a domain type (``{InterfaceType}Converter``). symbol = f"{ctx.name}Converter" if entry is not None else f"{it}Converter" diff --git a/src/packages/harp-protocol/src/harp/protocol/__init__.py b/src/packages/harp-protocol/src/harp/protocol/__init__.py index 989af34..6927a46 100644 --- a/src/packages/harp-protocol/src/harp/protocol/__init__.py +++ b/src/packages/harp-protocol/src/harp/protocol/__init__.py @@ -1,6 +1,7 @@ from ._message import HarpMessage, HarpParseError from ._message_type import MessageType from ._payload_converters import ( + ArrayConverter, BoolConverter, Converter, EnumConverter, @@ -78,6 +79,7 @@ # Converters "Converter", "IdentityConverter", + "ArrayConverter", "StringConverter", "BoolConverter", "EnumConverter", diff --git a/src/packages/harp-protocol/src/harp/protocol/_payload_converters.py b/src/packages/harp-protocol/src/harp/protocol/_payload_converters.py index 65a7a5c..cabccf7 100644 --- a/src/packages/harp-protocol/src/harp/protocol/_payload_converters.py +++ b/src/packages/harp-protocol/src/harp/protocol/_payload_converters.py @@ -1,6 +1,6 @@ import enum as _enum from abc import ABC, abstractmethod -from typing import Any, Generic, TypeVar, cast +from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast from dataclasses import dataclass import numpy as np from numpy.typing import NDArray @@ -121,6 +121,33 @@ def __init__(self) -> None: super().__init__(np.float32) +if TYPE_CHECKING: + + class ArrayConverter(Converter["NDArray[NpScalarT]"]): + """Pass-through converter for a member spanning several elements of one type. + + The array counterpart of :class:`IdentityConverter`, decoding to an ``NDArray`` + of the element type rather than to a single scalar. It exists to carry that type + and builds an :class:`IdentityConverter` over the equivalent sub-array dtype, so + every code path and every ``isinstance`` sees the passthrough converter it + already knows. Passing that dtype directly instead types the member as a scalar, + since a sub-array dtype carries ``np.void`` as its scalar type. + """ + + def __init__(self, dtype: "np.dtype[NpScalarT] | type[NpScalarT]", length: int) -> None: ... + + def decode_scalar(self, view: np.generic) -> "NDArray[NpScalarT]": ... + + def decode_batch(self, view: "NDArray[np.generic]") -> Any: ... + + def encode_into(self, view: "NDArray[np.generic]", value: "NDArray[NpScalarT]") -> None: ... + +else: + + def ArrayConverter(dtype, length): + return IdentityConverter(np.dtype((dtype, (length,)))) + + class BoolConverter(Converter[bool]): """Whole-element ``interfaceType: bool`` (or a single masked bit via ``Field(BoolConverter(), mask=...)``). diff --git a/tests/conformance.py b/tests/conformance.py index a6aa72a..7d4195f 100644 --- a/tests/conformance.py +++ b/tests/conformance.py @@ -12,8 +12,16 @@ from harp.device.client import Device, ITransport from harp.device.core import OperationControl, OperationControlPayload, WhoAmI from harp.device.schema import DeviceModule, DeviceModuleLike, create_device_module -from harp.protocol import HarpMessage, RegisterBase +from harp.protocol import ( + ArrayConverter, + Field, + HarpMessage, + IdentityConverter, + RegisterBase, + StructPayload, +) from harp.serial import open_device +from numpy.typing import NDArray def schema_built_registers(yml: str) -> None: @@ -131,3 +139,20 @@ class Hybrid(Device[None]): device = open_device(Hybrid, port="COM3") assert_type(device, Hybrid) + + +def array_payload_members() -> None: + """A member spanning several elements resolves as an array of the element type. + + The equivalent sub-array dtype passed to IdentityConverter produces the same bytes + but resolves as a scalar, since a sub-array dtype carries np.void as its scalar + type, so ArrayConverter is what carries the element type through the descriptor. + """ + + class Payload(StructPayload[np.float32], length=6): + analog0: np.float32 = Field(IdentityConverter(np.float32), offset=0) + accelerometer: NDArray[np.float32] = Field(ArrayConverter(np.float32, 3), offset=3) + + payload = Payload(analog0=np.float32(0), accelerometer=np.zeros(3, dtype=np.float32)) + assert_type(payload.analog0, np.float32) + assert_type(payload.accelerometer, NDArray[np.float32])