From 2b2e9f7a7a2b70d727b62b96aad229231d58d3d3 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Thu, 10 Sep 2026 08:48:40 +0100 Subject: [PATCH 1/4] Dependency-free script for making zarr nodes --- scripts/mknode.py | 272 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100755 scripts/mknode.py diff --git a/scripts/mknode.py b/scripts/mknode.py new file mode 100755 index 0000000..966cffb --- /dev/null +++ b/scripts/mknode.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Script to create a new zarr node.""" + +from __future__ import annotations + +import json +import logging +import sys +from argparse import ArgumentParser +from collections.abc import Callable +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from shutil import rmtree +from typing import TypeVar + +logger = logging.getLogger("mknode") + +T = TypeVar("T") + +JSON = float | int | str | None | bool | list["JSON"] | dict[str, "JSON"] +JSONObject = dict[str, "JSON"] +METADATA_FILE = "zarr.json" +DATA_TYPES = ["bool"] +for base in ("int", "uint"): + for precision in (8, 16, 32, 64): + DATA_TYPES.append(f"{base}{precision}") +for precision in (32, 64): + DATA_TYPES.append(f"float{precision}") +for precision in (64, 128): + DATA_TYPES.append(f"complex{precision}") + + +def parse_list(string: str, fn: Callable[[str], T], sep: str = ",") -> list[T]: + return [fn(s.strip()) for s in string.split(sep)] + + +def list_parser(fn: Callable[[str], T], sep: str = ",") -> Callable[[str], list[T]]: + return partial(parse_list, fn=fn, sep=sep) + + +def jso(s: str) -> JSONObject: + j = json.loads(s) + if not isinstance(j, dict): + raise TypeError(f"Expected JSON object, got {s}") + return j + + +@dataclass +class ArrayArgs: + shape: list[int] + data_type: str + fill_value: JSON + + @property + def chunk_shape(self) -> list[int]: + return self.shape.copy() + + @classmethod + def maybe_from_args( + cls, + shape: list[int] | None, + data_type: str | None, + fill_value: JSON | None = None, + ): + if shape is None and data_type is None: + return None + if (shape is None) != (data_type is None): + raise ValueError("All array args must be given or none") + if fill_value is None: + if data_type == "bool": + fill_value = False + else: + fill_value = 0 + return cls(shape, data_type, fill_value) # type:ignore + + def get_metadata(self, attributes: JSONObject | None = None) -> JSONObject: + if attributes is None: + attributes = {} + + a2b: JSONObject = {"name": "bytes"} + if self.data_type not in ("bool", "int8", "uint8"): + a2b["configuration"] = {"endian": "little"} + + d = { + "zarr_format": 3, + "node_type": "array", + "shape": self.shape, + "data_type": self.data_type, + "chunk_grid": { + "name": "regular", + "configuration": {"chunk_shape": self.shape}, + }, + "chunk_key_encoding": {"name": "default"}, + "fill_value": self.fill_value, + "codecs": [a2b], + "attributes": attributes, + } + return d + + +@dataclass +class Args: + path: Path + store: Path | None + attributes: JSONObject + force: bool + parents: bool + log_level: int + array_args: ArrayArgs | None + + def __post_init__(self): + if ( + self.store is not None + and self.store != self.path + and self.store not in self.path.parents + ): + raise ValueError("store must be an ancestor of path") + + @classmethod + def parse(cls, raw_args: list[str] | None = None): + parser = ArgumentParser(description=__doc__) + parser.add_argument("path", type=Path, help="file system path to new node") + parser.add_argument( + "--store", + "-s", + type=Path, + help="file system path to store root, which must be an ancestor of the `path` argument; if not given, defaults to the nearest ancestor with the extension .ome.zarr", + ) + parser.add_argument( + "-a", + "--attributes", + type=jso, + help="attributes to add to the new node, as a JSON string representing an object", + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + default=False, + help="if the node already exists, delete it", + ) + parser.add_argument( + "-p", + "--parents", + action="store_true", + default=False, + help="create parent Zarr groups, including the store root and its parent directory, if necessary", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="increase logging verbosity", + ) + g = parser.add_argument_group( + "array", "Additional arguments for creating an array rather than a group." + ) + g.add_argument( + "shape", + type=list_parser(int), + nargs="?", + help="comma-separated list of unsigned integers representing array shape", + ) + g.add_argument( + "datatype", + nargs="?", + choices=DATA_TYPES, + help="data type for the array", + ) + g.add_argument( + "--fill-value", + "-F", + type=jso, + help="JSON string representing fill value to be used; not type-checked", + ) + parsed = parser.parse_args(raw_args) + maybe_array = ArrayArgs.maybe_from_args( + parsed.shape, parsed.datatype, parsed.fill_value + ) + level = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}.get( + parsed.verbose, logging.DEBUG + ) + storepath: Path | None = parsed.store + nodepath: Path = parsed.path + if storepath is None: + if nodepath.name.endswith(".ome.zarr"): + storepath = nodepath + else: + for p in nodepath.parents: + if p.name.endswith(".ome.zarr"): + storepath = p + break + + return cls( + nodepath, + storepath, + parsed.attributes or {}, + parsed.force, + parsed.parents, + level, + maybe_array, + ) + + +def eprint(*args, **kwargs): + kwargs.setdefault("file", sys.stderr) + print(*args, **kwargs) + + +def grp_metadata(attrs: JSONObject | None = None) -> JSONObject: + if attrs is None: + attrs = {} + return {"zarr_format": 3, "node_type": "group", "attributes": attrs} + + +def write_node_metadata(path: Path, metadata: JSONObject): + s = json.dumps(metadata, indent=2, sort_keys=True) + "\n" + p = path.joinpath(METADATA_FILE) + p.write_text(s) + if logger.isEnabledFor(logging.INFO): + logger.info( + "Wrote metadata into %s : %s", p, json.dumps(metadata, sort_keys=True) + ) + + +def write_group_metadata(path: Path, attrs: JSONObject | None = None): + write_node_metadata(path, grp_metadata(attrs)) + + +def main(): + args = Args.parse() + logging.basicConfig(level=args.log_level) + + if args.store is not None: + if not args.store.name.endswith(".ome.zarr"): + logger.warning("Store path should end with .ome.zarr") + + if args.store != args.path: + args.store.mkdir(exist_ok=True, parents=args.parents) + + nodepath = args.path + if nodepath.exists(): + if args.force: + logger.warning("Removing existing node at %s", nodepath) + rmtree(nodepath) + else: + eprint(f"Node already exists at {nodepath} ; use --force to overwrite") + return 1 + nodepath.mkdir(parents=args.parents) + if args.array_args is None: + write_group_metadata(nodepath, args.attributes) + else: + meta = args.array_args.get_metadata(args.attributes) + write_node_metadata(nodepath, meta) + + if args.store is None: + logger.warning( + "No --store given, and could not infer from .ome.zarr extension; parent group metadata will not be written" + ) + else: + while nodepath != args.store: + nodepath = nodepath.parent + if not nodepath.joinpath(METADATA_FILE).exists(): + write_group_metadata(nodepath) + return 0 + + +if __name__ == "__main__": + status = main() + sys.exit(status) From 04eefe1b31c25f0406d33064b7d92972737e94be Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Thu, 10 Sep 2026 09:37:07 +0100 Subject: [PATCH 2/4] minor usability improvement --- scripts/mknode.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/mknode.py b/scripts/mknode.py index 966cffb..e87dacf 100755 --- a/scripts/mknode.py +++ b/scripts/mknode.py @@ -18,6 +18,8 @@ T = TypeVar("T") +ROOT_DIR_EXT = ".zarr" + JSON = float | int | str | None | bool | list["JSON"] | dict[str, "JSON"] JSONObject = dict[str, "JSON"] METADATA_FILE = "zarr.json" @@ -125,7 +127,7 @@ def parse(cls, raw_args: list[str] | None = None): "--store", "-s", type=Path, - help="file system path to store root, which must be an ancestor of the `path` argument; if not given, defaults to the nearest ancestor with the extension .ome.zarr", + help=f"file system path to store root, which must be an ancestor of the `path` argument; if not given, defaults to the nearest ancestor with the extension {ROOT_DIR_EXT}", ) parser.add_argument( "-a", @@ -173,7 +175,7 @@ def parse(cls, raw_args: list[str] | None = None): "--fill-value", "-F", type=jso, - help="JSON string representing fill value to be used; not type-checked", + help="fill value to be used as JSON; not type-checked", ) parsed = parser.parse_args(raw_args) maybe_array = ArrayArgs.maybe_from_args( @@ -185,11 +187,11 @@ def parse(cls, raw_args: list[str] | None = None): storepath: Path | None = parsed.store nodepath: Path = parsed.path if storepath is None: - if nodepath.name.endswith(".ome.zarr"): + if nodepath.name.endswith(ROOT_DIR_EXT): storepath = nodepath else: for p in nodepath.parents: - if p.name.endswith(".ome.zarr"): + if p.name.endswith(ROOT_DIR_EXT): storepath = p break @@ -234,8 +236,8 @@ def main(): logging.basicConfig(level=args.log_level) if args.store is not None: - if not args.store.name.endswith(".ome.zarr"): - logger.warning("Store path should end with .ome.zarr") + if not args.store.name.endswith(ROOT_DIR_EXT): + logger.warning("Store path should end with %s", ROOT_DIR_EXT) if args.store != args.path: args.store.mkdir(exist_ok=True, parents=args.parents) @@ -248,16 +250,22 @@ def main(): else: eprint(f"Node already exists at {nodepath} ; use --force to overwrite") return 1 + nodepath.mkdir(parents=args.parents) + if args.array_args is None: write_group_metadata(nodepath, args.attributes) else: meta = args.array_args.get_metadata(args.attributes) write_node_metadata(nodepath, meta) + if not args.parents: + return 0 + if args.store is None: logger.warning( - "No --store given, and could not infer from .ome.zarr extension; parent group metadata will not be written" + "No --store given, and could not infer from %s extension; parent group metadata will not be written", + ROOT_DIR_EXT, ) else: while nodepath != args.store: From b80555e2349be3d5c0d8eb513e90287825cb6d3c Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Thu, 10 Sep 2026 17:37:27 +0100 Subject: [PATCH 3/4] Use zarr-python in mknode --- scripts/mknode.py | 82 ++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/scripts/mknode.py b/scripts/mknode.py index e87dacf..95c0293 100755 --- a/scripts/mknode.py +++ b/scripts/mknode.py @@ -1,4 +1,8 @@ -#!/usr/bin/env python3 +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">= 3.12" +# dependencies = ["zarr >= 3.3.0"] +# /// """Script to create a new zarr node.""" from __future__ import annotations @@ -14,14 +18,16 @@ from shutil import rmtree from typing import TypeVar +import zarr +from zarr.core.common import JSON, ZarrFormat + logger = logging.getLogger("mknode") T = TypeVar("T") ROOT_DIR_EXT = ".zarr" -JSON = float | int | str | None | bool | list["JSON"] | dict[str, "JSON"] -JSONObject = dict[str, "JSON"] +JSONObject = dict[str, JSON] METADATA_FILE = "zarr.json" DATA_TYPES = ["bool"] for base in ("int", "uint"): @@ -104,25 +110,30 @@ def get_metadata(self, attributes: JSONObject | None = None) -> JSONObject: @dataclass class Args: path: Path - store: Path | None + store: Path attributes: JSONObject force: bool parents: bool log_level: int + zarr_version: ZarrFormat array_args: ArrayArgs | None def __post_init__(self): - if ( - self.store is not None - and self.store != self.path - and self.store not in self.path.parents - ): + if self.store != self.path and self.store not in self.path.parents: raise ValueError("store must be an ancestor of path") @classmethod def parse(cls, raw_args: list[str] | None = None): parser = ArgumentParser(description=__doc__) parser.add_argument("path", type=Path, help="file system path to new node") + parser.add_argument( + "--zarr-version", + "-z", + choices=[2, 3], + type=int, + default=3, + help="Zarr version to write; default 3", + ) parser.add_argument( "--store", "-s", @@ -192,9 +203,17 @@ def parse(cls, raw_args: list[str] | None = None): else: for p in nodepath.parents: if p.name.endswith(ROOT_DIR_EXT): + logger.info("Inferring %s as store root", p) storepath = p break + if storepath is None: + logger.warning( + "No --store given, and could not infer from %s extension; node path will be used as store root, but should be renamed", + ROOT_DIR_EXT, + ) + storepath = nodepath + return cls( nodepath, storepath, @@ -202,6 +221,7 @@ def parse(cls, raw_args: list[str] | None = None): parsed.force, parsed.parents, level, + parsed.zarr_version, maybe_array, ) @@ -235,12 +255,14 @@ def main(): args = Args.parse() logging.basicConfig(level=args.log_level) - if args.store is not None: - if not args.store.name.endswith(ROOT_DIR_EXT): - logger.warning("Store path should end with %s", ROOT_DIR_EXT) - - if args.store != args.path: - args.store.mkdir(exist_ok=True, parents=args.parents) + if args.store != args.path: + args.store.parent.mkdir(exist_ok=True, parents=args.parents) + grp = zarr.open_group(args.store, zarr_format=args.zarr_version) + for name in args.path.relative_to(args.store).parts[:-1]: + if args.parents: + grp = grp.require_group(name) + else: + grp = grp.get_group(name) nodepath = args.path if nodepath.exists(): @@ -251,27 +273,21 @@ def main(): eprint(f"Node already exists at {nodepath} ; use --force to overwrite") return 1 - nodepath.mkdir(parents=args.parents) - if args.array_args is None: - write_group_metadata(nodepath, args.attributes) - else: - meta = args.array_args.get_metadata(args.attributes) - write_node_metadata(nodepath, meta) - - if not args.parents: - return 0 - - if args.store is None: - logger.warning( - "No --store given, and could not infer from %s extension; parent group metadata will not be written", - ROOT_DIR_EXT, + zarr.create_group( + args.path, zarr_format=args.zarr_version, attributes=args.attributes ) else: - while nodepath != args.store: - nodepath = nodepath.parent - if not nodepath.joinpath(METADATA_FILE).exists(): - write_group_metadata(nodepath) + aargs = args.array_args + zarr.create_array( + args.path, + shape=aargs.shape, + dtype=aargs.data_type, + chunks=aargs.chunk_shape, + fill_value=aargs.fill_value, + attributes=args.attributes, + ) + return 0 From 4041f5bec825530665bca8e4108c90cc654ee076 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Thu, 10 Sep 2026 17:45:57 +0100 Subject: [PATCH 4/4] minor formatting --- scripts/mknode.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/mknode.py b/scripts/mknode.py index 95c0293..8b93a28 100755 --- a/scripts/mknode.py +++ b/scripts/mknode.py @@ -24,11 +24,11 @@ logger = logging.getLogger("mknode") T = TypeVar("T") +JSONObject = dict[str, JSON] ROOT_DIR_EXT = ".zarr" - -JSONObject = dict[str, JSON] METADATA_FILE = "zarr.json" + DATA_TYPES = ["bool"] for base in ("int", "uint"): for precision in (8, 16, 32, 64): @@ -257,7 +257,8 @@ def main(): if args.store != args.path: args.store.parent.mkdir(exist_ok=True, parents=args.parents) - grp = zarr.open_group(args.store, zarr_format=args.zarr_version) + mode = "a" if args.parents else "r+" + grp = zarr.open_group(args.store, mode=mode, zarr_format=args.zarr_version) for name in args.path.relative_to(args.store).parts[:-1]: if args.parents: grp = grp.require_group(name)