From f50cce1300cc76203757c503cc9af92e6d1f2203 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 8 Nov 2026 23:11:27 +0100 Subject: [PATCH 1/9] feat(zarr-metadata): add a reader that reports declarations a reader cannot open (#4604) * feat(zarr-metadata): add a reader that reports what a reader cannot open `read_openable_node_metadata_v3` reads a v3 `zarr.json` as `read_node_metadata_v3` does and adds a problem of the new kind `unclaimed_declaration` for each declaration a reader must understand that nothing in scope claims: an unknown codec, data type or chunk grid, or an unknown top-level member without `"must_understand": false`. Its reading has no problems exactly when a reader that understands the scope can open the node. The other readers keep accepting unclaimed declarations, so the format stays open; their docstrings and the parsing guide now point to `unclaimed_declarations` and the new reader. Prompted by refkv's report that an unknown v3 member parses without error. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * docs(zarr-metadata): number the changelog fragment for #440 Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * docs(zarr-metadata): number the unopenable-declarations fragment after upstream PR 4604 Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../zarr-metadata/changes/4604.feature.md | 1 + .../zarr-metadata/docs/parsing-validation.md | 24 +++++- .../zarr-metadata/src/zarr_metadata/_json.py | 13 +++- .../src/zarr_metadata/model/__init__.py | 9 ++- .../src/zarr_metadata/model/_array_v3.py | 14 +++- .../src/zarr_metadata/model/_group_v3.py | 46 ++++++++++- .../model/test_unclaimed_declarations.py | 78 +++++++++++++++++++ 7 files changed, 174 insertions(+), 11 deletions(-) create mode 100644 packages/zarr-metadata/changes/4604.feature.md diff --git a/packages/zarr-metadata/changes/4604.feature.md b/packages/zarr-metadata/changes/4604.feature.md new file mode 100644 index 0000000000..21ba7def2d --- /dev/null +++ b/packages/zarr-metadata/changes/4604.feature.md @@ -0,0 +1 @@ +`zarr_metadata.model.read_openable_node_metadata_v3` reads a v3 `zarr.json` as `read_node_metadata_v3` does, and also reports each declaration a reader must understand to open the node that nothing in scope claims, such as an unknown codec or an unknown top-level member without `"must_understand": false`, as a problem of the new kind `unclaimed_declaration`. Its reading has no problems exactly when a reader that understands the scope can open the node, so a validator does not have to check `unclaimed_declarations` itself. The other readers still accept unclaimed declarations, and the `from_json` and reader docstrings now point to `unclaimed_declarations` and the new reader. diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index e8c111ea93..7383f51b88 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -203,11 +203,29 @@ for problem in problems: # ('configuration', 'level') invalid_value ``` -A declaration with an unknown name is called **unclaimed**, because no definition in the scope claims that name. It is not reported as a problem: it is read as an `UnclaimedDeclaration`, and the checks that depend on it are skipped, as described above. So an empty `reading.problems` does not mean that every declaration is known. A program that decodes chunks should also check that no declaration is unclaimed. A model's `unclaimed_declarations` lists each one, at any depth, by its location, and leaves out an extra field that says `"must_understand": false`, with everything in it, since a reader may ignore it. A reading with problems has no model, and a node with problems cannot be opened either: +A declaration with an unknown name is called **unclaimed**, because no definition in the scope claims that name. It is not reported as a problem: it is read as an `UnclaimedDeclaration`, and the checks that depend on it are skipped, as described above. So an empty `reading.problems` does not mean that every declaration is known. A program that decodes chunks should also check that no declaration is unclaimed. A model's `unclaimed_declarations` lists each one, at any depth, by its location, and leaves out an extra field that says `"must_understand": false`, with everything in it, since a reader may ignore it. To report each of these as a problem too, read the document with `read_openable_node_metadata_v3`. Its problems are those of `read_node_metadata_v3`, then one of kind `unclaimed_declaration` for each declaration a reader must understand that nothing in scope claims. So its reading has no problems exactly when a reader that understands the scope can open the node: ```python test="true" session="parsing-validation" -unclaimed = None if reading.metadata is None else reading.metadata.unclaimed_declarations -can_open = unclaimed is not None and len(unclaimed) == 0 +from zarr_metadata.model import read_openable_node_metadata_v3 + +openable = read_openable_node_metadata_v3( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [{"name": "bytes"}, {"name": "my_codec"}], + "my_member": {"name": "my_member"}, + "ignorable": {"must_understand": False}, + } +) +for problem in openable.problems: + print(problem.loc, problem.kind) +# ('codecs', 1) unclaimed_declaration +# ('my_member',) unclaimed_declaration ``` To make an unknown data type or codec known, add its definition to the scope's slot and read the document again: diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index 69829d32f4..d5de1537a2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -20,7 +20,14 @@ from zarr_metadata._common import JSONValue from zarr_metadata._sentinel import UNSET -ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json", "unknown_key"] +ProblemKind = Literal[ + "missing_key", + "invalid_type", + "invalid_value", + "invalid_json", + "unknown_key", + "unclaimed_declaration", +] """Machine-readable classification of a `ValidationProblem`. - `missing_key`: a required key (document key or store key) is absent. @@ -35,6 +42,10 @@ open since 2023), and many readers reject such a key. It has its own kind so that a caller who tolerates it can distinguish it from a wrong value, and so that it never hides other problems in the same object. +- `unclaimed_declaration`: a declaration that nothing in scope claims, which + a reader must understand to open the node. Only + `read_openable_node_metadata_v3` reports it: the other readers accept an + unclaimed declaration, since that is what keeps the format open. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 0228861744..2501347273 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -25,7 +25,12 @@ `read_repaired_node_metadata_v3` or `read_repaired_node_metadata_v2`. They undo each bug with `repair_node_metadata_v3` or `repair_node_metadata_v2` before the strict read, and report what they -changed. `zarr_metadata.json_schema` writes a JSON Schema of what the v3 +changed. A reader accepts a declaration that nothing in its scope claims +(an unknown codec, or an unknown top-level member), and a model lists each +one a reader must understand in `unclaimed_declarations`. To also report +each one as a problem, so that a reading has no problems exactly when a +reader can open the node, use `read_openable_node_metadata_v3`. +`zarr_metadata.json_schema` writes a JSON Schema of what the v3 readers read, except for the rules. One declaration on its own is read with `zarr_metadata.definition.resolve`. @@ -91,6 +96,7 @@ read_consolidated_metadata_v3, read_group_metadata_v3, read_node_metadata_v3, + read_openable_node_metadata_v3, ) from zarr_metadata.model._repair import ( Repair, @@ -160,6 +166,7 @@ "read_group_metadata_v3", "read_node_metadata_v2", "read_node_metadata_v3", + "read_openable_node_metadata_v3", "read_repaired_consolidated_metadata_v2", "read_repaired_node_metadata_v2", "read_repaired_node_metadata_v3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 061819fb3d..fa175ed005 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -443,14 +443,14 @@ def must_understand_subset( } -def waived_members(values: Mapping[str, ZarrV3ExtensionField]) -> list[str]: +def waived_members(values: Mapping[str, object]) -> list[str]: """The keys of the extra fields in `values` (by key, as written) that say `"must_understand": false`, which a reader may ignore.""" return [key for key, value in values.items() if _waived(value)] -def _waived(value: ZarrV3ExtensionField) -> bool: +def _waived(value: object) -> bool: """Whether `value`, an extra field as written, is an object that says `"must_understand": false`.""" - return isinstance(value, Mapping) and value.get("must_understand") is False + return is_object(value) and value.get("must_understand") is False class ZarrV3ArrayMetadataUpdate(TypedDict, total=False, closed=True): @@ -835,6 +835,10 @@ def from_json( Raises `MetadataValidationError` with every problem the read finds. `read_array_metadata_v3` returns the reading this model is built from, and the problems of a document that has any. + A declaration that nothing in `context` claims, such as an unknown + codec or an unknown top-level member, is not a problem; the model + lists each one a reader must understand in `unclaimed_declarations`. + `read_openable_node_metadata_v3` reports each one as a problem. """ return cls(data, context=context) @@ -919,7 +923,9 @@ def read_array_metadata_v3( type. Unknown top-level keys are allowed (they map to `extra_fields`). A reader must understand each one that does not declare `must_understand: false`; the model reports these as - `must_understand_fields`. + `must_understand_fields`, and every declaration a reader must + understand, at any depth, as `unclaimed_declarations`. + `read_openable_node_metadata_v3` reports each of those as a problem. To apply a policy over the declarations (for example, allow only the core spec's declarations), iterate over the reading's `declarations()`. A value that diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 42f3e61357..82aad927ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -450,7 +450,10 @@ def from_json( scope that claims the member, as `CORE_CONTEXT` does, this is a problem, because the spec requires an object. Use `read_repaired_node_metadata_v3` to read such a store. A member the - spec does not define is kept in `extra_fields`. + spec does not define is kept in `extra_fields`, and is not a + problem when nothing in `context` claims it; the model lists each + such member a reader must understand in `unclaimed_declarations`. + `read_openable_node_metadata_v3` reports each one as a problem. """ return cls(data, context=context) @@ -769,6 +772,44 @@ def read_node_metadata_v3( return ZarrV3UnknownNodeMetadataReading(problems) +def read_openable_node_metadata_v3( + value: object, *, context: ZarrV3Context | None = None +) -> ZarrV3NodeMetadataReading: + """Read `value`, a v3 `zarr.json`, in `context` as `read_node_metadata_v3` does, and also report each declaration a reader must understand to open the node that nothing in `context` claims. + + The spec requires a reader to refuse a node with a declaration it does + not understand: an unknown codec, data type or chunk grid, or an + unknown top-level member that does not say `"must_understand": false`. + `read_node_metadata_v3` accepts these, as `UnclaimedDeclaration`s, + since that is what keeps the format open. This reader reports each one + as a problem of kind `unclaimed_declaration`, at its location, and + gives no model when there is one. A member that says + `"must_understand": false` is ignored whole, with every declaration in + it, as `unclaimed_declarations` ignores it. So the reading has no + problems exactly when a reader that understands `context` can open the + node. The documents a group's `consolidated_metadata` lists are not + read, as `read_group_metadata_v3` does not read them. + """ + reading = read_node_metadata_v3(value, context=context) + # A node reading is only of an array or a group when `value` is an object. + if isinstance(reading, ZarrV3UnknownNodeMetadataReading) or not is_object(value): + return reading + waived = waived_members({key: value[key] for key in reading.extra_fields}) + unclaimed = [ + ValidationProblem( + loc, + f"nothing in scope claims the {kind_name(declaration.read_as)} {declaration.name!r}, " + "which a reader must understand to open the node", + "unclaimed_declaration", + ) + for loc, declaration in unclaimed_declarations_of(reading.declarations(), waived).items() + ] + if len(unclaimed) == 0: + return reading + problems = (*reading.problems, *with_input(unclaimed, value)) + return dataclasses.replace(reading, problems=problems, metadata=None) + + ZarrV3NodeMetadata = TypeAliasType("ZarrV3NodeMetadata", ZarrV3ArrayMetadata | ZarrV3GroupMetadata) """The model of a v3 `zarr.json`: an array model or a group model, as its `node_type` declares.""" @@ -875,7 +916,8 @@ def read_group_metadata_v3( Unknown top-level keys are allowed (they map to `extra_fields`). A reader must understand each one that does not declare `must_understand: false`; the model reports these as - `must_understand_fields`. A `consolidated_metadata` member, if + `must_understand_fields`, and `read_openable_node_metadata_v3` reports + each one as a problem. A `consolidated_metadata` member, if present and claimed by the scope, as `CORE_CONTEXT` claims it, is checked against the envelope the core spec fixes. The documents it lists are not read; `read_consolidated_metadata_v3` reads them. This is the diff --git a/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py b/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py index 02b5521882..e66914b6d2 100644 --- a/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py +++ b/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py @@ -7,6 +7,7 @@ import pytest from typing_extensions import TypedDict +from zarr_metadata import UNSET from zarr_metadata.definition import UnclaimedDeclaration from zarr_metadata.model import ( ZarrV2ArrayMetadata, @@ -15,6 +16,8 @@ ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV3NodeMetadata, + read_node_metadata_v3, + read_openable_node_metadata_v3, ) from zarr_metadata.v3 import CORE_CONTEXT from zarr_metadata.v3.definition import ( @@ -190,3 +193,78 @@ def test_node_type_tells_a_type_checker_which_model_a_node_is() -> None: for node in nodes: if node.zarr_format == 3 and node.node_type == "group": assert node.consolidated() is None + + +@pytest.mark.parametrize( + ("document", "context", "unclaimed", "problems"), + [ + (ARRAY, CORE_CONTEXT, set(), set()), + ({**ARRAY, "codecs": ("bytes", ZSTD)}, CORE_CONTEXT, {("codecs", 1)}, set()), + ( + {**ARRAY, "codecs": (SHARD,)}, + CORE_CONTEXT, + {("codecs", 0, "configuration", "codecs", 1)}, + set(), + ), + ({**ARRAY, "acme": 1}, CORE_CONTEXT, {("acme",)}, set()), + ({**ARRAY, "acme": {"must_understand": False}}, CORE_CONTEXT, set(), set()), + ({**GROUP, "acme": {"name": "acme"}}, CORE_CONTEXT, {("acme",)}, set()), + ({**GROUP, "acme": {"must_understand": False}}, CORE_CONTEXT, set(), set()), + ( + {**ARRAY, "acme.mirror": {"codec": {"name": "acme.zip"}}}, + MIRRORING, + {("acme.mirror", "codec")}, + set(), + ), + ( + {**ARRAY, "acme.mirror": {"codec": {"name": "acme.zip"}, "must_understand": False}}, + MIRRORING, + set(), + set(), + ), + ( + {**ARRAY, "fill_value": "x", "codecs": ("bytes", ZSTD)}, + CORE_CONTEXT, + {("codecs", 1)}, + {("fill_value",)}, + ), + ({**GROUP, "node_type": "acme"}, CORE_CONTEXT, set(), {("node_type",)}), + ("not a node", CORE_CONTEXT, set(), {()}), + ], + ids=[ + "all-claimed", + "a-codec", + "a-codec-inside-a-shard", + "an-extra-member", + "a-waived-extra-member", + "a-group-member", + "a-waived-group-member", + "a-codec-inside-a-claimed-member", + "a-codec-inside-a-claimed-member-that-waives", + "with-other-problems", + "an-unknown-node-type", + "not-an-object", + ], +) +def test_read_openable_node_metadata_v3_reports_what_a_reader_must_understand( + document: object, + context: Any, # noqa: ANN401 - a v3 scope + unclaimed: set[tuple[str | int, ...]], + problems: set[tuple[str | int, ...]], +) -> None: + """`read_openable_node_metadata_v3` keeps every problem `read_node_metadata_v3` finds and adds one of kind `unclaimed_declaration` at the location of each declaration `unclaimed_declarations` lists, with what the document writes there as its input, whether or not the document has other problems. It gives a model exactly when there are no problems, and the model is the one `read_node_metadata_v3` gives.""" + plain = read_node_metadata_v3(document, context=context) + reading = read_openable_node_metadata_v3(document, context=context) + assert reading.problems[: len(plain.problems)] == plain.problems + assert {problem.loc for problem in plain.problems} == problems + added = reading.problems[len(plain.problems) :] + assert {problem.loc for problem in added} == unclaimed + assert all(problem.kind == "unclaimed_declaration" for problem in added) + assert all(problem.input is not UNSET for problem in added) + if plain.metadata is not None: + assert set(plain.metadata.unclaimed_declarations) == unclaimed + if len(reading.problems) == 0: + assert reading.metadata == plain.metadata + assert reading.metadata is not None + else: + assert reading.metadata is None From 97834892022b899ba512ef470387add6b75fd9fa Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Sun, 8 Nov 2026 23:41:57 +0100 Subject: [PATCH 2/9] docs(zarr-metadata): split the landing page into guide pages and link each spec claim (#4605) * docs(zarr-metadata): split the landing page into guide pages and link each spec claim Shrink index.md to a short pitch, one example and a reader-by-reader list of where to go. Move the rest into guide pages: models and readers, what the readers check, consolidated metadata, repairing known writer bugs, writing an extension, JSON Schema and Pydantic. Each claim backed by a Zarr spec links to the spec section that makes it; the consolidated metadata page says what the specs do and do not settle and links the zarr-specs PRs and issues and the original zarr-python implementation. Trim the README so it no longer mirrors the landing page, and say "context" where the pages said "scope". Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): number the guide-pages fragment after its fork PR Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): say "member" for a JSON name/value pair in the guide pages The rename branch keeps "member" for a name/value pair of a metadata object, so the new pages follow: "a top-level member that the spec does not define is an extra field". "Extra field" and "extension field" stay, after the names extra_fields and ZarrV3ExtensionField. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): say "member" for a JSON name/value pair on every page Sweep the parsing-validation and context pages, which said "field" for a name/value pair of a metadata object, and state the one exception on the models page: "extra field" follows the spec's "additional fields" and the names extra_fields, ExtraFieldDefinition and must_understand_fields; "field" otherwise means a field of a Python class, or a part of a structured data type. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): say "member" in the two must_understand_fields sentences The models page said "each unclaimed field" and "any such field" where the landing page and every neighboring sentence say "member". Roborev flagged the mismatch. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): say why the additional-fields link points at the latest spec v3.1.0 has no "Additional fields" heading; PR 373 added it after the release, so the link cannot be pinned. Say so beside the link. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): split the additional-fields sentence and date its claim Roborev found the three-clause sentence hard to parse and its "no tagged release yet" unanchored in time. Two sentences now, anchored to v3.1.0, with a note to pin the link once a release carries the heading. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): fix six wording and example issues roborev found on the guide pages The pydantic example now builds the manifest with model_validate, so it type-checks as the next sentence claims. The consolidated page says three other readers take a context, not two. The equality section quotes the v2 spec on a missing .zattrs in two sentences. The two numbered Sphinx anchors are replaced by line ranges in the spec source at the v3.1.0 tag. The landing page says once that the library does no chunk or store I/O. The extensions page example and the zarr_metadata.definition docstring example are pinned to each other by a test. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * test(zarr-metadata): make the example-drift test say what it could not find Roborev asked for diagnostics: the test now asserts there is exactly one extensions-session block and exactly one marker line in the docstring, each with a message, and compares line lists so a mismatch points at the differing line. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 * docs(zarr-metadata): build the extension example with union and say "context" throughout On main a Context has no update method, so the acme.lz4 example on the extensions page now adds its definition with union, as the definition module docstring does. Two leftover "scope"s in the parsing page say "context", and the models page says "member" for an extra field. The example-drift test imports at module level. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * docs(zarr-metadata): number the guide-pages fragment after upstream PR 4605 Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * docs(zarr-metadata): fence the module docstring examples as python so the API pages highlight them The examples in the definition, json_schema, pydantic and typed_json module docstrings were indented code blocks. An indented block names no language and pymdownx.highlight does not guess one, so the API reference rendered them as plain text. Each is now a ```python fence, and the example-drift test reads the definition docstring's fence. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * docs(zarr-metadata): write the guide pages in short sentences and lists Long comma-chained sentences on the guide pages, README and landing page are split into short sentences, and enumerations become bulleted or numbered lists. models, validation-rules, consolidated-metadata, repair, extensions, json-schema and pydantic are restructured; context and parsing-validation keep their narrative voice and only have their long sentences split. No link, claim or code block changes. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 --- packages/zarr-metadata/README.md | 410 +--------------- packages/zarr-metadata/changes/4605.doc.md | 1 + packages/zarr-metadata/docs/api/index.md | 9 +- .../docs/consolidated-metadata.md | 143 ++++++ packages/zarr-metadata/docs/context.md | 6 +- packages/zarr-metadata/docs/extensions.md | 116 +++++ packages/zarr-metadata/docs/index.md | 453 +++--------------- packages/zarr-metadata/docs/json-schema.md | 42 ++ packages/zarr-metadata/docs/models.md | 220 +++++++++ .../zarr-metadata/docs/parsing-validation.md | 106 ++-- packages/zarr-metadata/docs/pydantic.md | 66 +++ packages/zarr-metadata/docs/repair.md | 59 +++ .../zarr-metadata/docs/validation-rules.md | 148 ++++++ packages/zarr-metadata/mkdocs.yml | 12 +- .../src/zarr_metadata/definition/__init__.py | 54 ++- .../src/zarr_metadata/json_schema.py | 16 +- .../src/zarr_metadata/pydantic.py | 18 +- .../src/zarr_metadata/typed_json.py | 14 +- packages/zarr-metadata/tests/test_docs.py | 30 ++ 19 files changed, 1055 insertions(+), 868 deletions(-) create mode 100644 packages/zarr-metadata/changes/4605.doc.md create mode 100644 packages/zarr-metadata/docs/consolidated-metadata.md create mode 100644 packages/zarr-metadata/docs/extensions.md create mode 100644 packages/zarr-metadata/docs/json-schema.md create mode 100644 packages/zarr-metadata/docs/models.md create mode 100644 packages/zarr-metadata/docs/pydantic.md create mode 100644 packages/zarr-metadata/docs/repair.md create mode 100644 packages/zarr-metadata/docs/validation-rules.md diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index a01ed26842..d4f3c6579a 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -2,410 +2,46 @@ Documentation: -Basic tools for modelling Zarr metadata, with minimal dependencies. +Types, models and validators for Zarr metadata documents, with minimal +dependencies. `zarr-metadata` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata) -and released independently of `zarr` itself. Install it with: +and released independently of `zarr`. Install it with: ``` pip install zarr-metadata ``` -## Who needs this - -This library supports software that interacts with Zarr metadata documents. - ## What this is -This is *not* a full Zarr implementation. It's a collection of data structures and routines that -closely model the content of the Zarr specifications, such as: - -- **Typed JSON shapes** ([`zarr_metadata.v2`](https://zarr-metadata.readthedocs.io/en/latest/api/v2/) and - [`zarr_metadata.v3`](https://zarr-metadata.readthedocs.io/en/latest/api/v3/)): `TypedDict` definitions and - `Literal` aliases for the JSON documents specified by the - [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) and - [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) - specifications, plus types for - [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/) and a - few widely-used-but-unspecified entities (e.g. consolidated metadata). -- **Document models** ([`zarr_metadata.model`](https://zarr-metadata.readthedocs.io/en/latest/api/model/)): immutable - models of whole metadata documents, each one a document and the scope - it was read in, with readers that report every problem at its - location, and store-key (de)serialization. A document - produced by `to_json` shares no mutable state with the model that produced - it. -- **Definitions and scopes** ([`zarr_metadata.definition`](https://zarr-metadata.readthedocs.io/en/latest/api/definition/), - with each format's kinds in `zarr_metadata.v3.definition` and - `zarr_metadata.v2.definition`): each extension's configuration as a - `TypedDict` with its rules, and the scope that decides which definitions - read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every - definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS_CONTEXT`. -- **JSON type checking** ([`zarr_metadata.typed_json`](https://zarr-metadata.readthedocs.io/en/latest/api/typed_json/)): - `check` checks a JSON value against any `TypedDict` and reports every - problem with its location. -- **Problems** ([`zarr_metadata.problems`](https://zarr-metadata.readthedocs.io/en/latest/api/problems/)): the problem - values every reader returns, and the error that carries them. -- **JSON Schema** ([`zarr_metadata.json_schema`](https://zarr-metadata.readthedocs.io/en/latest/api/json_schema/)): the - JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. -- **Optional Pydantic integration** ([`zarr_metadata.pydantic`](https://zarr-metadata.readthedocs.io/en/latest/api/pydantic/), - requires Pydantic 2.13 or newer): each model as a Pydantic field type that - validates raw documents through the same strict parser. - -## What this is for - -- **Static Type Checking** The public `TypedDict` definitions describe the static JSON shape of Zarr - metadata. - -- **Dynamic JSON validation** For strict, loc-aware validation of JSON loaded from disk, use the - model parser: - - ```python test="true" session="readme" - import json - from zarr_metadata.model import ZarrV3ArrayMetadata - - with open("zarr.json", "rb") as f: - raw = json.load(f) - - metadata = ZarrV3ArrayMetadata.from_json(raw) - ``` - - The optional Pydantic integration delegates raw input to the same strict - parser and returns the same normalized model class: - - ```python test="true" session="readme" - from pydantic import TypeAdapter - import zarr_metadata.pydantic as zmp - - adapter: TypeAdapter[zmp.ZarrV3ArrayMetadataField] = TypeAdapter(zmp.ZarrV3ArrayMetadataField) - metadata = adapter.validate_python(raw) - encoded = metadata.to_key_value()["zarr.json"] - ``` - - A field annotated `Annotated[ZarrV3ArrayMetadata, zmp.InScope(CONTEXT)]` - is read in `CONTEXT`, whatever the validation context holds, and a model - instance given to it is read again in `CONTEXT`. - - Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape - adapter, not a Zarr conformance validator; it may coerce values or discard - members that the strict model parser rejects. This is better suited for handling input provided - from Python objects (e.g., user-authored dicts) than JSON. - -## Validation boundary - -The readers and model constructors enforce the declared document structure and a small set -of context-free consistency rules, e.g.: -- fixed format literals, -- finite JSON numbers outside `attributes` and the members nothing in scope reads -- non-negative dimensions -- one `dimension_names` entry per array dimension. - -In a v3 document, validation also checks each extension point: the data -type, the chunk grid, the chunk key encoding, each codec, and each storage -transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. -`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no -specification registers. -If the definition rejects a configuration, the reader rejects it too, -and a configuration key the definition does not declare is reported as -`unknown_key`. If no definition in the scope claims a name, the reader -does not check that extension point, and the consumer decides whether to -support it. - -Some checks relate one part of a v3 document to another: - -- The fill value is checked against the data type the document names, by - that data type's definition. -- The chunk grid is checked against the shape, by the grid's definition. -- The codecs are checked as a pipeline. In order, each codec's definition - checks the codec against the chunk it receives. This includes a shard's - inner codecs and index codecs. - -The readers do no arithmetic on values. For example, they do not check -whether a fill value survives a `cast_value` round trip. - -Some choices that the spec text leaves open, or answers in two ways: - -- **`attributes` may hold `NaN`, `Infinity` and `-Infinity`.** The spec - does not interpret attributes, and zarr-python and xarray write these - numbers there (for example, a CF `_FillValue`). The models read them, - and `to_key_value` writes them back as bare tokens, which a strict JSON - parser rejects. So may a member the spec does not define when no - definition in scope claims it: nothing reads it, so it is kept as - written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite - number anywhere, including in attributes. -- **A reader walks at most 256 levels of nesting.** A value nested deeper - is an `invalid_value` problem at the first level past the limit, - wherever it is. Every reader, writer and comparison uses one stack - frame per level, and `copy.deepcopy` (and `pickle` before Python 3.12) - uses two. A document at the limit therefore uses about half of the - interpreter's default recursion limit, and the rest is left for the - caller. -- **`consolidated_metadata: null` is a problem.** A zarr-python 3.0.x bug - wrote it. The spec requires an object, and the package accepts only an - object. A reader of those stores removes the key before reading. -- **An extension name must follow the spec's naming rule**, - `^[a-z][a-z0-9-_.]+$`, or be a URI, which earlier versions of the spec - required. Any other name is rejected before any definition is - consulted, so `""` and `"foo/bar"` are problems, not unknown extensions. -- **`must_understand: false` is rejected at every extension point**, - including codecs and storage transformers, although the core spec names - only the data type, chunk grid and chunk key encoding. A reader that - skips a codec reads wrong bytes, just as a reader that skips a data type - reads wrong values. On an unknown top-level member, `must_understand` - keeps its meaning: a reader can skip that member. -- **A v2 datetime or timedelta dtype must have a unit**, as in ` Iterator[ValidationProblem]: + if "dictionary" in configuration and "dictionary_size" not in configuration: + yield ValidationProblem( + ("dictionary_size",), "a dictionary needs its size", "missing_key" + ) + + +ACME_LZ4 = CodecDefinition( + name="acme.lz4", + configuration=AcmeLz4Configuration, + kind="bytes_bytes", + size="dynamic", + rules=acme_lz4_rules, +) +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[ACME_LZ4])) +``` + +Pass `context=CONTEXT` to any reader or model constructor. A document that +names `acme.lz4` is then checked against this definition. + +## Rules for the pieces + +**The `TypedDict`** + +- It must be a `typing_extensions.TypedDict`. +- It must say how undeclared keys are handled: `closed=True`, + `extra_items=`, or `closed=False`. An open-by-default `TypedDict` is + refused, because it would accept a misspelled key silently. + +**Its members** must be JSON shapes: + +- `int`, `float`, `bool`, `str`, `None` or `JSONValue`, +- a `Literal`, +- `tuple[T, ...]` or `tuple[T1, T2]`, +- a union, +- a `TypedDict` or `Mapping[str, V]`, +- a `NewType` or a type alias. + +A number may carry bounds from annotated-types: `Gt`, `Ge`, `Lt`, `Le` or +`Interval`. Any other `Annotated` metadata is refused when the definition +is built. + +**The rules function** + +- It runs only on a configuration within its bounds, so it can rely on + them. +- It reports the location of each problem. The value there becomes the + problem's `input`. +- Show values in messages with `zarr_metadata.problems.shown`. Then they + read like the package's own messages. + +**Pickling.** Define each function at module top level. A model holds the +definitions that read it, so the model can be pickled only if they can. + +## Other kinds of definition + +Other kinds declare more: + +- A data type declares its fill value (`fill_value`, `fill_value_rules`, + `fill_value_canonical`). It also declares how its values are stored + (`storage`). +- A chunk grid declares which shapes it fits (`shape_rules`) and its chunk + lengths (`chunk_lengths`). +- A codec declares: + - what it checks about the chunk it receives (`chunk_rules`), + - what it hands on (`transition`), + - any pipelines it contains (`pipelines`). + +The [`zarr_metadata.v3.definition`](api/v3/definition.md) docstring +describes each of these. diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index b139dfd307..da545079e3 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -1,411 +1,92 @@ # zarr-metadata -Basic tools for modelling Zarr metadata, with minimal dependencies. +Types, models and validators for Zarr metadata documents, with minimal +dependencies. `zarr-metadata` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata) -and released independently of `zarr` itself. Install it with: +and released independently of `zarr`. Install it with: ``` pip install zarr-metadata ``` -## Who needs this - -This library supports software that interacts with Zarr metadata documents. - ## What this is -This is *not* a full Zarr implementation. It's a collection of data structures and routines that -closely model the content of the Zarr specifications, such as: - -- **Typed JSON shapes** ([`zarr_metadata.v2`](api/v2.md) and - [`zarr_metadata.v3`](api/v3/index.md)): `TypedDict` definitions and - `Literal` aliases for the JSON documents specified by the - [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) and - [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) - specifications, plus types for - [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/) and a - few widely-used-but-unspecified entities (e.g. consolidated metadata). -- **Document models** ([`zarr_metadata.model`](api/model.md)): immutable - models of whole metadata documents, each one a document and the scope - it was read in, with readers that report every problem at its - location, and store-key (de)serialization. A document - produced by `to_json` shares no mutable state with the model that produced - it. -- **Definitions and scopes** ([`zarr_metadata.definition`](api/definition.md), - with each format's kinds in `zarr_metadata.v3.definition` and - `zarr_metadata.v2.definition`): each extension's configuration as a - `TypedDict` with its rules, and the scope that decides which definitions - read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every - definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS_CONTEXT`. -- **JSON type checking** ([`zarr_metadata.typed_json`](api/typed_json.md)): - `check` checks a JSON value against any `TypedDict` and reports every - problem with its location. -- **Problems** ([`zarr_metadata.problems`](api/problems.md)): the problem - values every reader returns, and the error that carries them. -- **JSON Schema** ([`zarr_metadata.json_schema`](api/json_schema.md)): the - JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. -- **Optional Pydantic integration** ([`zarr_metadata.pydantic`](api/pydantic.md), - requires Pydantic 2.13 or newer): each model as a Pydantic field type that - validates raw documents through the same strict parser. - -## What this is for - -- **Static Type Checking** The public `TypedDict` definitions describe the static JSON shape of Zarr - metadata. - -- **Dynamic JSON validation** For strict, location-aware validation of JSON loaded from disk, use the - model parser: - - ```python test="true" session="index" - import json - from zarr_metadata.model import ZarrV3ArrayMetadata - - with open("zarr.json", "rb") as f: - raw = json.load(f) - - metadata = ZarrV3ArrayMetadata.from_json(raw) - ``` - - The optional Pydantic integration delegates raw input to the same strict - parser and returns the same normalized model class: - - ```python test="true" session="index" - from pydantic import TypeAdapter - import zarr_metadata.pydantic as zmp - - adapter: TypeAdapter[zmp.ZarrV3ArrayMetadataField] = TypeAdapter(zmp.ZarrV3ArrayMetadataField) - metadata = adapter.validate_python(raw) - encoded = metadata.to_key_value()["zarr.json"] - ``` - - A field annotated `Annotated[ZarrV3ArrayMetadata, zmp.InScope(CONTEXT)]` - is read in `CONTEXT`, whatever the validation context holds, and a model - instance given to it is read again in `CONTEXT`. - - Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape - adapter, not a Zarr conformance validator; it may coerce values or discard - members that the strict model parser rejects. This is better suited for handling input provided - from Python objects (e.g., user-authored dicts) than JSON. - -## Validation boundary - -The readers and model constructors enforce the declared document structure and a small set -of context-free consistency rules, e.g.: -- fixed format literals, -- finite JSON numbers outside `attributes` and the members nothing in scope reads -- non-negative dimensions -- one `dimension_names` entry per array dimension. - -In a v3 document, validation also checks each extension point: the data -type, the chunk grid, the chunk key encoding, each codec, and each storage -transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. -`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no -specification registers. -If the definition rejects a configuration, the reader rejects it too, -and a configuration key the definition does not declare is reported as -`unknown_key`. If no definition in the scope claims a name, the reader -does not check that extension point, and the consumer decides whether to -support it. - -Making the context and explicit part of metadata parsing allows different libraries, which might support different -sets of Zarr features, to formally represent exactly the codecs, data types, etc they support when -parsing metadata. - -Some checks relate one part of a v3 document to another: - -- The fill value is checked against the data type the document names, by - that data type's definition. -- The chunk grid is checked against the shape, by the grid's definition. -- The codecs are checked as a pipeline. In order, each codec's definition - checks the codec against the chunk it receives. This includes a shard's - inner codecs and index codecs. - -The readers do no arithmetic on values. For example, they do not check -whether a fill value survives a `cast_value` round trip. - -Some choices that the spec text leaves open, or answers in two ways: +A collection of data structures and routines that closely model the +metadata rules in the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) +and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) +specifications and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/): -- **`attributes` may hold `NaN`, `Infinity` and `-Infinity`.** The spec - does not interpret attributes, and zarr-python and xarray write these - numbers there (for example, a CF `_FillValue`). The models read them, - and `to_key_value` writes them back as bare tokens, which a strict JSON - parser rejects. So may a member the spec does not define when no - definition in scope claims it: nothing reads it, so it is kept as - written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite - number anywhere, including in attributes. -- **A reader walks at most 256 levels of nesting.** A value nested deeper - is an `invalid_value` problem at the first level past the limit, - wherever it is. Every reader, writer and comparison uses one stack - frame per level, and `copy.deepcopy` (and `pickle` before Python 3.12) - uses two. A document at the limit therefore uses about half of the - interpreter's default recursion limit, and the rest is left for the - caller. -- **`consolidated_metadata: null` is a problem.** A zarr-python 3.0.x bug - wrote it. The spec requires an object, and the package accepts only an - object. A reader of those stores removes the key before reading. -- **An extension name must follow the spec's naming rule**, - `^[a-z][a-z0-9-_.]+$`, or be a URI, which earlier versions of the spec - required. Any other name is rejected before any definition is - consulted, so `""` and `"foo/bar"` are problems, not unknown extensions. -- **`must_understand: false` is rejected at every extension point**, - including codecs and storage transformers, although the core spec names - only the data type, chunk grid and chunk key encoding. A reader that - skips a codec reads wrong bytes, just as a reader that skips a data type - reads wrong values. On an unknown top-level member, `must_understand` - keeps its meaning: a reader can skip that member. -- **A v2 datetime or timedelta dtype must have a unit**, as in ` zarr_metadata.definition': api/definition.md diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 019725679a..2ba4055ba6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -116,40 +116,42 @@ a closure cannot be pickled, and a `functools.partial` can be pickled but does not compare equal to its loaded copy. - from collections.abc import Iterator - from typing import Annotated, NotRequired +```python +from collections.abc import Iterator +from typing import Annotated, NotRequired - from annotated_types import Ge - from typing_extensions import TypedDict +from annotated_types import Ge +from typing_extensions import TypedDict - from zarr_metadata import ValidationProblem - from zarr_metadata.definition import Nested - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context +from zarr_metadata import ValidationProblem +from zarr_metadata.definition import Nested +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context - class AcmeLz4Configuration(TypedDict, closed=True): - acceleration: Annotated[int, Ge(1)] - dictionary: NotRequired[str] - dictionary_size: NotRequired[Annotated[int, Ge(1)]] +class AcmeLz4Configuration(TypedDict, closed=True): + acceleration: Annotated[int, Ge(1)] + dictionary: NotRequired[str] + dictionary_size: NotRequired[Annotated[int, Ge(1)]] - def acme_lz4_rules( - configuration: AcmeLz4Configuration, nested: Nested - ) -> Iterator[ValidationProblem]: - if "dictionary" in configuration and "dictionary_size" not in configuration: - yield ValidationProblem( - ("dictionary_size",), "a dictionary needs its size", "missing_key" - ) +def acme_lz4_rules( + configuration: AcmeLz4Configuration, nested: Nested +) -> Iterator[ValidationProblem]: + if "dictionary" in configuration and "dictionary_size" not in configuration: + yield ValidationProblem( + ("dictionary_size",), "a dictionary needs its size", "missing_key" + ) - ACME_LZ4 = CodecDefinition( - name="acme.lz4", - configuration=AcmeLz4Configuration, - kind="bytes_bytes", - size="dynamic", - rules=acme_lz4_rules, - ) - CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[ACME_LZ4])) +ACME_LZ4 = CodecDefinition( + name="acme.lz4", + configuration=AcmeLz4Configuration, + kind="bytes_bytes", + size="dynamic", + rules=acme_lz4_rules, +) +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[ACME_LZ4])) +``` The TypedDict declares how keys it does not declare are handled. With `closed=True`, such a key is a problem, as above. With `extra_items=`, diff --git a/packages/zarr-metadata/src/zarr_metadata/json_schema.py b/packages/zarr-metadata/src/zarr_metadata/json_schema.py index 2296b78652..7ff4bde29d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/json_schema.py @@ -15,13 +15,15 @@ accepts a number with no fractional part, such as `1.0`, as an integer, while `check` requires `1`: - from zarr_metadata.json_schema import json_schema - - json_schema(GzipCodecConfiguration) - # {'$schema': 'https://json-schema.org/draft/2020-12/schema', - # 'type': 'object', - # 'properties': {'level': {'type': 'integer', 'minimum': 0, 'maximum': 9}}, - # 'required': ['level'], 'additionalProperties': False} +```python +from zarr_metadata.json_schema import json_schema + +json_schema(GzipCodecConfiguration) +# {'$schema': 'https://json-schema.org/draft/2020-12/schema', +# 'type': 'object', +# 'properties': {'level': {'type': 'integer', 'minimum': 0, 'maximum': 9}}, +# 'required': ['level'], 'additionalProperties': False} +``` **One v3 declaration.** `declaration_json_schema_v3(kind, context=None)` writes one declaration read as `kind` in the scope `context`: a declaration that a definition in diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 4434660d8f..308519c0ef 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -29,9 +29,11 @@ For example: - TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(document, context=CONTEXT) - ArrayManifest.model_validate(data, context={"zarr_metadata_context": CONTEXT}) - Mixed.model_validate(data, context={"zarr_metadata_context": V3, "zarr_metadata_context_v2": V2}) +```python +TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(document, context=CONTEXT) +ArrayManifest.model_validate(data, context={"zarr_metadata_context": CONTEXT}) +Mixed.model_validate(data, context={"zarr_metadata_context": V3, "zarr_metadata_context_v2": V2}) +``` An existing model instance passes through unchanged when no scope is given. When one is, the instance is read again in it with `refined_in`, so @@ -63,11 +65,13 @@ Usage: - import zarr_metadata.pydantic as zmp +```python +import zarr_metadata.pydantic as zmp - class ArrayManifest(BaseModel): - path: str - metadata: zmp.ZarrV3ArrayMetadataField +class ArrayManifest(BaseModel): + path: str + metadata: zmp.ZarrV3ArrayMetadataField +``` Static type checkers see each field type as its core model class, so `manifest.metadata` is a `zarr_metadata.model.ZarrV3ArrayMetadata`. diff --git a/packages/zarr-metadata/src/zarr_metadata/typed_json.py b/packages/zarr-metadata/src/zarr_metadata/typed_json.py index 30ed95f695..982a3d4868 100644 --- a/packages/zarr-metadata/src/zarr_metadata/typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/typed_json.py @@ -6,14 +6,16 @@ value against the TypedDict and returns a value of the TypedDict (or None) and every problem, each located in the value: - import json +```python +import json - from zarr_metadata import ZarrV3ArrayMetadataJSON - from zarr_metadata.typed_json import check +from zarr_metadata import ZarrV3ArrayMetadataJSON +from zarr_metadata.typed_json import check - document, problems = check(json.loads(raw), ZarrV3ArrayMetadataJSON) - for problem in problems: - print(problem.loc, problem.kind, problem.message) +document, problems = check(json.loads(raw), ZarrV3ArrayMetadataJSON) +for problem in problems: + print(problem.loc, problem.kind, problem.message) +``` `check` reads a TypedDict as the typing spec defines it, regardless of how its module writes annotations: diff --git a/packages/zarr-metadata/tests/test_docs.py b/packages/zarr-metadata/tests/test_docs.py index 7726151126..a05596075d 100644 --- a/packages/zarr-metadata/tests/test_docs.py +++ b/packages/zarr-metadata/tests/test_docs.py @@ -8,12 +8,14 @@ from __future__ import annotations +import textwrap from collections import defaultdict from pathlib import Path import pytest from pytest_examples import CodeExample, EvalExample, find_examples +import zarr_metadata.definition from zarr_metadata.model import ZarrV3ArrayMetadata PACKAGE_ROOT = Path(__file__).parent.parent @@ -71,3 +73,31 @@ def test_every_python_block_says_whether_it_runs() -> None: ) ] assert unmarked == [] + + +def test_the_extensions_page_example_is_the_definition_module_docstring_example() -> None: + """The acme.lz4 example on the extensions page is, line for line, the example in the `zarr_metadata.definition` module docstring. Only the page's copy runs under these tests, so this is what keeps the docstring's copy from drifting.""" + page_blocks = [ + example + for example in _examples() + if example.path == PACKAGE_ROOT / "docs" / "extensions.md" + and example.prefix_settings().get("session") == "extensions" + ] + assert len(page_blocks) == 1, ( + "expected exactly one extensions-session block on docs/extensions.md" + ) + docstring = zarr_metadata.definition.__doc__ + assert docstring is not None, "zarr_metadata.definition has no module docstring" + lines = docstring.splitlines() + marker = "from collections.abc import Iterator" + starts = [i for i, line in enumerate(lines) if line.strip() == marker] + assert len(starts) == 1, f"expected one line {marker!r} in the docstring, found {len(starts)}" + assert lines[starts[0] - 1].strip() == "```python", ( + f"expected {marker!r} to open a ```python fence in the docstring" + ) + end = starts[0] + while end < len(lines) and lines[end].strip() != "```": + end += 1 + assert end < len(lines), "the docstring's ```python fence is not closed" + docstring_example = textwrap.dedent("\n".join(lines[starts[0] : end])).strip() + assert page_blocks[0].source.strip().splitlines() == docstring_example.splitlines() From 95857ac051643c96031c42b20b7ab44d954bbda4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 9 Nov 2026 12:19:56 +0100 Subject: [PATCH 3/9] feat(zarr-metadata)!: write store bytes through a JSONWriter `to_key_value` takes a writer instead of an `indent` keyword. A `JSONWriter` owns how a document becomes JSON bytes: it holds the settings (`indent`, `sort_keys`, `ensure_ascii`, `trailing_newline`) and its `encode` method does the writing, so the separators, the bare non-finite tokens and the surrogate escapes are no longer split off in a private helper. `to_key_value` also accepts a mapping of the settings, `JSONWriterSchema`, so callers need not import the class: `to_key_value({"indent": 2})`. The defaults are those of `json.dumps`, so a bare `to_key_value()` writes the same bytes as before. A setting `json.dumps` would misread raises, and so does a mapping key that is not a setting. Rebased onto main as one commit, replacing the JSONWriterConfig design. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/changes/429.removal.md | 1 + packages/zarr-metadata/docs/models.md | 28 +++ .../src/zarr_metadata/__init__.py | 6 + .../src/zarr_metadata/model/__init__.py | 11 +- .../src/zarr_metadata/model/_array_v2.py | 13 +- .../src/zarr_metadata/model/_array_v3.py | 9 +- .../src/zarr_metadata/model/_group_v2.py | 22 +-- .../src/zarr_metadata/model/_group_v3.py | 9 +- .../src/zarr_metadata/model/_json_writer.py | 143 ++++++++++++++ .../src/zarr_metadata/model/_shared.py | 11 -- .../tests/model/test_json_writer.py | 181 ++++++++++++++++++ .../tests/model/test_store_json.py | 4 +- .../zarr-metadata/tests/test_public_api.py | 8 + 13 files changed, 405 insertions(+), 41 deletions(-) create mode 100644 packages/zarr-metadata/changes/429.removal.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/model/_json_writer.py create mode 100644 packages/zarr-metadata/tests/model/test_json_writer.py diff --git a/packages/zarr-metadata/changes/429.removal.md b/packages/zarr-metadata/changes/429.removal.md new file mode 100644 index 0000000000..b1bee506fa --- /dev/null +++ b/packages/zarr-metadata/changes/429.removal.md @@ -0,0 +1 @@ +`to_key_value` takes a writer instead of an `indent` keyword. The writer is a `JSONWriter`, a frozen dataclass exported from `zarr_metadata` and `zarr_metadata.model` that owns how a document becomes JSON bytes: its settings are `indent`, `sort_keys`, `ensure_ascii` and `trailing_newline`, and its `encode` method does the writing. `to_key_value` also accepts a mapping of the settings (`JSONWriterSchema`), so `to_key_value(indent=2)` becomes `to_key_value({"indent": 2})`, with no import. The defaults are those of `json.dumps`, which the models already used, so a bare `to_key_value()` writes the same bytes as before. A setting that `json.dumps` would accept but misread, such as `indent=True`, a negative indent or a non-whitespace string indent, raises, and so does a mapping key that is not a setting. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index 713dd2fe21..dc41fd6b22 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -120,6 +120,34 @@ To change a model, use one of these methods: must claim what this context left unclaimed and contradict nothing. Otherwise it raises `ScopeConflictError`. +## Writing store bytes + +`to_key_value` writes a model's document as the JSON bytes a store holds. +A `JSONWriter` decides how those bytes are spelled: `indent`, `sort_keys`, +`ensure_ascii` and `trailing_newline`. Its defaults are those of +`json.dumps`. `to_key_value` takes a `JSONWriter` or a mapping of its +settings, so you do not need to import the class to change one: + +```python test="true" session="models" +from zarr_metadata.model import JSONWriter, ZarrV3GroupMetadata + +group = ZarrV3GroupMetadata.create_default(attributes={"b": 1, "a": 2}) +print(group.to_key_value({"indent": 2, "sort_keys": True})["zarr.json"].decode()) +# { +# "attributes": { +# "a": 2, +# "b": 1 +# }, +# "node_type": "group", +# "zarr_format": 3 +# } +assert group.to_key_value({"indent": 2}) == group.to_key_value(JSONWriter(indent=2)) +``` + +A mapping with a key that is not a setting raises `TypeError`, so a +misspelled setting is not ignored. `JSONWriter.from_dict` turns a mapping +into a writer, and `to_dict` turns a writer back into a mapping. + ## Extra fields An extra field is a top-level member that the spec does not define. The diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index e4a24e083e..3cde0f7230 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -19,6 +19,9 @@ from zarr_metadata.model import ( UNSET, + JSONWriter, + JSONWriterLike, + JSONWriterSchema, ZarrV2ArrayMetadata, ZarrV2ArrayMetadataUpdate, ZarrV2ConsolidatedMetadata, @@ -363,6 +366,9 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", + "JSONWriter", + "JSONWriterLike", + "JSONWriterSchema", "MetadataValidationError", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 2501347273..52ea6ac08d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -43,8 +43,11 @@ invalid model is never built. `to_json` returns the document as written. `update` reads new members in the model's own scope. `with_context` and `refined_in` read the document in another scope. `to_key_value` writes -the model as it is. A v3 group reads its `consolidated_metadata` as an -envelope and does not read the documents it lists, because consolidated +the model as it is, with a `JSONWriter`, or a mapping of its settings, deciding how +the bytes are spelled (indentation, key order, ASCII escaping, trailing +newline); the default is that of `json.dumps`. A v3 group reads its +`consolidated_metadata` as an envelope and does not read the documents +it lists, because consolidated metadata is a convention of the reference implementation, not core metadata. `ZarrV3ConsolidatedMetadata` and `read_consolidated_metadata_v3` read those documents. The model also @@ -98,6 +101,7 @@ read_node_metadata_v3, read_openable_node_metadata_v3, ) +from zarr_metadata.model._json_writer import JSONWriter, JSONWriterLike, JSONWriterSchema from zarr_metadata.model._repair import ( Repair, RepairKind, @@ -122,6 +126,9 @@ __all__ = [ "UNSET", + "JSONWriter", + "JSONWriterLike", + "JSONWriterSchema", "Repair", "RepairKind", "ZarrV2ArrayMetadata", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index ecaaaa621f..71ed7e79d8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -42,13 +42,13 @@ ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of from zarr_metadata.definition._scope import refines as refines_declaration +from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, attributes_of, check_literal, dimension_lengths, - dump_store_json, load_store_json, located_conflicts, member_changes, @@ -594,17 +594,16 @@ def to_json(self) -> ZarrV2ArrayMetadataJSON: return cast("ZarrV2ArrayMetadataJSON", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: - """The document as a store holds it: `.zarray` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" + """The document as a store holds it, spelled by `writer`, a `JSONWriter` or a mapping of its settings: `.zarray` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" + writes = JSONWriter.from_dict(writer) zarray = {key: value for key, value in self._document.items() if key != "attributes"} out: dict[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { - ZARR_V2_ARRAY_METADATA_STORE_KEY: dump_store_json(zarray, indent=indent) + ZARR_V2_ARRAY_METADATA_STORE_KEY: writes.encode(zarray) } if "attributes" in self._document: - out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( - self._document["attributes"], indent=indent - ) + out[ZARR_V2_ATTRIBUTES_STORE_KEY] = writes.encode(self._document["attributes"]) return out def __repr__(self) -> str: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index fa175ed005..021b6b8953 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -44,13 +44,13 @@ ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of from zarr_metadata.definition._scope import refines as refines_declaration +from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, attributes_of, check_literal, dimension_lengths, - dump_store_json, load_store_json, located_conflicts, member_changes, @@ -585,15 +585,16 @@ def to_json(self) -> ZarrV3ArrayMetadataJSON: return cast("ZarrV3ArrayMetadataJSON", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]: - """The document as a store holds it: JSON bytes at `zarr.json`, indented by `indent`. + """The document as a store holds it: JSON bytes at `zarr.json`, spelled by `writer`, a `JSONWriter` or a mapping of its settings. `NaN`, `Infinity` and `-Infinity` in `attributes` are written as bare tokens, as zarr-python writes them. A strict JSON parser rejects these tokens. """ - return {ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json(self._document, indent=indent)} + writes = JSONWriter.from_dict(writer) + return {ZARR_V3_ARRAY_METADATA_STORE_KEY: writes.encode(self._document)} def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index f5c7e8b405..9fba42baf9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -44,11 +44,11 @@ read_array_node_v2, read_array_v2, ) +from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, check_literal, - dump_store_json, load_store_json, located_conflicts, member_changes, @@ -342,17 +342,16 @@ def to_json(self) -> ZarrV2GroupMetadataJSON: return cast("ZarrV2GroupMetadataJSON", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: - """The document as a store holds it: `.zgroup` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" + """The document as a store holds it, spelled by `writer`, a `JSONWriter` or a mapping of its settings: `.zgroup` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" + writes = JSONWriter.from_dict(writer) zgroup = {key: value for key, value in self._document.items() if key != "attributes"} out: dict[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { - ZARR_V2_GROUP_METADATA_STORE_KEY: dump_store_json(zgroup, indent=indent) + ZARR_V2_GROUP_METADATA_STORE_KEY: writes.encode(zgroup) } if "attributes" in self._document: - out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( - self._document["attributes"], indent=indent - ) + out[ZARR_V2_ATTRIBUTES_STORE_KEY] = writes.encode(self._document["attributes"]) return out def __repr__(self) -> str: @@ -550,12 +549,11 @@ def to_json(self) -> dict[str, JSONValue]: return cast("dict[str, JSONValue]", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]: - """The document as a store holds it: JSON bytes at `.zmetadata`, indented by `indent`.""" - return { - ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json(self._document, indent=indent) - } + """The document as a store holds it: JSON bytes at `.zmetadata`, spelled by `writer`, a `JSONWriter` or a mapping of its settings.""" + writes = JSONWriter.from_dict(writer) + return {ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: writes.encode(self._document)} def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 82aad927ab..dee4bcbe6d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -62,12 +62,12 @@ read_extra_fields_v3, waived_members, ) +from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, attributes_of, check_literal, - dump_store_json, load_store_json, located_conflicts, member_changes, @@ -262,16 +262,17 @@ def to_json(self) -> ZarrV3GroupMetadataJSON: return cast("ZarrV3GroupMetadataJSON", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]: - """The document as a store holds it: JSON bytes at `zarr.json`, indented by `indent`. + """The document as a store holds it: JSON bytes at `zarr.json`, spelled by `writer`, a `JSONWriter` or a mapping of its settings. `NaN`, `Infinity` and `-Infinity` in `attributes`, or in the documents the consolidated metadata lists, are written as bare tokens, as zarr-python writes them. A strict JSON parser rejects these tokens. """ - return {ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json(self._document, indent=indent)} + writes = JSONWriter.from_dict(writer) + return {ZARR_V3_GROUP_METADATA_STORE_KEY: writes.encode(self._document)} def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_json_writer.py b/packages/zarr-metadata/src/zarr_metadata/model/_json_writer.py new file mode 100644 index 0000000000..59ca10a6e1 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_json_writer.py @@ -0,0 +1,143 @@ +"""`JSONWriter`, the one object that turns a document into the JSON bytes a store holds.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, fields +from typing import Final, cast, final + +from typing_extensions import TypedDict + + +class JSONWriterSchema(TypedDict, total=False, closed=True): + """The settings of a `JSONWriter` as a mapping, each by name and every one optional. + + A setting left out takes its default. `to_key_value` accepts this mapping + wherever it accepts a `JSONWriter`, so `model.to_key_value({"indent": 2})` + writes what `model.to_key_value(JSONWriter(indent=2))` writes. + """ + + indent: int | str | None + """`JSONWriter.indent`.""" + sort_keys: bool + """`JSONWriter.sort_keys`.""" + ensure_ascii: bool + """`JSONWriter.ensure_ascii`.""" + trailing_newline: bool + """`JSONWriter.trailing_newline`.""" + + +@final +@dataclass(frozen=True, slots=True, kw_only=True) +class JSONWriter: + """How a document is written as JSON bytes. `encode` does the writing. + + The defaults are those of `json.dumps`: everything on one line, keys + in document order, non-ASCII characters escaped, and no trailing + newline. zarr-python indents its metadata by its `json_indent` + setting, which is 2 unless configured otherwise, so it writes what + `JSONWriter(indent=2)` writes. + + A setting that would change the bytes silently is refused when the + writer is built: `json.dumps` reads `True` as an indent of 1 and a + negative indent as 0, and writes a `str` indent verbatim, so one that + is not whitespace would write bytes no JSON reader parses. + + The class is final: `encode` is the only way the models write JSON, so + a subclass cannot change how a model is written behind its settings. + """ + + indent: int | str | None = None + """`None` to write one line, an `int` to nest by that many spaces, or a `str` of JSON whitespace (spaces, tabs, line feeds and carriage returns) to nest by that string.""" + sort_keys: bool = False + """Write the keys of every object in sorted order instead of document order.""" + ensure_ascii: bool = True + """Escape every non-ASCII character (`"\\u00b5"`). Off, the bytes are UTF-8 text.""" + trailing_newline: bool = False + """End the bytes with `"\\n"`, for stores whose files are read by line-oriented tools.""" + + def __post_init__(self) -> None: + indent = cast("object", self.indent) + if isinstance(indent, bool) or not isinstance(indent, int | str | None): + msg = f"indent must be an int, a str or None, got {type(indent).__name__}" + raise TypeError(msg) + if isinstance(indent, int) and indent < 0: + msg = f"indent must not be negative, got {indent}" + raise ValueError(msg) + if isinstance(indent, str) and indent.strip(" \t\n\r") != "": + msg = f"indent must hold only JSON whitespace, got {indent!r}" + raise ValueError(msg) + for name in ("sort_keys", "ensure_ascii", "trailing_newline"): + flag = cast("object", getattr(self, name)) + if not isinstance(flag, bool): + msg = f"{name} must be a bool, got {type(flag).__name__}" + raise TypeError(msg) + + @classmethod + def from_dict(cls, data: JSONWriterLike) -> JSONWriter: + """The writer that `data` describes: `data` itself when it is a `JSONWriter`, else one built from its settings. + + A key that is not a setting raises `TypeError`, and a value the + setting does not allow raises as the constructor does. + """ + if isinstance(data, JSONWriter): + return data + given = cast("object", data) + if not isinstance(given, Mapping): + msg = f"expected a JSONWriter or a mapping of its settings, got {type(given).__name__}" + raise TypeError(msg) + settings = cast("Mapping[object, object]", given) + unknown = sorted(repr(key) for key in settings if key not in _SETTINGS) + if unknown: + msg = f"unknown JSONWriter setting(s) {', '.join(unknown)}; the settings are {', '.join(_SETTINGS)}" + raise TypeError(msg) + return cls(**cast("JSONWriterSchema", settings)) + + def to_dict(self) -> JSONWriterSchema: + """Every setting of this writer by name, so that `JSONWriter.from_dict(writer.to_dict()) == writer`.""" + return { + "indent": self.indent, + "sort_keys": self.sort_keys, + "ensure_ascii": self.ensure_ascii, + "trailing_newline": self.trailing_newline, + } + + def encode(self, value: object) -> bytes: + """`value`, a document its reader accepted, as UTF-8 JSON bytes spelled by this writer's settings. + + The separators are those of Python's `json` module: `", "` between + items and `": "` after a key, with the item separator becoming + `","` when `indent` is set. A non-finite number is written as a + bare token (`NaN`, `Infinity`, `-Infinity`), as zarr-python writes + attributes; the readers keep such numbers out of every other place + in a document. With `ensure_ascii` off, a lone surrogate (a string + a reader accepts from an escape such as `"\\ud800"`, but which + UTF-8 cannot encode) is still written as that escape. + """ + text = json.dumps( + value, + indent=self.indent, + sort_keys=self.sort_keys, + ensure_ascii=self.ensure_ascii, + allow_nan=True, + ) + if self.trailing_newline: + text += "\n" + # `json.dumps` writes non-ASCII characters only inside strings, and the + # only ones UTF-8 cannot encode are lone surrogates, so the replacement + # writes each as the JSON escape `ensure_ascii` would have written. + return text.encode("utf-8", "backslashreplace") + + +JSONWriterLike = JSONWriter | JSONWriterSchema +"""What `to_key_value` accepts as its writer: a `JSONWriter`, or a `JSONWriterSchema` mapping of the settings to change.""" + +_SETTINGS: Final = tuple(each.name for each in fields(JSONWriter)) +"""The name of each setting, in declaration order.""" + +DEFAULT_JSON_WRITER: Final = JSONWriter() +"""The writer `to_key_value` uses when given none: the spelling of `json.dumps` with its own defaults.""" + + +__all__ = ["DEFAULT_JSON_WRITER", "JSONWriter", "JSONWriterLike", "JSONWriterSchema"] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py index 1a5ab0156a..831a4b5be2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py @@ -310,17 +310,6 @@ def load_store_json(mapping: Mapping[StoreKey, bytes], key: str) -> object: ) from exc -def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: - """Encode a document that its reader has accepted as JSON bytes. - - A non-finite number is written as Python's `json` module writes it - (`NaN`, `Infinity`, `-Infinity`), which matches how zarr-python - writes attributes. The reader keeps such numbers out of every - other place in the document. - """ - return json.dumps(value, indent=indent, allow_nan=True).encode("utf-8") - - def located_conflicts( declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], conflicts: Sequence[Conflict] ) -> tuple[Conflict, ...]: diff --git a/packages/zarr-metadata/tests/model/test_json_writer.py b/packages/zarr-metadata/tests/model/test_json_writer.py new file mode 100644 index 0000000000..a2c39f1691 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_json_writer.py @@ -0,0 +1,181 @@ +"""`JSONWriter` decides how `to_key_value` spells the bytes it writes. + +The writer changes only the spelling of the JSON text, never the document: +whatever its settings, the bytes decode to the same document and +`from_key_value` reads them back to an equal model. The default writer +spells the document exactly as `json.dumps` does with its own defaults. +`to_key_value` takes a `JSONWriter` or a mapping of its settings. +""" + +from __future__ import annotations + +import json +from dataclasses import fields +from typing import TYPE_CHECKING, Any, cast + +import pytest + +from zarr_metadata.model import ( + JSONWriter, + JSONWriterLike, + JSONWriterSchema, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3GroupMetadata, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.typed_json import JSONValue + +# Keys out of sorted order, and a non-ASCII value, so key order and ASCII +# escaping are each visible in the written bytes. +ATTRIBUTES: dict[str, JSONValue] = {"unit": "µm", "axis": [1, 2]} + +MODELS = [ + ZarrV3ArrayMetadata.create_default(shape=(4,), attributes=ATTRIBUTES), + ZarrV3GroupMetadata.create_default(attributes=ATTRIBUTES), + ZarrV2ArrayMetadata.create_default(shape=(4,), attributes=ATTRIBUTES), + ZarrV2GroupMetadata.create_default(attributes=ATTRIBUTES), + ZarrV2ConsolidatedMetadata.from_json( + { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}, ".zattrs": ATTRIBUTES}, + } + ), +] +MODEL_IDS = ["v3-array", "v3-group", "v2-array", "v2-group", "v2-consolidated"] + +# Each writer, given as a `JSONWriter` or as a mapping of its settings. +WRITERS: list[JSONWriterLike] = [ + JSONWriter(), + JSONWriter(indent=2), + JSONWriter(indent=0), + JSONWriter(indent=""), + JSONWriter(indent="\t", trailing_newline=True), + JSONWriter(sort_keys=True), + JSONWriter(ensure_ascii=False), + JSONWriter(trailing_newline=True), + JSONWriter(indent=4, sort_keys=True, ensure_ascii=False, trailing_newline=True), + {}, + {"indent": 2}, + {"indent": 4, "sort_keys": True, "ensure_ascii": False, "trailing_newline": True}, +] + + +def _decoded(written: Mapping[str, bytes]) -> dict[str, object]: + return {key: json.loads(raw) for key, raw in written.items()} + + +@pytest.mark.parametrize("given", WRITERS, ids=repr) +@pytest.mark.parametrize("model", MODELS, ids=MODEL_IDS) +def test_writer_governs_spelling_and_round_trips(model: object, given: JSONWriterLike) -> None: + """`to_key_value(writer)` writes each key's bytes as `json.dumps` spells the same document under the same settings, whether the writer is a `JSONWriter` or a mapping of its settings. The written store holds the documents a default write holds, and reads back to an equal model.""" + held = cast("Any", model) + writer = JSONWriter.from_dict(given) + written: Mapping[str, bytes] = held.to_key_value(given) + assert written == held.to_key_value(writer) + for key, raw in written.items(): + text = raw.decode("utf-8") + if writer.trailing_newline: + assert text.endswith("\n"), key + text = text.removesuffix("\n") + expected = json.dumps( + json.loads(text), + indent=writer.indent, + sort_keys=writer.sort_keys, + ensure_ascii=writer.ensure_ascii, + ) + assert text == expected, key + assert _decoded(written) == _decoded(held.to_key_value()) + assert type(held).from_key_value(written) == model + + +@pytest.mark.parametrize("model", MODELS, ids=MODEL_IDS) +def test_default_writer_writes_what_json_dumps_writes(model: object) -> None: + """A bare `to_key_value()` writes each document with the defaults of Python's `json` module, byte for byte. `ensure_ascii` escapes the non-ASCII attribute.""" + written: Mapping[str, bytes] = cast("Any", model).to_key_value() + for key, raw in written.items(): + assert raw == json.dumps(json.loads(raw)).encode("utf-8"), key + assert any(b"\\u00b5m" in raw for raw in written.values()) + + +@pytest.mark.parametrize( + "writer", + [ + JSONWriter(), + JSONWriter(indent=" ", sort_keys=True, ensure_ascii=False, trailing_newline=True), + ], + ids=repr, +) +def test_from_dict_returns_a_writer_and_round_trips_its_settings(writer: JSONWriter) -> None: + """`from_dict` hands back a `JSONWriter` unchanged, and builds an equal one from the mapping `to_dict` returns.""" + assert JSONWriter.from_dict(writer) is writer + assert JSONWriter.from_dict(writer.to_dict()) == writer + + +def test_schema_names_every_setting_of_the_writer() -> None: + """`JSONWriterSchema` and `JSONWriter` declare the same settings, so every setting can be given as a mapping.""" + assert set(JSONWriterSchema.__annotations__) == {each.name for each in fields(JSONWriter)} + + +def test_ensure_ascii_off_escapes_only_a_lone_surrogate() -> None: + """With `ensure_ascii` off, a lone surrogate the reader accepted from an escape is written back as that escape, since UTF-8 cannot encode it, while every other non-ASCII character is written as UTF-8. The store reads back to an equal model.""" + stored = { + "zarr.json": json.dumps( + {"zarr_format": 3, "node_type": "group", "attributes": {"x": "\ud800", "y": "µ"}} + ).encode("utf-8") + } + model = ZarrV3GroupMetadata.from_key_value(stored) + written = model.to_key_value({"ensure_ascii": False})["zarr.json"] + assert b'"\\ud800"' in written + assert "µ".encode() in written + assert ZarrV3GroupMetadata.from_key_value({"zarr.json": written}) == model + + +def test_error_from_dict_refuses_an_unknown_setting() -> None: + """A key that is not a setting is refused, so a misspelled setting is not dropped.""" + with pytest.raises(TypeError, match="unknown JSONWriter setting\\(s\\) 'indnet'"): + JSONWriter.from_dict(cast("Any", {"indnet": 2})) + + +def test_error_from_dict_refuses_a_value_that_is_not_a_mapping() -> None: + """Only a `JSONWriter` or a mapping of its settings describes a writer.""" + with pytest.raises( + TypeError, match="expected a JSONWriter or a mapping of its settings, got int" + ): + JSONWriter.from_dict(cast("Any", 2)) + + +def test_error_writer_refuses_a_bool_indent() -> None: + """A `bool` is not an indent, though `json.dumps` would read `True` as an indent of 1.""" + with pytest.raises(TypeError, match="indent must be an int, a str or None, got bool"): + JSONWriter(indent=cast("Any", True)) + + +def test_error_writer_refuses_an_indent_of_another_type() -> None: + """An indent that is not an `int`, a `str` or `None` is refused.""" + with pytest.raises(TypeError, match="indent must be an int, a str or None, got float"): + JSONWriter(indent=cast("Any", 2.0)) + + +def test_error_writer_refuses_a_negative_indent() -> None: + """A negative indent is refused, though `json.dumps` would read it as 0.""" + with pytest.raises(ValueError, match="indent must not be negative, got -3"): + JSONWriter(indent=-3) + + +def test_error_writer_refuses_an_indent_that_is_not_json_whitespace() -> None: + """A `str` indent must hold only JSON whitespace: `json.dumps` writes it verbatim, so any other character writes bytes no JSON reader parses.""" + with pytest.raises(ValueError, match="indent must hold only JSON whitespace, got 'x'"): + JSONWriter(indent="x") + + +@pytest.mark.parametrize("name", ["sort_keys", "ensure_ascii", "trailing_newline"]) +def test_error_writer_refuses_a_flag_that_is_not_a_bool(name: str) -> None: + """Each flag takes only a `bool`: a truthy value such as `1` is refused.""" + with pytest.raises(TypeError, match=f"{name} must be a bool, got int"): + JSONWriter(**{name: cast("Any", 1)}) diff --git a/packages/zarr-metadata/tests/model/test_store_json.py b/packages/zarr-metadata/tests/model/test_store_json.py index 411e215613..5a8d135e41 100644 --- a/packages/zarr-metadata/tests/model/test_store_json.py +++ b/packages/zarr-metadata/tests/model/test_store_json.py @@ -33,6 +33,8 @@ if TYPE_CHECKING: from collections.abc import Iterable + from zarr_metadata.model import JSONWriterLike + class _Written(Protocol): """The output of a model write: store keys, each with its bytes.""" @@ -45,7 +47,7 @@ class _Stored(Protocol): def to_json(self) -> Mapping[str, object]: ... - def to_key_value(self, *, indent: int | str | None = None) -> _Written: ... + def to_key_value(self, writer: JSONWriterLike = ...) -> _Written: ... class _Reading(Protocol): diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 392c61a7a8..926b7753d4 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -60,6 +60,10 @@ def _group_rank(s: str) -> int: "MetadataValidationError", "ProblemKind", "UNSET", + # How `to_key_value` encodes a document as JSON bytes. + "JSONWriter", + "JSONWriterSchema", + "JSONWriterLike", # Store keys — the names the documents are persisted under. Defined in the # v2/v3 spec modules, re-exported through `zarr_metadata.model`. "ZARR_V2_ARRAY_METADATA_STORE_KEY", @@ -339,6 +343,10 @@ def test_all_is_grouped_and_unique() -> None: "HexFloat64", "JSONSchema", "JSONValue", + # How `to_key_value` encodes a document as JSON bytes. + "JSONWriter", + "JSONWriterSchema", + "JSONWriterLike", "UserJSONValue", # A pydantic field's scope, fixed: `Annotated[Model, InScope(CONTEXT)]`. "InScope", From b837a660e9e77d4336515c32b0159d3e9f9cdf18 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 9 Nov 2026 13:43:06 +0100 Subject: [PATCH 4/9] docs(zarr-metadata): number the changelog fragment for zarr-developers/zarr-python#4610 Assisted-by: ClaudeCode:claude-opus-5-5 --- .../zarr-metadata/changes/{429.removal.md => 4610.removal.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{429.removal.md => 4610.removal.md} (100%) diff --git a/packages/zarr-metadata/changes/429.removal.md b/packages/zarr-metadata/changes/4610.removal.md similarity index 100% rename from packages/zarr-metadata/changes/429.removal.md rename to packages/zarr-metadata/changes/4610.removal.md From 9d20d445c8f1e21f1a19ba2871dc5ea1954ef862 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:13:59 -0500 Subject: [PATCH 5/9] chore!: remove zarr.tree and the Array.compressor properties (#4521) * Remove zarr.tree and the Array.compressor properties Both were deprecated in 3.0.0 and were scheduled for removal in 3.2.0 by #3325. The rest of that PR shipped in 3.2.0 through #3900-#3903. Removed: - zarr.tree, zarr.api.synchronous.tree, zarr.api.asynchronous.tree. Use Group.tree / AsyncGroup.tree. - Array.compressor and AsyncArray.compressor. Use Array.compressors. The warning-only test for zarr.tree is gone, and the v2 compressor assertions in tests/test_v2.py now use `compressors`. The zarr.storage.default_compressor shim from the same #3325 list is left alone; #4505 removes it. --------- Co-authored-by: Davis Bennett --- changes/4521.removal.md | 4 ++++ src/zarr/__init__.py | 2 -- src/zarr/api/asynchronous.py | 28 ---------------------------- src/zarr/api/synchronous.py | 29 ----------------------------- src/zarr/core/array.py | 26 -------------------------- tests/test_api.py | 13 ------------- tests/test_v2.py | 6 +++--- 7 files changed, 7 insertions(+), 101 deletions(-) create mode 100644 changes/4521.removal.md diff --git a/changes/4521.removal.md b/changes/4521.removal.md new file mode 100644 index 0000000000..9d8ce85d24 --- /dev/null +++ b/changes/4521.removal.md @@ -0,0 +1,4 @@ +The following have been removed, having been deprecated since Zarr-Python 3.0.0: + +- `zarr.tree`, `zarr.api.synchronous.tree` and `zarr.api.asynchronous.tree`. Use `Group.tree` / `AsyncGroup.tree` instead. +- The `Array.compressor` and `AsyncArray.compressor` properties. Use `Array.compressors` instead (for Zarr format 2 arrays, `compressors[0]` is the old `compressor`, and an empty tuple means no compressor). diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index f305245f4e..2eb8cb5597 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -30,7 +30,6 @@ save, save_array, save_group, - tree, zeros, zeros_like, ) @@ -179,7 +178,6 @@ def set_format(log_format: str) -> None: "save", "save_array", "save_group", - "tree", "zeros", "zeros_like", ] diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index f162c13a7b..c0a53bac89 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -7,7 +7,6 @@ import numpy as np import numpy.typing as npt -from typing_extensions import deprecated from zarr.abc.store import Store from zarr.core.array import ( @@ -43,7 +42,6 @@ ArrayNotFoundError, GroupNotFoundError, NodeTypeValidationError, - ZarrDeprecationWarning, ZarrRuntimeWarning, ZarrUserWarning, ) @@ -91,7 +89,6 @@ "save", "save_array", "save_group", - "tree", "zeros", "zeros_like", ] @@ -632,31 +629,6 @@ async def save_group( await asyncio.gather(*aws) -@deprecated("Use AsyncGroup.tree instead.", category=ZarrDeprecationWarning) -async def tree(grp: AsyncGroup, expand: bool | None = None, level: int | None = None) -> Any: - """Provide a rich display of the hierarchy. - - !!! warning "Deprecated" - `zarr.tree()` is deprecated since v3.0.0 and will be removed in a future release. - Use `group.tree()` instead. - - Parameters - ---------- - grp : Group - Zarr or h5py group. - expand : bool, optional - Only relevant for HTML representation. If True, tree will be fully expanded. - level : int, optional - Maximum depth to descend into hierarchy. - - Returns - ------- - TreeRepr - A pretty-printable object displaying the hierarchy. - """ - return await grp.tree(expand=expand, level=level) - - async def array(data: npt.ArrayLike | AnyArray, **kwargs: Any) -> AnyAsyncArray: """Create an array filled with `data`. diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 86b72dedaf..76bada1910 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -2,15 +2,12 @@ from typing import TYPE_CHECKING, Any, Literal -from typing_extensions import deprecated - import zarr.api.asynchronous as async_api import zarr.core.array from zarr.core.array import DEFAULT_FILL_VALUE, Array, AsyncArray, CompressorLike from zarr.core.group import Group from zarr.core.sync import sync from zarr.core.sync_group import create_hierarchy -from zarr.errors import ZarrDeprecationWarning if TYPE_CHECKING: from collections.abc import Iterable @@ -69,7 +66,6 @@ "save", "save_array", "save_group", - "tree", "zeros", "zeros_like", ] @@ -389,31 +385,6 @@ def save_group( ) -@deprecated("Use Group.tree instead.", category=ZarrDeprecationWarning) -def tree(grp: Group, expand: bool | None = None, level: int | None = None) -> Any: - """Provide a rich display of the hierarchy. - - !!! warning "Deprecated" - `zarr.tree()` is deprecated since v3.0.0 and will be removed in a future release. - Use `group.tree()` instead. - - Parameters - ---------- - grp : Group - Zarr or h5py group. - expand : bool, optional - Only relevant for HTML representation. If True, tree will be fully expanded. - level : int, optional - Maximum depth to descend into hierarchy. - - Returns - ------- - TreeRepr - A pretty-printable object displaying the hierarchy. - """ - return sync(async_api.tree(grp._async_group, expand=expand, level=level)) - - # TODO: add type annotations for kwargs def array(data: npt.ArrayLike | AnyArray, **kwargs: Any) -> AnyArray: """Create an array filled with `data`. diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 8c473f3223..3735a4da18 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1011,20 +1011,6 @@ def serializer(self) -> ArrayBytesCodec | None: codec for codec in self.metadata.inner_codecs if isinstance(codec, ArrayBytesCodec) ) - @property - @deprecated("Use AsyncArray.compressors instead.", category=ZarrDeprecationWarning) - def compressor(self) -> Numcodec | None: - """ - Compressor that is applied to each chunk of the array. - - !!! warning "Deprecated" - `Array.compressor` is deprecated since v3.0.0 and will be removed in a future release. - Use [`Array.compressors`][zarr.AsyncArray.compressors] instead. - """ - if self.metadata.zarr_format == 2: - return self.metadata.compressor - raise TypeError("`compressor` is not available for Zarr format 3 arrays.") - @property def compressors(self) -> tuple[Numcodec, ...] | tuple[BytesBytesCodec, ...]: """ @@ -2309,18 +2295,6 @@ def serializer(self) -> ArrayBytesCodec | None: """ return self.async_array.serializer - @property - @deprecated("Use Array.compressors instead.", category=ZarrDeprecationWarning) - def compressor(self) -> Numcodec | None: - """ - Compressor that is applied to each chunk of the array. - - !!! warning "Deprecated" - `array.compressor` is deprecated since v3.0.0 and will be removed in a future release. - Use [`array.compressors`][zarr.Array.compressors] instead. - """ - return self.async_array.compressor - @property def compressors(self) -> tuple[Numcodec, ...] | tuple[BytesBytesCodec, ...]: """ diff --git a/tests/test_api.py b/tests/test_api.py index e162ddfeea..416da6d837 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -54,7 +54,6 @@ ArrayNotFoundError, ContainsArrayError, MetadataValidationError, - ZarrDeprecationWarning, ZarrUserWarning, ) from zarr.storage import MemoryStore @@ -1188,18 +1187,6 @@ def test_load_local(tmp_path: Path, path: str | None, load_read_only: bool) -> N assert np.array_equal(result, data) -def test_tree() -> None: - g1 = zarr.group() - g1.create_group("foo") - g3 = g1.create_group("bar") - g3.create_group("baz") - g5 = g3.create_group("qux") - g5.create_array("baz", shape=(100,), chunks=(10,), dtype="float64") - with pytest.warns(ZarrDeprecationWarning, match=r"Group\.tree instead\."): # noqa: PT031 - assert repr(zarr.tree(g1)) == repr(g1.tree()) - assert str(zarr.tree(g1)) == str(g1.tree()) - - # @pytest.mark.parametrize("stores_from_path", [False, True]) # @pytest.mark.parametrize( # "with_chunk_store,listable", diff --git a/tests/test_v2.py b/tests/test_v2.py index c6842c4902..fb617e92c4 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -147,13 +147,13 @@ def test_create_array_defaults(store: Store) -> None: g = zarr.open(store, mode="w", zarr_format=2) assert isinstance(g, Group) arr = g.create_array("one", dtype="i8", shape=(1,), chunks=(1,), compressor=None) - assert arr.async_array.compressor is None + assert arr.compressors == () assert not (arr.filters) arr = g.create_array("two", dtype="i8", shape=(1,), chunks=(1,)) - assert arr.async_array.compressor is not None + assert arr.compressors != () assert not (arr.filters) arr = g.create_array("three", dtype="i8", shape=(1,), chunks=(1,), compressor=Zstd()) - assert arr.async_array.compressor is not None + assert arr.compressors != () assert not (arr.filters) with pytest.raises(ValueError): g.create_array( From 5abce1f2c292fd83dfeae706bd0046b0aaef9aff Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Mon, 9 Nov 2026 15:15:40 +0100 Subject: [PATCH 6/9] test(codec_pipeline): pin the pool size in the fork regression test (#4504) * fix: reset the codec-pipeline thread pool after fork A forked child inherits codec_pipeline._pool pointing at threads that do not exist in the child, plus the parent's idle-semaphore permits. Submissions up to the permit count are consumed without spawning a thread, so small FusedCodecPipeline reads hang forever. Add an after_in_child hook that drops the inherited pool, size, and lock. The lock is recreated because it may have been held by a thread that no longer exists in the child. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs: add changelog entry for #4483 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(codec_pipeline): pin pool size in the fork regression test Move the #4478 regression test next to the other FusedCodecPipeline pool tests and pin `codec_pipeline.max_workers` to 4 through the shared `_FUSED_POOL_CONFIG`. With the default, a 1-CPU host resolves `max_workers` to 1, never creates the pool, and passes the test without exercising the bug. The test now asserts that the parent created the pool before forking, and it runs on macOS, where the fork start method is available. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: barlowa124 Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- tests/test_fused_pipeline.py | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_fused_pipeline.py b/tests/test_fused_pipeline.py index 06e79d3de7..ea44eda324 100644 --- a/tests/test_fused_pipeline.py +++ b/tests/test_fused_pipeline.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import multiprocessing as mp +import sys from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any from unittest.mock import patch @@ -37,6 +39,7 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable, Iterable + from pathlib import Path from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, NDBuffer @@ -543,6 +546,48 @@ def read_row_block(i: int, handle: zarr.Array[Any] = fresh) -> np.ndarray: np.testing.assert_array_equal(fut.result(), data[i * 4 : (i + 1) * 4, :]) +def _read_region(arr: zarr.Array[Any], selection: Any) -> np.ndarray: + return np.asarray(arr[selection]) + + +@pytest.mark.skipif(sys.platform == "win32", reason="fork is not available on Windows") +@pytest.mark.filterwarnings( + r"ignore:This process \(pid=\d+\) is multi-threaded, use of fork\(\):DeprecationWarning" +) +def test_forked_child_does_not_reuse_parent_pool(tmp_path: Path) -> None: + """A forked child must not reuse the parent's codec-pipeline thread pool. + + Regression test for https://github.com/zarr-developers/zarr-python/issues/4478. + Executor threads do not survive `fork()`, but the inherited executor keeps the + idle-semaphore permits its parent's workers released. Each submission in the + child consumes a permit instead of starting a thread, so a read of no more + chunks than there are permits queues work that nothing runs. + + `max_workers` is pinned rather than left to `os.cpu_count()`: on a 1-CPU host + the default resolves to 1, the parent never creates the pool, and the test + would pass without exercising the bug. + """ + import zarr.core.codec_pipeline as cp_mod + + data = np.arange(100).reshape(10, 10) + with zarr_config.set(_FUSED_POOL_CONFIG): + arr = zarr.create_array( + store=str(tmp_path / "a.zarr"), shape=data.shape, chunks=(5, 5), dtype=data.dtype + ) + arr[:] = data + # Each multi-chunk read releases one idle permit per completed task, so after + # two 4-chunk reads the pool holds more permits than the child read submits. + arr[:] + arr[:] + assert cp_mod._pool is not None, "the parent read did not create the pool" + with mp.get_context("fork").Pool(1) as pool: + # `[:5]` spans 2 chunks. Without the after-fork reset, both submissions + # are swallowed by the dead pool and `get` times out. + result = pool.starmap_async(_read_region, [(arr, np.s_[:5])]) + (read_back,) = result.get(timeout=30) + np.testing.assert_array_equal(read_back, data[:5]) + + def test_shared_transform_decode_alternating_specs() -> None: """A single ChunkTransform must decode chunks of DIFFERENT specs correctly when calls alternate, exercising eviction/refill of its single-entry From 90faa99d2ab9722599c0ab1fbd4d721bc5030df6 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Mon, 9 Nov 2026 09:29:53 -0500 Subject: [PATCH 7/9] feat: Bytes partial decode (#4458) * feat: read only the needed rows of uncompressed chunks BytesCodec now implements partial decoding. An uncompressed chunk is stored in C order, so each row along its first axis is a contiguous run of bytes; for a selection that does not touch every row, the rows from the first to the last one it touches are fetched with a single range request and the selection is applied to them. The pipeline already uses partial decoding when the array-to-bytes codec supports it and there are no array-to-array or bytes-to-bytes codecs, so this applies to uncompressed arrays only. Reading 100 rows of a single-chunk 20 MB array now fetches the bytes of those rows instead of the whole chunk. Co-Authored-By: Claude Opus 5.5 * Fix boolean masks and stores that ignore ranges in BytesCodec partial reads A boolean mask on the first axis, as from arr.oindex[mask], was trimmed to start at the first selected row but not to end at the last, so its length no longer matched the rows fetched and numpy raised an IndexError. The row window and the shifted selection are now computed together in _row_window, which trims the mask at both ends. This also gives mypy the narrowing it needs. FsspecStore over HTTP returns the whole object when a server ignores the Range header. The partial read then failed to reshape a chunk that the full read would have decoded, so a read that worked before this branch raised an error. A response longer than the requested range is now sliced to it. Co-Authored-By: Claude Opus 5.5 * Add sync partial decode to BytesCodec and check the length of ranged reads Add _decode_partial_sync so FusedCodecPipeline reads only the needed rows of an uncompressed chunk, sharing the decode step with the async method. Decide what a store sent for a range request by its exact length: the requested range, the whole chunk, or everything from the range start. Any other length raises instead of returning wrong data. Run the partial read tests on both codec pipelines and add a changelog entry. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Davis Bennett --- changes/4458.feature.md | 1 + src/zarr/codecs/bytes.py | 135 +++++++++++++++++++- tests/test_codecs/test_bytes.py | 212 ++++++++++++++++++++++++++++++++ 3 files changed, 345 insertions(+), 3 deletions(-) create mode 100644 changes/4458.feature.md diff --git a/changes/4458.feature.md b/changes/4458.feature.md new file mode 100644 index 0000000000..eb4497c9b4 --- /dev/null +++ b/changes/4458.feature.md @@ -0,0 +1 @@ +Reading part of an uncompressed chunk now fetches only the bytes it needs. When `BytesCodec` is the only codec, a selection is read with a single byte range request covering the rows along the first axis that it touches, instead of the whole chunk. Chunks with a compressor or any other codec are read whole, as before. diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 622161c458..621dca78e9 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -3,9 +3,12 @@ import sys import warnings from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, ClassVar, Final, Literal +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal -from zarr.abc.codec import ArrayBytesCodec +import numpy as np + +from zarr.abc.codec import ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin +from zarr.abc.store import RangeByteRequest from zarr.codecs._deprecated_enum import _coerce_enum_input, _DeprecatedStrEnumMeta from zarr.core.common import JSON, parse_named_configuration from zarr.core.dtype.common import HasEndianness @@ -14,8 +17,10 @@ if TYPE_CHECKING: from typing import Self + from zarr.abc.store import ByteGetter from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer + from zarr.core.indexing import Selector, SelectorTuple EndianLiteral = Literal["little", "big"] @@ -40,7 +45,7 @@ def _parse_endian(data: object) -> EndianLiteral: @dataclass(frozen=True) -class BytesCodec(ArrayBytesCodec): +class BytesCodec(ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin): """bytes codec""" is_fixed_size = True @@ -137,6 +142,73 @@ async def _decode_single( ) -> NDBuffer: return self._decode_sync(chunk_bytes, chunk_spec) + async def _decode_partial_single( + self, + byte_getter: ByteGetter, + selection: SelectorTuple, + chunk_spec: ArraySpec, + ) -> NDBuffer | None: + """Read only the part of an uncompressed chunk that a selection needs. + + The chunk is stored in C order, so each row along its first axis is a + contiguous run of bytes. The rows from the first to the last one the + selection touches are fetched with a single range request, and the + selection is applied to them. A selection that touches every row reads + the whole chunk, as before. + """ + window = _row_window(selection, chunk_spec.shape) + chunk_bytes = await byte_getter.get( + prototype=chunk_spec.prototype, byte_range=_row_byte_range(window, chunk_spec) + ) + return self._decode_rows(chunk_bytes, selection, window, chunk_spec) + + def _decode_partial_sync( + self, + byte_getter: Any, + selection: SelectorTuple, + chunk_spec: ArraySpec, + ) -> NDBuffer | None: + """Sync equivalent of `_decode_partial_single`.""" + window = _row_window(selection, chunk_spec.shape) + chunk_bytes = byte_getter.get_sync( + prototype=chunk_spec.prototype, byte_range=_row_byte_range(window, chunk_spec) + ) + return self._decode_rows(chunk_bytes, selection, window, chunk_spec) + + def _decode_rows( + self, + chunk_bytes: Buffer | None, + selection: SelectorTuple, + window: tuple[int, int, SelectorTuple] | None, + chunk_spec: ArraySpec, + ) -> NDBuffer | None: + """Decode the bytes fetched for a row window and apply the selection to them.""" + if chunk_bytes is None: + return None + if window is None: + return self._decode_sync(chunk_bytes, chunk_spec)[selection] + first, stop, rows_selection = window + row_bytes = _row_bytes(chunk_spec) + start, end = first * row_bytes, stop * row_bytes + chunk_size = chunk_spec.shape[0] * row_bytes + # A store may not honor the byte range it was given. The length of what + # it sent tells which bytes those are. + if len(chunk_bytes) == end - start: + pass + elif len(chunk_bytes) == chunk_size: + # The whole chunk, as an HTTP server that ignores the Range header sends. + chunk_bytes = chunk_bytes[start:end] + elif len(chunk_bytes) == chunk_size - start: + # Everything from the start of the range to the end of the chunk. + chunk_bytes = chunk_bytes[: end - start] + else: + raise ValueError( + f"Requested bytes {start} to {end} of a {chunk_size} byte chunk, " + f"but the store returned {len(chunk_bytes)} bytes." + ) + rows_spec = replace(chunk_spec, shape=(stop - first, *chunk_spec.shape[1:])) + return self._decode_sync(chunk_bytes, rows_spec)[rows_selection] + def _encode_sync( self, chunk_array: NDBuffer, @@ -165,3 +237,60 @@ async def _encode_single( def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: return input_byte_length + + +def _row_window( + selection: SelectorTuple, shape: tuple[int, ...] +) -> tuple[int, int, SelectorTuple] | None: + """The rows along axis 0 that a selection touches, and the selection relative to them. + + Returns the first row, one past the last row, and the selection shifted so + that it indexes an array holding only those rows. Returns None when the + selection touches every row or the rows cannot be determined, in which case + the whole chunk is read. + """ + if len(shape) == 0 or not isinstance(selection, tuple) or len(selection) == 0: + return None + first_axis = selection[0] + shifted: Selector + if isinstance(first_axis, slice): + rows = range(*first_axis.indices(shape[0])) + if len(rows) == 0 or rows.step < 0: + return None + first, stop = rows[0], rows[-1] + 1 + shifted = slice(0, stop - first, rows.step) + elif isinstance(first_axis, int | np.integer): + first = int(first_axis) % shape[0] + stop = first + 1 + shifted = 0 + elif isinstance(first_axis, np.ndarray): + if first_axis.dtype == bool: + indices = np.nonzero(first_axis)[0] + else: + indices = first_axis % shape[0] + if indices.size == 0: + return None + first, stop = int(indices.min()), int(indices.max()) + 1 + shifted = first_axis[first:stop] if first_axis.dtype == bool else indices - first + else: + return None + if (first, stop) == (0, shape[0]): + return None + return first, stop, (shifted, *selection[1:]) + + +def _row_bytes(chunk_spec: ArraySpec) -> int: + """The number of bytes in one row along axis 0 of a chunk.""" + row_items = int(np.prod(chunk_spec.shape[1:])) + return chunk_spec.dtype.to_native_dtype().itemsize * row_items + + +def _row_byte_range( + window: tuple[int, int, SelectorTuple] | None, chunk_spec: ArraySpec +) -> RangeByteRequest | None: + """The byte range holding a row window, or None to read the whole chunk.""" + if window is None: + return None + first, stop, _ = window + row_bytes = _row_bytes(chunk_spec) + return RangeByteRequest(first * row_bytes, stop * row_bytes) diff --git a/tests/test_codecs/test_bytes.py b/tests/test_codecs/test_bytes.py index ead778f526..8ac4df7ba1 100644 --- a/tests/test_codecs/test_bytes.py +++ b/tests/test_codecs/test_bytes.py @@ -13,11 +13,13 @@ import zarr from tests.conftest import Expect, ExpectFail from zarr.abc.codec import SupportsSyncCodec +from zarr.abc.store import OffsetByteRequest, RangeByteRequest from zarr.codecs.bytes import ( ENDIAN, BytesCodec, Endian, EndianLiteral, + _row_window, ) from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import NDBuffer, default_buffer_prototype @@ -29,6 +31,8 @@ from .test_codecs import _AsyncArrayProxy if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from zarr.abc.store import Store @@ -338,3 +342,211 @@ def test_bytes_codec_evolve_structured_single_byte_fields_clears_endian() -> Non spec = _make_array_spec(dtype) evolved = codec.evolve_from_array_spec(spec) assert evolved.endian is None + + +@pytest.fixture( + params=[ + "zarr.core.codec_pipeline.BatchedCodecPipeline", + "zarr.core.codec_pipeline.FusedCodecPipeline", + ] +) +def codec_pipeline(request: pytest.FixtureRequest) -> Iterator[None]: + """Run a test once with each codec pipeline.""" + with zarr.config.set({"codec_pipeline.path": request.param}): + yield + + +class _RangeLoggingStore(zarr.storage.MemoryStore): + """A memory store that records the byte ranges it serves for chunk keys.""" + + def __init__(self) -> None: + super().__init__() + self.reads: list[tuple[str, Any, int]] = [] + + def _log(self, key: str, byte_range: Any, buf: Any) -> None: + if buf is not None and not key.endswith("zarr.json"): + self.reads.append((key, byte_range, len(buf))) + + async def get(self, key: str, prototype: Any = None, byte_range: Any = None) -> Any: + buf = await super().get(key, prototype, byte_range) + self._log(key, byte_range, buf) + return buf + + def get_sync(self, key: str, *, prototype: Any = None, byte_range: Any = None) -> Any: + buf = super().get_sync(key, prototype=prototype, byte_range=byte_range) + self._log(key, byte_range, buf) + return buf + + +def _single_chunk_array( + data: np.ndarray[Any, Any], **kwargs: Any +) -> tuple[Any, _RangeLoggingStore]: + store = _RangeLoggingStore() + arr = zarr.create_array( + store, shape=data.shape, dtype=data.dtype, chunks=data.shape, fill_value=0, **kwargs + ) + arr[...] = data + store.reads.clear() + return arr, store + + +@pytest.mark.usefixtures("codec_pipeline") +@pytest.mark.parametrize( + ("selection", "rows_read"), + [ + (np.s_[500:600, 3], 100), + (np.s_[500:600], 100), + (np.s_[777, 5], 1), + (np.s_[-1], 1), + (np.s_[10:20:3, ::2], 10), # rows 10 to 19 are fetched + (np.s_[...], 10_000), + ], +) +def test_uncompressed_partial_read(selection: Any, rows_read: int) -> None: + """Reading part of an uncompressed chunk fetches only the rows it touches.""" + data = np.arange(100_000, dtype="int16").reshape(10_000, 10) + arr, store = _single_chunk_array(data, compressors=None) + np.testing.assert_array_equal(arr[selection], data[selection]) + assert sum(n for *_, n in store.reads) == rows_read * 10 * 2 + + +@pytest.mark.usefixtures("codec_pipeline") +@pytest.mark.parametrize("dtype", [">u2", " None: + """Partial reads return the same values as numpy for several dtypes, byte orders, and shapes.""" + shape = {1: (1000,), 2: (100, 7), 3: (20, 5, 3)}[ndim] + data = (np.arange(int(np.prod(shape))) % 200).astype(dtype).reshape(shape) + arr, _ = _single_chunk_array(data, compressors=None) + selections: list[Any] = [np.s_[3:9], np.s_[4], np.s_[-3:], np.s_[1:15:4]] + for selection in selections: + np.testing.assert_array_equal(arr[selection], data[selection]) + np.testing.assert_array_equal(arr.oindex[[1, 5, 2]], data[[1, 5, 2]]) + mask = np.zeros(shape[0], dtype=bool) + mask[[2, 5, 11]] = True + np.testing.assert_array_equal(arr.oindex[mask], data[mask]) + coords = tuple(np.array([0, 6, 3]) % n for n in shape) + np.testing.assert_array_equal(arr.vindex[coords], data[coords]) + + +@pytest.mark.usefixtures("codec_pipeline") +def test_uncompressed_partial_read_across_chunks() -> None: + """A selection spanning several chunks reads only the touched rows of each.""" + data = np.arange(40_000, dtype="int32").reshape(4000, 10) + store = _RangeLoggingStore() + arr = zarr.create_array( + store, shape=data.shape, dtype="int32", chunks=(1000, 10), compressors=None + ) + arr[...] = data + store.reads.clear() + np.testing.assert_array_equal(arr[990:1010, 2], data[990:1010, 2]) + assert sorted((key, n) for key, _, n in store.reads) == [("c/0/0", 400), ("c/1/0", 400)] + + +@pytest.mark.usefixtures("codec_pipeline") +def test_compressed_chunks_are_read_whole() -> None: + """With a compressor the chunk bytes cannot be split, so the whole chunk is fetched.""" + data = np.arange(100_000, dtype="int16").reshape(10_000, 10) + arr, store = _single_chunk_array(data) # default compressor + np.testing.assert_array_equal(arr[500:600, 3], data[500:600, 3]) + ((_, byte_range, _),) = store.reads + assert byte_range is None + + +@pytest.mark.usefixtures("codec_pipeline") +@pytest.mark.parametrize("selection", [np.s_[10:20], np.s_[...]]) +def test_uncompressed_partial_read_missing_chunk(selection: Any) -> None: + """An unwritten chunk reads as the fill value, for a partial and a whole-chunk read.""" + store = _RangeLoggingStore() + arr = zarr.create_array( + store, shape=(100, 4), dtype="int16", chunks=(100, 4), compressors=None, fill_value=7 + ) + expected = np.full((100, 4), 7, dtype="int16")[selection] + np.testing.assert_array_equal(arr[selection], expected) + + +class _RewritesRangeStore(zarr.storage.MemoryStore): + """A memory store that does not serve the byte range it is asked for. + + `rewrite` maps the requested range to the one that is served instead. + """ + + def __init__(self, rewrite: Callable[[RangeByteRequest], Any]) -> None: + super().__init__() + self.rewrite = rewrite + + def _served(self, byte_range: Any) -> Any: + if isinstance(byte_range, RangeByteRequest): + return self.rewrite(byte_range) + return byte_range + + async def get(self, key: str, prototype: Any = None, byte_range: Any = None) -> Any: + return await super().get(key, prototype, self._served(byte_range)) + + def get_sync(self, key: str, *, prototype: Any = None, byte_range: Any = None) -> Any: + return super().get_sync(key, prototype=prototype, byte_range=self._served(byte_range)) + + +def _ignore_range(byte_range: RangeByteRequest) -> None: + """Send the whole value, as an HTTP server that ignores the Range header does.""" + + +def _ignore_range_end(byte_range: RangeByteRequest) -> OffsetByteRequest: + """Send everything from the start of the range to the end of the value.""" + return OffsetByteRequest(byte_range.start) + + +def _one_byte_short(byte_range: RangeByteRequest) -> RangeByteRequest: + return RangeByteRequest(byte_range.start, byte_range.end - 1) + + +def _rewriting_array( + rewrite: Callable[[RangeByteRequest], Any], +) -> tuple[Any, np.ndarray[Any, Any]]: + data = np.arange(400, dtype=" None: + """A store that sends more than the requested range still reads correctly.""" + arr, data = _rewriting_array(rewrite) + np.testing.assert_array_equal(arr[selection], data[selection]) + + +@pytest.mark.usefixtures("codec_pipeline") +def test_uncompressed_partial_read_store_sends_unexpected_length() -> None: + """A response whose length matches no known byte range behavior is an error.""" + arr, _ = _rewriting_array(_one_byte_short) + with pytest.raises(ValueError, match="the store returned 15 bytes"): + arr[50:52, 0] + + +@pytest.mark.parametrize( + ("selection", "shape"), + [ + ((slice(5, 5), slice(None)), (10, 4)), # no rows + ((slice(8, 2, -1), slice(None)), (10, 4)), # negative step + ((np.array([], dtype=np.intp), slice(None)), (10, 4)), # no rows + ((slice(None), 2), (10, 4)), # every row + ((np.array([0, 9]), slice(None)), (10, 4)), # first and last row + ((..., 2), (10, 4)), # rows not determined + (slice(2, 4), (10,)), # not a tuple + ((), ()), # zero-dimensional chunk + ], +) +def test_row_window_reads_whole_chunk(selection: Any, shape: tuple[int, ...]) -> None: + """Selections with no narrower row window than the whole chunk give None.""" + assert _row_window(selection, shape) is None From 98aff7b103c7ccc71d536a6dd0a26bd42940c192 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:10:03 -0500 Subject: [PATCH 8/9] chore!: remove the ignored Zarr-Python 2 parameters (#4522) * Deprecate the ignored Zarr-Python 2 keyword arguments `synchronizer`, `chunk_store`, `cache_metadata`, `cache_attrs`, `object_codec`, `read_only` and `meta_array` are accepted by `create`, `group` and `open_group` (and by everything that wraps them, such as `zeros`) but have never done anything in Zarr-Python 3. Passing one used to emit a `ZarrRuntimeWarning` saying "not yet implemented", which read as a promise to implement it. Emit a `ZarrDeprecationWarning` instead, saying the argument is ignored and will be removed in a release after April 2027, and say the same in the docstrings and the v3 migration guide. Calls that do not pass these arguments are unaffected. First step of the plan for #3976. --------- Co-authored-by: Davis Bennett --- changes/4522.removal.md | 1 + docs/user-guide/v3_migration.md | 6 ++-- src/zarr/api/asynchronous.py | 62 ++++++++++++++------------------- src/zarr/api/synchronous.py | 44 ++++++++--------------- tests/test_api.py | 25 +++++++++++-- 5 files changed, 68 insertions(+), 70 deletions(-) create mode 100644 changes/4522.removal.md diff --git a/changes/4522.removal.md b/changes/4522.removal.md new file mode 100644 index 0000000000..94fa353f9a --- /dev/null +++ b/changes/4522.removal.md @@ -0,0 +1 @@ +The Zarr-Python 2 keyword arguments `synchronizer`, `chunk_store`, `cache_metadata`, `cache_attrs`, `object_codec`, `read_only` and `meta_array` accepted by `zarr.create`, `zarr.group` and `zarr.open_group` (and the functions that wrap them, such as `zarr.zeros`) have never had any effect in Zarr-Python 3. Passing one of them now emits a `ZarrDeprecationWarning` instead of a "not yet implemented" `ZarrRuntimeWarning`. They will be removed in a release after February 2027. Calls that do not pass these arguments are unaffected. diff --git a/docs/user-guide/v3_migration.md b/docs/user-guide/v3_migration.md index 0e65649650..f8c8a99f67 100644 --- a/docs/user-guide/v3_migration.md +++ b/docs/user-guide/v3_migration.md @@ -231,9 +231,9 @@ The following functions / methods have not been ported to Zarr-Python 3: - `zarr.copy_all` ([issue #2407](https://github.com/zarr-developers/zarr-python/issues/2407)) - `zarr.copy_store` ([issue #2407](https://github.com/zarr-developers/zarr-python/issues/2407)) -The following features (corresponding to function arguments to functions in - `zarr`) have not been ported to Zarr-Python 3. Using these features - will raise a warning or a `NotImplementedError`: +The following function arguments from Zarr-Python 2 are accepted but ignored by + Zarr-Python 3. Passing them emits a `ZarrDeprecationWarning`, and they will be + removed in a release after February 2027: - `cache_attrs` - `cache_metadata` diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index c0a53bac89..3993ce35cc 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -42,7 +42,7 @@ ArrayNotFoundError, GroupNotFoundError, NodeTypeValidationError, - ZarrRuntimeWarning, + ZarrDeprecationWarning, ZarrUserWarning, ) from zarr.storage import StorePath @@ -106,9 +106,11 @@ def _infer_overwrite(mode: AccessModeLiteral) -> bool: return mode in _OVERWRITE_MODES -def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: +def _warn_ignored_kwargs(kwargs: dict[str, Any]) -> None: """ - Emit a "not yet implemented" warning for each provided keyword argument that is not None. + Emit a deprecation warning for each provided keyword argument that is not None. + + These are Zarr-Python 2 keyword arguments that have no effect in Zarr-Python 3. `kwargs` maps a keyword argument name to its supplied value. The `stacklevel` is chosen so the warning points at the caller of the public API function (the same location as an @@ -116,7 +118,11 @@ def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: """ for name, value in kwargs.items(): if value is not None: - warnings.warn(f"{name} is not yet implemented", ZarrRuntimeWarning, stacklevel=3) + warnings.warn( + f"{name} is ignored, and will be removed in a release after February 2027", + ZarrDeprecationWarning, + stacklevel=3, + ) def _get_shape_chunks(a: ArrayLike | Any) -> tuple[tuple[int, ...] | None, tuple[int, ...] | None]: @@ -706,18 +712,15 @@ async def group( If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : StoreLike or None, default=None - Separate storage for chunks. Not implemented. + Ignored. Deprecated and will be removed in a release after February 2027. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. path : str, optional Group path within store. meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. + Ignored. Deprecated and will be removed in a release after February 2027. zarr_format : {2, 3, None}, optional The zarr format to use when saving. storage_options : dict @@ -830,23 +833,18 @@ async def open_group( (fail if exists). On a store that cannot delete keys, 'w' raises an error instead of replacing an existing node. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. path : str, optional Group path within store. chunk_store : StoreLike or None, default=None - Separate storage for chunks. See the - [storage documentation in the user guide][user-guide-store-like] - for a description of all valid StoreLike values. + Ignored. Deprecated and will be removed in a release after February 2027. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. + Ignored. Deprecated and will be removed in a release after February 2027. attributes : dict A dictionary of JSON-serializable values with user-defined attributes. use_consolidated : bool or str, default None @@ -872,7 +870,7 @@ async def open_group( The new group. """ - _warn_unimplemented_kwargs( + _warn_ignored_kwargs( { "cache_attrs": cache_attrs, "synchronizer": synchronizer, @@ -977,15 +975,14 @@ async def create( [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. overwrite : bool, optional If True, delete all pre-existing data in `store` at `path` before creating the array. path : str, optional Path under which array is stored. chunk_store : StoreLike or None, default=None - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. + Ignored. Deprecated and will be removed in a release after February 2027. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. @@ -1006,18 +1003,13 @@ async def create( To create an array with no filters, provide an empty iterable or the value `None`. cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). + Ignored. Deprecated and will be removed in a release after February 2027. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. read_only : bool, optional - True if array should be protected against modification. + Ignored. Deprecated and will be removed in a release after February 2027. object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. + Ignored. Deprecated and will be removed in a release after February 2027. dimension_separator : {'.', '/'}, optional Separator placed between the dimensions of a chunk. Zarr format 2 only. Zarr format 3 arrays should use `chunk_key_encoding` instead. @@ -1034,7 +1026,7 @@ async def create( which instructs Zarr to choose the default Zarr format value defined in the runtime configuration. meta_array : array-like, optional - Not implemented. + Ignored. Deprecated and will be removed in a release after February 2027. attributes : dict[str, JSON], optional A dictionary of user attributes to store with the array. chunk_shape : ChunksLike, optional @@ -1100,7 +1092,7 @@ async def create( if zarr_format is None: zarr_format = _default_zarr_format() - _warn_unimplemented_kwargs( + _warn_ignored_kwargs( { "synchronizer": synchronizer, "chunk_store": chunk_store, diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 76bada1910..461c5b800b 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -430,18 +430,15 @@ def group( If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : StoreLike or None, default=None - Separate storage for chunks. Not implemented. + Ignored. Deprecated and will be removed in a release after February 2027. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. path : str, optional Group path within store. meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. + Ignored. Deprecated and will be removed in a release after February 2027. zarr_format : {2, 3, None}, optional The zarr format to use when saving. storage_options : dict @@ -500,23 +497,18 @@ def open_group( (fail if exists). On a store that cannot delete keys, 'w' raises an error instead of replacing an existing node. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. path : str, optional Group path within store. chunk_store : StoreLike or None, default=None - Separate storage for chunks. See the - [storage documentation in the user guide][user-guide-store-like] - for a description of all valid StoreLike values. + Ignored. Deprecated and will be removed in a release after February 2027. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. + Ignored. Deprecated and will be removed in a release after February 2027. attributes : dict A dictionary of JSON-serializable values with user-defined attributes. use_consolidated : bool or str, default None @@ -681,15 +673,14 @@ def create( [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. overwrite : bool, optional If True, delete all pre-existing data in `store` at `path` before creating the array. path : str, optional Path under which array is stored. chunk_store : StoreLike or None, default=None - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. + Ignored. Deprecated and will be removed in a release after February 2027. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. @@ -710,18 +701,13 @@ def create( To create an array with no filters, provide an empty iterable or the value `None`. cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). + Ignored. Deprecated and will be removed in a release after February 2027. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. read_only : bool, optional - True if array should be protected against modification. + Ignored. Deprecated and will be removed in a release after February 2027. object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. + Ignored. Deprecated and will be removed in a release after February 2027. dimension_separator : {'.', '/'}, optional Separator placed between the dimensions of a chunk. Zarr format 2 only. Zarr format 3 arrays should use `chunk_key_encoding` instead. @@ -738,7 +724,7 @@ def create( which instructs Zarr to choose the default Zarr format value defined in the runtime configuration. meta_array : array-like, optional - Not implemented. + Ignored. Deprecated and will be removed in a release after February 2027. attributes : dict[str, JSON], optional A dictionary of user attributes to store with the array. chunk_shape : ChunksLike, optional diff --git a/tests/test_api.py b/tests/test_api.py index 416da6d837..53d8353b7f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -54,6 +54,7 @@ ArrayNotFoundError, ContainsArrayError, MetadataValidationError, + ZarrDeprecationWarning, ZarrUserWarning, ) from zarr.storage import MemoryStore @@ -2120,8 +2121,26 @@ def test_auto_chunks(f: Callable[..., AnyArray]) -> None: assert a.chunks == (500, 500) -@pytest.mark.parametrize("kwarg_name", ["synchronizer", "chunk_store", "cache_attrs", "meta_array"]) -def test_unimplemented_kwarg_warnings(kwarg_name: str) -> None: +@pytest.mark.parametrize( + "kwarg_name", + [ + "synchronizer", + "chunk_store", + "cache_metadata", + "cache_attrs", + "object_codec", + "read_only", + "meta_array", + ], +) +def test_ignored_kwarg_warnings(kwarg_name: str) -> None: kwargs = {kwarg_name: 1} - with pytest.warns(RuntimeWarning, match=".* is not yet implemented"): + with pytest.warns(ZarrDeprecationWarning, match=f"{kwarg_name} is ignored"): zarr.create(shape=(1,), **kwargs) # type: ignore[arg-type] + + +@pytest.mark.parametrize("kwarg_name", ["synchronizer", "chunk_store", "cache_attrs", "meta_array"]) +def test_ignored_kwarg_warnings_open_group(kwarg_name: str) -> None: + kwargs = {kwarg_name: 1} + with pytest.warns(ZarrDeprecationWarning, match=f"{kwarg_name} is ignored"): + zarr.open_group(store=MemoryStore(), **kwargs) # type: ignore[arg-type] From 27f034ed8a3ba7eb70ea67a097ab53d46835eec5 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:36:29 -0500 Subject: [PATCH 9/9] fix: point two deprecation warnings at the calling code (#4612) Python shows a DeprecationWarning by default only when it is attributed to __main__, so warnings that pointed at zarr internals were hidden from users running scripts: - `ArrayV2Metadata.chunk_grid` is a cached_property, which adds a frame; stacklevel 2 pointed at functools. Now 3. - Setting `zarr.storage.default_compressor` used stacklevel 1, which pointed at zarr itself. Now 2. --- changes/4612.bugfix.md | 1 + docs/contributing.md | 2 +- src/zarr/core/metadata/v2.py | 2 +- src/zarr/storage/__init__.py | 2 +- tests/test_v2.py | 12 +++++++++++- 5 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 changes/4612.bugfix.md diff --git a/changes/4612.bugfix.md b/changes/4612.bugfix.md new file mode 100644 index 0000000000..722b289454 --- /dev/null +++ b/changes/4612.bugfix.md @@ -0,0 +1 @@ +The `ArrayV2Metadata.chunk_grid` and `zarr.storage.default_compressor` deprecation warnings now point at the calling code instead of zarr internals, so Python no longer hides them by default. diff --git a/docs/contributing.md b/docs/contributing.md index 6054e86e97..38a722acde 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -479,7 +479,7 @@ If, after weighing these factors, a removal is not clearly worthwhile, prefer to Once a deprecation is decided, it must follow these steps so that users get a consistent, actionable signal: -1. **Emit a warning at runtime.** Raise `zarr.errors.ZarrDeprecationWarning` for an API that will be *removed*. Use `zarr.errors.ZarrFutureWarning` instead when behavior will *change* rather than disappear (for example, a default value that will change). The distinction is about who needs to see the warning. Python hides `DeprecationWarning` by default except in `__main__` and under test runners, so a removal warning reaches the authors of the code that calls the deprecated API, whereas `FutureWarning` is shown to everyone, which a behavior change needs because it alters results without any change to the user's code. The [`typing_extensions.deprecated`](https://typing-extensions.readthedocs.io/en/latest/#deprecated) decorator is the preferred way to deprecate whole functions, methods, and classes. When calling `warnings.warn` directly, set `stacklevel` so the warning points at the caller's code rather than zarr internals. +1. **Emit a warning at runtime.** Raise `zarr.errors.ZarrDeprecationWarning` for an API that will be *removed*. Use `zarr.errors.ZarrFutureWarning` instead when behavior will *change* rather than disappear (for example, a default value that will change). The distinction is about who needs to see the warning. Python hides `DeprecationWarning` by default except in `__main__` and under test runners, so a removal warning reaches the authors of the code that calls the deprecated API, whereas `FutureWarning` is shown to everyone, which a behavior change needs because it alters results without any change to the user's code. The [`typing_extensions.deprecated`](https://typing-extensions.readthedocs.io/en/latest/#deprecated) decorator is the preferred way to deprecate whole functions, methods, and classes. When calling `warnings.warn` directly, set `stacklevel` so the warning points at the caller's code rather than zarr internals. This matters because Python shows a `DeprecationWarning` by default only when it points at `__main__`; one that points at zarr internals is seen only under test runners. A warning raised inside a coroutine cannot point at the caller when the synchronous API runs it, because `sync()` runs coroutines on a separate IO thread, so if the deprecation must reach users running scripts, emit it from the public synchronous function instead. Test the location, for example with `assert caught[0].filename == __file__`. 2. **Write an actionable message.** The warning message must state the planned removal or change (a specific version if known, otherwise "a future release") and the migration path (i.e., what the user should do instead). A deprecation warning with no replacement guidance is incomplete. 3. **Document it.** Add a "Deprecated" admonition to the docstring, stating the same removal target and migration path as the warning, so the deprecation appears in the rendered API reference, and update any affected user-guide prose. 4. **Add a changelog entry.** Add a `removal` changelog fragment in `changes/` (for example, `changes/1234.removal.md`) so the deprecation is announced under "Deprecations and Removals" in the release notes. When the API is finally removed, add a second `removal` fragment for that release. diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 136267fc58..806401d2b4 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -134,7 +134,7 @@ def chunk_grid(self) -> ChunkGrid: "ArrayV2Metadata.chunk_grid is deprecated. " "Use ChunkGrid.from_metadata(metadata) instead.", DeprecationWarning, - stacklevel=2, + stacklevel=3, # skip the `cached_property.__get__` frame ) return ChunkGrid.from_sizes(self.shape, tuple(self.chunks)) diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index 074463af85..fc04a79057 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -42,7 +42,7 @@ def __setattr__(self, attr: str, value: Any) -> None: "zarr.config to configure array.v2_default_compressor " "e.g. config.set({'codecs.zstd':'numcodecs.Zstd', 'array.v2_default_compressor.numeric': 'zstd'})", ZarrDeprecationWarning, - stacklevel=1, + stacklevel=2, ) else: super().__setattr__(attr, value) diff --git a/tests/test_v2.py b/tests/test_v2.py index fb617e92c4..52068724fd 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -229,8 +229,18 @@ def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal[" def test_default_compressor_deprecation_warning() -> None: - with pytest.warns(ZarrDeprecationWarning, match="default_compressor is deprecated"): + with pytest.warns(ZarrDeprecationWarning, match="default_compressor is deprecated") as caught: zarr.storage.default_compressor = "zarr.codecs.zstd.ZstdCodec()" # type: ignore[attr-defined] + assert caught[0].filename == __file__ + + +def test_v2_metadata_chunk_grid_deprecation_warning() -> None: + metadata = zarr.create(shape=(2,), chunks=(1,), zarr_format=2).metadata + with pytest.warns( + DeprecationWarning, match="ArrayV2Metadata.chunk_grid is deprecated" + ) as caught: + metadata.chunk_grid # noqa: B018 + assert caught[0].filename == __file__ @pytest.mark.parametrize("fill_value", [None, (b"", 0, 0.0)], ids=["no_fill", "fill"])