diff --git a/changes/4458.feature.md b/changes/4458.feature.md new file mode 100644 index 0000000000..eb4497c9b4 --- /dev/null +++ b/changes/4458.feature.md @@ -0,0 +1 @@ +Reading part of an uncompressed chunk now fetches only the bytes it needs. When `BytesCodec` is the only codec, a selection is read with a single byte range request covering the rows along the first axis that it touches, instead of the whole chunk. Chunks with a compressor or any other codec are read whole, as before. diff --git a/changes/4521.removal.md b/changes/4521.removal.md new file mode 100644 index 0000000000..9d8ce85d24 --- /dev/null +++ b/changes/4521.removal.md @@ -0,0 +1,4 @@ +The following have been removed, having been deprecated since Zarr-Python 3.0.0: + +- `zarr.tree`, `zarr.api.synchronous.tree` and `zarr.api.asynchronous.tree`. Use `Group.tree` / `AsyncGroup.tree` instead. +- The `Array.compressor` and `AsyncArray.compressor` properties. Use `Array.compressors` instead (for Zarr format 2 arrays, `compressors[0]` is the old `compressor`, and an empty tuple means no compressor). diff --git a/changes/4522.removal.md b/changes/4522.removal.md new file mode 100644 index 0000000000..94fa353f9a --- /dev/null +++ b/changes/4522.removal.md @@ -0,0 +1 @@ +The Zarr-Python 2 keyword arguments `synchronizer`, `chunk_store`, `cache_metadata`, `cache_attrs`, `object_codec`, `read_only` and `meta_array` accepted by `zarr.create`, `zarr.group` and `zarr.open_group` (and the functions that wrap them, such as `zarr.zeros`) have never had any effect in Zarr-Python 3. Passing one of them now emits a `ZarrDeprecationWarning` instead of a "not yet implemented" `ZarrRuntimeWarning`. They will be removed in a release after February 2027. Calls that do not pass these arguments are unaffected. diff --git a/changes/4612.bugfix.md b/changes/4612.bugfix.md new file mode 100644 index 0000000000..722b289454 --- /dev/null +++ b/changes/4612.bugfix.md @@ -0,0 +1 @@ +The `ArrayV2Metadata.chunk_grid` and `zarr.storage.default_compressor` deprecation warnings now point at the calling code instead of zarr internals, so Python no longer hides them by default. diff --git a/docs/contributing.md b/docs/contributing.md index 6054e86e97..38a722acde 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -479,7 +479,7 @@ If, after weighing these factors, a removal is not clearly worthwhile, prefer to Once a deprecation is decided, it must follow these steps so that users get a consistent, actionable signal: -1. **Emit a warning at runtime.** Raise `zarr.errors.ZarrDeprecationWarning` for an API that will be *removed*. Use `zarr.errors.ZarrFutureWarning` instead when behavior will *change* rather than disappear (for example, a default value that will change). The distinction is about who needs to see the warning. Python hides `DeprecationWarning` by default except in `__main__` and under test runners, so a removal warning reaches the authors of the code that calls the deprecated API, whereas `FutureWarning` is shown to everyone, which a behavior change needs because it alters results without any change to the user's code. The [`typing_extensions.deprecated`](https://typing-extensions.readthedocs.io/en/latest/#deprecated) decorator is the preferred way to deprecate whole functions, methods, and classes. When calling `warnings.warn` directly, set `stacklevel` so the warning points at the caller's code rather than zarr internals. +1. **Emit a warning at runtime.** Raise `zarr.errors.ZarrDeprecationWarning` for an API that will be *removed*. Use `zarr.errors.ZarrFutureWarning` instead when behavior will *change* rather than disappear (for example, a default value that will change). The distinction is about who needs to see the warning. Python hides `DeprecationWarning` by default except in `__main__` and under test runners, so a removal warning reaches the authors of the code that calls the deprecated API, whereas `FutureWarning` is shown to everyone, which a behavior change needs because it alters results without any change to the user's code. The [`typing_extensions.deprecated`](https://typing-extensions.readthedocs.io/en/latest/#deprecated) decorator is the preferred way to deprecate whole functions, methods, and classes. When calling `warnings.warn` directly, set `stacklevel` so the warning points at the caller's code rather than zarr internals. This matters because Python shows a `DeprecationWarning` by default only when it points at `__main__`; one that points at zarr internals is seen only under test runners. A warning raised inside a coroutine cannot point at the caller when the synchronous API runs it, because `sync()` runs coroutines on a separate IO thread, so if the deprecation must reach users running scripts, emit it from the public synchronous function instead. Test the location, for example with `assert caught[0].filename == __file__`. 2. **Write an actionable message.** The warning message must state the planned removal or change (a specific version if known, otherwise "a future release") and the migration path (i.e., what the user should do instead). A deprecation warning with no replacement guidance is incomplete. 3. **Document it.** Add a "Deprecated" admonition to the docstring, stating the same removal target and migration path as the warning, so the deprecation appears in the rendered API reference, and update any affected user-guide prose. 4. **Add a changelog entry.** Add a `removal` changelog fragment in `changes/` (for example, `changes/1234.removal.md`) so the deprecation is announced under "Deprecations and Removals" in the release notes. When the API is finally removed, add a second `removal` fragment for that release. diff --git a/docs/user-guide/v3_migration.md b/docs/user-guide/v3_migration.md index 0e65649650..f8c8a99f67 100644 --- a/docs/user-guide/v3_migration.md +++ b/docs/user-guide/v3_migration.md @@ -231,9 +231,9 @@ The following functions / methods have not been ported to Zarr-Python 3: - `zarr.copy_all` ([issue #2407](https://github.com/zarr-developers/zarr-python/issues/2407)) - `zarr.copy_store` ([issue #2407](https://github.com/zarr-developers/zarr-python/issues/2407)) -The following features (corresponding to function arguments to functions in - `zarr`) have not been ported to Zarr-Python 3. Using these features - will raise a warning or a `NotImplementedError`: +The following function arguments from Zarr-Python 2 are accepted but ignored by + Zarr-Python 3. Passing them emits a `ZarrDeprecationWarning`, and they will be + removed in a release after February 2027: - `cache_attrs` - `cache_metadata` diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index a01ed26842..d4f3c6579a 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -2,410 +2,46 @@ Documentation: -Basic tools for modelling Zarr metadata, with minimal dependencies. +Types, models and validators for Zarr metadata documents, with minimal +dependencies. `zarr-metadata` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata) -and released independently of `zarr` itself. Install it with: +and released independently of `zarr`. Install it with: ``` pip install zarr-metadata ``` -## Who needs this - -This library supports software that interacts with Zarr metadata documents. - ## What this is -This is *not* a full Zarr implementation. It's a collection of data structures and routines that -closely model the content of the Zarr specifications, such as: - -- **Typed JSON shapes** ([`zarr_metadata.v2`](https://zarr-metadata.readthedocs.io/en/latest/api/v2/) and - [`zarr_metadata.v3`](https://zarr-metadata.readthedocs.io/en/latest/api/v3/)): `TypedDict` definitions and - `Literal` aliases for the JSON documents specified by the - [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) and - [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) - specifications, plus types for - [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/) and a - few widely-used-but-unspecified entities (e.g. consolidated metadata). -- **Document models** ([`zarr_metadata.model`](https://zarr-metadata.readthedocs.io/en/latest/api/model/)): immutable - models of whole metadata documents, each one a document and the scope - it was read in, with readers that report every problem at its - location, and store-key (de)serialization. A document - produced by `to_json` shares no mutable state with the model that produced - it. -- **Definitions and scopes** ([`zarr_metadata.definition`](https://zarr-metadata.readthedocs.io/en/latest/api/definition/), - with each format's kinds in `zarr_metadata.v3.definition` and - `zarr_metadata.v2.definition`): each extension's configuration as a - `TypedDict` with its rules, and the scope that decides which definitions - read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every - definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS_CONTEXT`. -- **JSON type checking** ([`zarr_metadata.typed_json`](https://zarr-metadata.readthedocs.io/en/latest/api/typed_json/)): - `check` checks a JSON value against any `TypedDict` and reports every - problem with its location. -- **Problems** ([`zarr_metadata.problems`](https://zarr-metadata.readthedocs.io/en/latest/api/problems/)): the problem - values every reader returns, and the error that carries them. -- **JSON Schema** ([`zarr_metadata.json_schema`](https://zarr-metadata.readthedocs.io/en/latest/api/json_schema/)): the - JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. -- **Optional Pydantic integration** ([`zarr_metadata.pydantic`](https://zarr-metadata.readthedocs.io/en/latest/api/pydantic/), - requires Pydantic 2.13 or newer): each model as a Pydantic field type that - validates raw documents through the same strict parser. - -## What this is for - -- **Static Type Checking** The public `TypedDict` definitions describe the static JSON shape of Zarr - metadata. - -- **Dynamic JSON validation** For strict, loc-aware validation of JSON loaded from disk, use the - model parser: - - ```python test="true" session="readme" - import json - from zarr_metadata.model import ZarrV3ArrayMetadata - - with open("zarr.json", "rb") as f: - raw = json.load(f) - - metadata = ZarrV3ArrayMetadata.from_json(raw) - ``` - - The optional Pydantic integration delegates raw input to the same strict - parser and returns the same normalized model class: - - ```python test="true" session="readme" - from pydantic import TypeAdapter - import zarr_metadata.pydantic as zmp - - adapter: TypeAdapter[zmp.ZarrV3ArrayMetadataField] = TypeAdapter(zmp.ZarrV3ArrayMetadataField) - metadata = adapter.validate_python(raw) - encoded = metadata.to_key_value()["zarr.json"] - ``` - - A field annotated `Annotated[ZarrV3ArrayMetadata, zmp.InScope(CONTEXT)]` - is read in `CONTEXT`, whatever the validation context holds, and a model - instance given to it is read again in `CONTEXT`. - - Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape - adapter, not a Zarr conformance validator; it may coerce values or discard - members that the strict model parser rejects. This is better suited for handling input provided - from Python objects (e.g., user-authored dicts) than JSON. - -## Validation boundary - -The readers and model constructors enforce the declared document structure and a small set -of context-free consistency rules, e.g.: -- fixed format literals, -- finite JSON numbers outside `attributes` and the members nothing in scope reads -- non-negative dimensions -- one `dimension_names` entry per array dimension. - -In a v3 document, validation also checks each extension point: the data -type, the chunk grid, the chunk key encoding, each codec, and each storage -transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. -`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no -specification registers. -If the definition rejects a configuration, the reader rejects it too, -and a configuration key the definition does not declare is reported as -`unknown_key`. If no definition in the scope claims a name, the reader -does not check that extension point, and the consumer decides whether to -support it. - -Some checks relate one part of a v3 document to another: - -- The fill value is checked against the data type the document names, by - that data type's definition. -- The chunk grid is checked against the shape, by the grid's definition. -- The codecs are checked as a pipeline. In order, each codec's definition - checks the codec against the chunk it receives. This includes a shard's - inner codecs and index codecs. - -The readers do no arithmetic on values. For example, they do not check -whether a fill value survives a `cast_value` round trip. - -Some choices that the spec text leaves open, or answers in two ways: - -- **`attributes` may hold `NaN`, `Infinity` and `-Infinity`.** The spec - does not interpret attributes, and zarr-python and xarray write these - numbers there (for example, a CF `_FillValue`). The models read them, - and `to_key_value` writes them back as bare tokens, which a strict JSON - parser rejects. So may a member the spec does not define when no - definition in scope claims it: nothing reads it, so it is kept as - written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite - number anywhere, including in attributes. -- **A reader walks at most 256 levels of nesting.** A value nested deeper - is an `invalid_value` problem at the first level past the limit, - wherever it is. Every reader, writer and comparison uses one stack - frame per level, and `copy.deepcopy` (and `pickle` before Python 3.12) - uses two. A document at the limit therefore uses about half of the - interpreter's default recursion limit, and the rest is left for the - caller. -- **`consolidated_metadata: null` is a problem.** A zarr-python 3.0.x bug - wrote it. The spec requires an object, and the package accepts only an - object. A reader of those stores removes the key before reading. -- **An extension name must follow the spec's naming rule**, - `^[a-z][a-z0-9-_.]+$`, or be a URI, which earlier versions of the spec - required. Any other name is rejected before any definition is - consulted, so `""` and `"foo/bar"` are problems, not unknown extensions. -- **`must_understand: false` is rejected at every extension point**, - including codecs and storage transformers, although the core spec names - only the data type, chunk grid and chunk key encoding. A reader that - skips a codec reads wrong bytes, just as a reader that skips a data type - reads wrong values. On an unknown top-level member, `must_understand` - keeps its meaning: a reader can skip that member. -- **A v2 datetime or timedelta dtype must have a unit**, as in ` Iterator[ValidationProblem]: + if "dictionary" in configuration and "dictionary_size" not in configuration: + yield ValidationProblem( + ("dictionary_size",), "a dictionary needs its size", "missing_key" + ) + + +ACME_LZ4 = CodecDefinition( + name="acme.lz4", + configuration=AcmeLz4Configuration, + kind="bytes_bytes", + size="dynamic", + rules=acme_lz4_rules, +) +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[ACME_LZ4])) +``` + +Pass `context=CONTEXT` to any reader or model constructor. A document that +names `acme.lz4` is then checked against this definition. + +## Rules for the pieces + +**The `TypedDict`** + +- It must be a `typing_extensions.TypedDict`. +- It must say how undeclared keys are handled: `closed=True`, + `extra_items=`, or `closed=False`. An open-by-default `TypedDict` is + refused, because it would accept a misspelled key silently. + +**Its members** must be JSON shapes: + +- `int`, `float`, `bool`, `str`, `None` or `JSONValue`, +- a `Literal`, +- `tuple[T, ...]` or `tuple[T1, T2]`, +- a union, +- a `TypedDict` or `Mapping[str, V]`, +- a `NewType` or a type alias. + +A number may carry bounds from annotated-types: `Gt`, `Ge`, `Lt`, `Le` or +`Interval`. Any other `Annotated` metadata is refused when the definition +is built. + +**The rules function** + +- It runs only on a configuration within its bounds, so it can rely on + them. +- It reports the location of each problem. The value there becomes the + problem's `input`. +- Show values in messages with `zarr_metadata.problems.shown`. Then they + read like the package's own messages. + +**Pickling.** Define each function at module top level. A model holds the +definitions that read it, so the model can be pickled only if they can. + +## Other kinds of definition + +Other kinds declare more: + +- A data type declares its fill value (`fill_value`, `fill_value_rules`, + `fill_value_canonical`). It also declares how its values are stored + (`storage`). +- A chunk grid declares which shapes it fits (`shape_rules`) and its chunk + lengths (`chunk_lengths`). +- A codec declares: + - what it checks about the chunk it receives (`chunk_rules`), + - what it hands on (`transition`), + - any pipelines it contains (`pipelines`). + +The [`zarr_metadata.v3.definition`](api/v3/definition.md) docstring +describes each of these. diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index b139dfd307..da545079e3 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -1,411 +1,92 @@ # zarr-metadata -Basic tools for modelling Zarr metadata, with minimal dependencies. +Types, models and validators for Zarr metadata documents, with minimal +dependencies. `zarr-metadata` is developed in the [zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata) -and released independently of `zarr` itself. Install it with: +and released independently of `zarr`. Install it with: ``` pip install zarr-metadata ``` -## Who needs this - -This library supports software that interacts with Zarr metadata documents. - ## What this is -This is *not* a full Zarr implementation. It's a collection of data structures and routines that -closely model the content of the Zarr specifications, such as: - -- **Typed JSON shapes** ([`zarr_metadata.v2`](api/v2.md) and - [`zarr_metadata.v3`](api/v3/index.md)): `TypedDict` definitions and - `Literal` aliases for the JSON documents specified by the - [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) and - [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) - specifications, plus types for - [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/) and a - few widely-used-but-unspecified entities (e.g. consolidated metadata). -- **Document models** ([`zarr_metadata.model`](api/model.md)): immutable - models of whole metadata documents, each one a document and the scope - it was read in, with readers that report every problem at its - location, and store-key (de)serialization. A document - produced by `to_json` shares no mutable state with the model that produced - it. -- **Definitions and scopes** ([`zarr_metadata.definition`](api/definition.md), - with each format's kinds in `zarr_metadata.v3.definition` and - `zarr_metadata.v2.definition`): each extension's configuration as a - `TypedDict` with its rules, and the scope that decides which definitions - read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every - definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS_CONTEXT`. -- **JSON type checking** ([`zarr_metadata.typed_json`](api/typed_json.md)): - `check` checks a JSON value against any `TypedDict` and reports every - problem with its location. -- **Problems** ([`zarr_metadata.problems`](api/problems.md)): the problem - values every reader returns, and the error that carries them. -- **JSON Schema** ([`zarr_metadata.json_schema`](api/json_schema.md)): the - JSON Schema of a `TypedDict`, a v3 declaration, or a whole `zarr.json`. -- **Optional Pydantic integration** ([`zarr_metadata.pydantic`](api/pydantic.md), - requires Pydantic 2.13 or newer): each model as a Pydantic field type that - validates raw documents through the same strict parser. - -## What this is for - -- **Static Type Checking** The public `TypedDict` definitions describe the static JSON shape of Zarr - metadata. - -- **Dynamic JSON validation** For strict, location-aware validation of JSON loaded from disk, use the - model parser: - - ```python test="true" session="index" - import json - from zarr_metadata.model import ZarrV3ArrayMetadata - - with open("zarr.json", "rb") as f: - raw = json.load(f) - - metadata = ZarrV3ArrayMetadata.from_json(raw) - ``` - - The optional Pydantic integration delegates raw input to the same strict - parser and returns the same normalized model class: - - ```python test="true" session="index" - from pydantic import TypeAdapter - import zarr_metadata.pydantic as zmp - - adapter: TypeAdapter[zmp.ZarrV3ArrayMetadataField] = TypeAdapter(zmp.ZarrV3ArrayMetadataField) - metadata = adapter.validate_python(raw) - encoded = metadata.to_key_value()["zarr.json"] - ``` - - A field annotated `Annotated[ZarrV3ArrayMetadata, zmp.InScope(CONTEXT)]` - is read in `CONTEXT`, whatever the validation context holds, and a model - instance given to it is read again in `CONTEXT`. - - Note that a bare `TypeAdapter` over a public `TypedDict` is a **coercive** shape - adapter, not a Zarr conformance validator; it may coerce values or discard - members that the strict model parser rejects. This is better suited for handling input provided - from Python objects (e.g., user-authored dicts) than JSON. - -## Validation boundary - -The readers and model constructors enforce the declared document structure and a small set -of context-free consistency rules, e.g.: -- fixed format literals, -- finite JSON numbers outside `attributes` and the members nothing in scope reads -- non-negative dimensions -- one `dimension_names` entry per array dimension. - -In a v3 document, validation also checks each extension point: the data -type, the chunk grid, the chunk key encoding, each codec, and each storage -transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. -`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no -specification registers. -If the definition rejects a configuration, the reader rejects it too, -and a configuration key the definition does not declare is reported as -`unknown_key`. If no definition in the scope claims a name, the reader -does not check that extension point, and the consumer decides whether to -support it. - -Making the context and explicit part of metadata parsing allows different libraries, which might support different -sets of Zarr features, to formally represent exactly the codecs, data types, etc they support when -parsing metadata. - -Some checks relate one part of a v3 document to another: - -- The fill value is checked against the data type the document names, by - that data type's definition. -- The chunk grid is checked against the shape, by the grid's definition. -- The codecs are checked as a pipeline. In order, each codec's definition - checks the codec against the chunk it receives. This includes a shard's - inner codecs and index codecs. - -The readers do no arithmetic on values. For example, they do not check -whether a fill value survives a `cast_value` round trip. - -Some choices that the spec text leaves open, or answers in two ways: +A collection of data structures and routines that closely model the +metadata rules in the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) +and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) +specifications and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/): -- **`attributes` may hold `NaN`, `Infinity` and `-Infinity`.** The spec - does not interpret attributes, and zarr-python and xarray write these - numbers there (for example, a CF `_FillValue`). The models read them, - and `to_key_value` writes them back as bare tokens, which a strict JSON - parser rejects. So may a member the spec does not define when no - definition in scope claims it: nothing reads it, so it is kept as - written. `check`, from `zarr_metadata.typed_json`, rejects a non-finite - number anywhere, including in attributes. -- **A reader walks at most 256 levels of nesting.** A value nested deeper - is an `invalid_value` problem at the first level past the limit, - wherever it is. Every reader, writer and comparison uses one stack - frame per level, and `copy.deepcopy` (and `pickle` before Python 3.12) - uses two. A document at the limit therefore uses about half of the - interpreter's default recursion limit, and the rest is left for the - caller. -- **`consolidated_metadata: null` is a problem.** A zarr-python 3.0.x bug - wrote it. The spec requires an object, and the package accepts only an - object. A reader of those stores removes the key before reading. -- **An extension name must follow the spec's naming rule**, - `^[a-z][a-z0-9-_.]+$`, or be a URI, which earlier versions of the spec - required. Any other name is rejected before any definition is - consulted, so `""` and `"foo/bar"` are problems, not unknown extensions. -- **`must_understand: false` is rejected at every extension point**, - including codecs and storage transformers, although the core spec names - only the data type, chunk grid and chunk key encoding. A reader that - skips a codec reads wrong bytes, just as a reader that skips a data type - reads wrong values. On an unknown top-level member, `must_understand` - keeps its meaning: a reader can skip that member. -- **A v2 datetime or timedelta dtype must have a unit**, as in ` zarr_metadata.definition': api/definition.md diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index e4a24e083e..3cde0f7230 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -19,6 +19,9 @@ from zarr_metadata.model import ( UNSET, + JSONWriter, + JSONWriterLike, + JSONWriterSchema, ZarrV2ArrayMetadata, ZarrV2ArrayMetadataUpdate, ZarrV2ConsolidatedMetadata, @@ -363,6 +366,9 @@ "Int64DataTypeName", "Int64FillValue", "JSONValue", + "JSONWriter", + "JSONWriterLike", + "JSONWriterSchema", "MetadataValidationError", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index 69829d32f4..d5de1537a2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -20,7 +20,14 @@ from zarr_metadata._common import JSONValue from zarr_metadata._sentinel import UNSET -ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json", "unknown_key"] +ProblemKind = Literal[ + "missing_key", + "invalid_type", + "invalid_value", + "invalid_json", + "unknown_key", + "unclaimed_declaration", +] """Machine-readable classification of a `ValidationProblem`. - `missing_key`: a required key (document key or store key) is absent. @@ -35,6 +42,10 @@ open since 2023), and many readers reject such a key. It has its own kind so that a caller who tolerates it can distinguish it from a wrong value, and so that it never hides other problems in the same object. +- `unclaimed_declaration`: a declaration that nothing in scope claims, which + a reader must understand to open the node. Only + `read_openable_node_metadata_v3` reports it: the other readers accept an + unclaimed declaration, since that is what keeps the format open. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 019725679a..2ba4055ba6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -116,40 +116,42 @@ a closure cannot be pickled, and a `functools.partial` can be pickled but does not compare equal to its loaded copy. - from collections.abc import Iterator - from typing import Annotated, NotRequired +```python +from collections.abc import Iterator +from typing import Annotated, NotRequired - from annotated_types import Ge - from typing_extensions import TypedDict +from annotated_types import Ge +from typing_extensions import TypedDict - from zarr_metadata import ValidationProblem - from zarr_metadata.definition import Nested - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context +from zarr_metadata import ValidationProblem +from zarr_metadata.definition import Nested +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition, ZarrV3Context - class AcmeLz4Configuration(TypedDict, closed=True): - acceleration: Annotated[int, Ge(1)] - dictionary: NotRequired[str] - dictionary_size: NotRequired[Annotated[int, Ge(1)]] +class AcmeLz4Configuration(TypedDict, closed=True): + acceleration: Annotated[int, Ge(1)] + dictionary: NotRequired[str] + dictionary_size: NotRequired[Annotated[int, Ge(1)]] - def acme_lz4_rules( - configuration: AcmeLz4Configuration, nested: Nested - ) -> Iterator[ValidationProblem]: - if "dictionary" in configuration and "dictionary_size" not in configuration: - yield ValidationProblem( - ("dictionary_size",), "a dictionary needs its size", "missing_key" - ) +def acme_lz4_rules( + configuration: AcmeLz4Configuration, nested: Nested +) -> Iterator[ValidationProblem]: + if "dictionary" in configuration and "dictionary_size" not in configuration: + yield ValidationProblem( + ("dictionary_size",), "a dictionary needs its size", "missing_key" + ) - ACME_LZ4 = CodecDefinition( - name="acme.lz4", - configuration=AcmeLz4Configuration, - kind="bytes_bytes", - size="dynamic", - rules=acme_lz4_rules, - ) - CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[ACME_LZ4])) +ACME_LZ4 = CodecDefinition( + name="acme.lz4", + configuration=AcmeLz4Configuration, + kind="bytes_bytes", + size="dynamic", + rules=acme_lz4_rules, +) +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[ACME_LZ4])) +``` The TypedDict declares how keys it does not declare are handled. With `closed=True`, such a key is a problem, as above. With `extra_items=`, diff --git a/packages/zarr-metadata/src/zarr_metadata/json_schema.py b/packages/zarr-metadata/src/zarr_metadata/json_schema.py index 2296b78652..7ff4bde29d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/json_schema.py @@ -15,13 +15,15 @@ accepts a number with no fractional part, such as `1.0`, as an integer, while `check` requires `1`: - from zarr_metadata.json_schema import json_schema - - json_schema(GzipCodecConfiguration) - # {'$schema': 'https://json-schema.org/draft/2020-12/schema', - # 'type': 'object', - # 'properties': {'level': {'type': 'integer', 'minimum': 0, 'maximum': 9}}, - # 'required': ['level'], 'additionalProperties': False} +```python +from zarr_metadata.json_schema import json_schema + +json_schema(GzipCodecConfiguration) +# {'$schema': 'https://json-schema.org/draft/2020-12/schema', +# 'type': 'object', +# 'properties': {'level': {'type': 'integer', 'minimum': 0, 'maximum': 9}}, +# 'required': ['level'], 'additionalProperties': False} +``` **One v3 declaration.** `declaration_json_schema_v3(kind, context=None)` writes one declaration read as `kind` in the scope `context`: a declaration that a definition in diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 0228861744..52ea6ac08d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -25,7 +25,12 @@ `read_repaired_node_metadata_v3` or `read_repaired_node_metadata_v2`. They undo each bug with `repair_node_metadata_v3` or `repair_node_metadata_v2` before the strict read, and report what they -changed. `zarr_metadata.json_schema` writes a JSON Schema of what the v3 +changed. A reader accepts a declaration that nothing in its scope claims +(an unknown codec, or an unknown top-level member), and a model lists each +one a reader must understand in `unclaimed_declarations`. To also report +each one as a problem, so that a reading has no problems exactly when a +reader can open the node, use `read_openable_node_metadata_v3`. +`zarr_metadata.json_schema` writes a JSON Schema of what the v3 readers read, except for the rules. One declaration on its own is read with `zarr_metadata.definition.resolve`. @@ -38,8 +43,11 @@ invalid model is never built. `to_json` returns the document as written. `update` reads new members in the model's own scope. `with_context` and `refined_in` read the document in another scope. `to_key_value` writes -the model as it is. A v3 group reads its `consolidated_metadata` as an -envelope and does not read the documents it lists, because consolidated +the model as it is, with a `JSONWriter`, or a mapping of its settings, deciding how +the bytes are spelled (indentation, key order, ASCII escaping, trailing +newline); the default is that of `json.dumps`. A v3 group reads its +`consolidated_metadata` as an envelope and does not read the documents +it lists, because consolidated metadata is a convention of the reference implementation, not core metadata. `ZarrV3ConsolidatedMetadata` and `read_consolidated_metadata_v3` read those documents. The model also @@ -91,7 +99,9 @@ read_consolidated_metadata_v3, read_group_metadata_v3, read_node_metadata_v3, + read_openable_node_metadata_v3, ) +from zarr_metadata.model._json_writer import JSONWriter, JSONWriterLike, JSONWriterSchema from zarr_metadata.model._repair import ( Repair, RepairKind, @@ -116,6 +126,9 @@ __all__ = [ "UNSET", + "JSONWriter", + "JSONWriterLike", + "JSONWriterSchema", "Repair", "RepairKind", "ZarrV2ArrayMetadata", @@ -160,6 +173,7 @@ "read_group_metadata_v3", "read_node_metadata_v2", "read_node_metadata_v3", + "read_openable_node_metadata_v3", "read_repaired_consolidated_metadata_v2", "read_repaired_node_metadata_v2", "read_repaired_node_metadata_v3", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index ecaaaa621f..71ed7e79d8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -42,13 +42,13 @@ ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of from zarr_metadata.definition._scope import refines as refines_declaration +from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, attributes_of, check_literal, dimension_lengths, - dump_store_json, load_store_json, located_conflicts, member_changes, @@ -594,17 +594,16 @@ def to_json(self) -> ZarrV2ArrayMetadataJSON: return cast("ZarrV2ArrayMetadataJSON", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: - """The document as a store holds it: `.zarray` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" + """The document as a store holds it, spelled by `writer`, a `JSONWriter` or a mapping of its settings: `.zarray` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" + writes = JSONWriter.from_dict(writer) zarray = {key: value for key, value in self._document.items() if key != "attributes"} out: dict[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { - ZARR_V2_ARRAY_METADATA_STORE_KEY: dump_store_json(zarray, indent=indent) + ZARR_V2_ARRAY_METADATA_STORE_KEY: writes.encode(zarray) } if "attributes" in self._document: - out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( - self._document["attributes"], indent=indent - ) + out[ZARR_V2_ATTRIBUTES_STORE_KEY] = writes.encode(self._document["attributes"]) return out def __repr__(self) -> str: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 061819fb3d..021b6b8953 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -44,13 +44,13 @@ ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of from zarr_metadata.definition._scope import refines as refines_declaration +from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, attributes_of, check_literal, dimension_lengths, - dump_store_json, load_store_json, located_conflicts, member_changes, @@ -443,14 +443,14 @@ def must_understand_subset( } -def waived_members(values: Mapping[str, ZarrV3ExtensionField]) -> list[str]: +def waived_members(values: Mapping[str, object]) -> list[str]: """The keys of the extra fields in `values` (by key, as written) that say `"must_understand": false`, which a reader may ignore.""" return [key for key, value in values.items() if _waived(value)] -def _waived(value: ZarrV3ExtensionField) -> bool: +def _waived(value: object) -> bool: """Whether `value`, an extra field as written, is an object that says `"must_understand": false`.""" - return isinstance(value, Mapping) and value.get("must_understand") is False + return is_object(value) and value.get("must_understand") is False class ZarrV3ArrayMetadataUpdate(TypedDict, total=False, closed=True): @@ -585,15 +585,16 @@ def to_json(self) -> ZarrV3ArrayMetadataJSON: return cast("ZarrV3ArrayMetadataJSON", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]: - """The document as a store holds it: JSON bytes at `zarr.json`, indented by `indent`. + """The document as a store holds it: JSON bytes at `zarr.json`, spelled by `writer`, a `JSONWriter` or a mapping of its settings. `NaN`, `Infinity` and `-Infinity` in `attributes` are written as bare tokens, as zarr-python writes them. A strict JSON parser rejects these tokens. """ - return {ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json(self._document, indent=indent)} + writes = JSONWriter.from_dict(writer) + return {ZARR_V3_ARRAY_METADATA_STORE_KEY: writes.encode(self._document)} def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" @@ -835,6 +836,10 @@ def from_json( Raises `MetadataValidationError` with every problem the read finds. `read_array_metadata_v3` returns the reading this model is built from, and the problems of a document that has any. + A declaration that nothing in `context` claims, such as an unknown + codec or an unknown top-level member, is not a problem; the model + lists each one a reader must understand in `unclaimed_declarations`. + `read_openable_node_metadata_v3` reports each one as a problem. """ return cls(data, context=context) @@ -919,7 +924,9 @@ def read_array_metadata_v3( type. Unknown top-level keys are allowed (they map to `extra_fields`). A reader must understand each one that does not declare `must_understand: false`; the model reports these as - `must_understand_fields`. + `must_understand_fields`, and every declaration a reader must + understand, at any depth, as `unclaimed_declarations`. + `read_openable_node_metadata_v3` reports each of those as a problem. To apply a policy over the declarations (for example, allow only the core spec's declarations), iterate over the reading's `declarations()`. A value that diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index f5c7e8b405..9fba42baf9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -44,11 +44,11 @@ read_array_node_v2, read_array_v2, ) +from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, check_literal, - dump_store_json, load_store_json, located_conflicts, member_changes, @@ -342,17 +342,16 @@ def to_json(self) -> ZarrV2GroupMetadataJSON: return cast("ZarrV2GroupMetadataJSON", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: - """The document as a store holds it: `.zgroup` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" + """The document as a store holds it, spelled by `writer`, a `JSONWriter` or a mapping of its settings: `.zgroup` without the attributes, and `.zattrs` with the attributes when they are set, even if empty.""" + writes = JSONWriter.from_dict(writer) zgroup = {key: value for key, value in self._document.items() if key != "attributes"} out: dict[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { - ZARR_V2_GROUP_METADATA_STORE_KEY: dump_store_json(zgroup, indent=indent) + ZARR_V2_GROUP_METADATA_STORE_KEY: writes.encode(zgroup) } if "attributes" in self._document: - out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( - self._document["attributes"], indent=indent - ) + out[ZARR_V2_ATTRIBUTES_STORE_KEY] = writes.encode(self._document["attributes"]) return out def __repr__(self) -> str: @@ -550,12 +549,11 @@ def to_json(self) -> dict[str, JSONValue]: return cast("dict[str, JSONValue]", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]: - """The document as a store holds it: JSON bytes at `.zmetadata`, indented by `indent`.""" - return { - ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json(self._document, indent=indent) - } + """The document as a store holds it: JSON bytes at `.zmetadata`, spelled by `writer`, a `JSONWriter` or a mapping of its settings.""" + writes = JSONWriter.from_dict(writer) + return {ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: writes.encode(self._document)} def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 42f3e61357..dee4bcbe6d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -62,12 +62,12 @@ read_extra_fields_v3, waived_members, ) +from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, attributes_of, check_literal, - dump_store_json, load_store_json, located_conflicts, member_changes, @@ -262,16 +262,17 @@ def to_json(self) -> ZarrV3GroupMetadataJSON: return cast("ZarrV3GroupMetadataJSON", copied(self._document)) def to_key_value( - self, *, indent: int | str | None = None + self, writer: JSONWriterLike = DEFAULT_JSON_WRITER ) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]: - """The document as a store holds it: JSON bytes at `zarr.json`, indented by `indent`. + """The document as a store holds it: JSON bytes at `zarr.json`, spelled by `writer`, a `JSONWriter` or a mapping of its settings. `NaN`, `Infinity` and `-Infinity` in `attributes`, or in the documents the consolidated metadata lists, are written as bare tokens, as zarr-python writes them. A strict JSON parser rejects these tokens. """ - return {ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json(self._document, indent=indent)} + writes = JSONWriter.from_dict(writer) + return {ZARR_V3_GROUP_METADATA_STORE_KEY: writes.encode(self._document)} def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" @@ -450,7 +451,10 @@ def from_json( scope that claims the member, as `CORE_CONTEXT` does, this is a problem, because the spec requires an object. Use `read_repaired_node_metadata_v3` to read such a store. A member the - spec does not define is kept in `extra_fields`. + spec does not define is kept in `extra_fields`, and is not a + problem when nothing in `context` claims it; the model lists each + such member a reader must understand in `unclaimed_declarations`. + `read_openable_node_metadata_v3` reports each one as a problem. """ return cls(data, context=context) @@ -769,6 +773,44 @@ def read_node_metadata_v3( return ZarrV3UnknownNodeMetadataReading(problems) +def read_openable_node_metadata_v3( + value: object, *, context: ZarrV3Context | None = None +) -> ZarrV3NodeMetadataReading: + """Read `value`, a v3 `zarr.json`, in `context` as `read_node_metadata_v3` does, and also report each declaration a reader must understand to open the node that nothing in `context` claims. + + The spec requires a reader to refuse a node with a declaration it does + not understand: an unknown codec, data type or chunk grid, or an + unknown top-level member that does not say `"must_understand": false`. + `read_node_metadata_v3` accepts these, as `UnclaimedDeclaration`s, + since that is what keeps the format open. This reader reports each one + as a problem of kind `unclaimed_declaration`, at its location, and + gives no model when there is one. A member that says + `"must_understand": false` is ignored whole, with every declaration in + it, as `unclaimed_declarations` ignores it. So the reading has no + problems exactly when a reader that understands `context` can open the + node. The documents a group's `consolidated_metadata` lists are not + read, as `read_group_metadata_v3` does not read them. + """ + reading = read_node_metadata_v3(value, context=context) + # A node reading is only of an array or a group when `value` is an object. + if isinstance(reading, ZarrV3UnknownNodeMetadataReading) or not is_object(value): + return reading + waived = waived_members({key: value[key] for key in reading.extra_fields}) + unclaimed = [ + ValidationProblem( + loc, + f"nothing in scope claims the {kind_name(declaration.read_as)} {declaration.name!r}, " + "which a reader must understand to open the node", + "unclaimed_declaration", + ) + for loc, declaration in unclaimed_declarations_of(reading.declarations(), waived).items() + ] + if len(unclaimed) == 0: + return reading + problems = (*reading.problems, *with_input(unclaimed, value)) + return dataclasses.replace(reading, problems=problems, metadata=None) + + ZarrV3NodeMetadata = TypeAliasType("ZarrV3NodeMetadata", ZarrV3ArrayMetadata | ZarrV3GroupMetadata) """The model of a v3 `zarr.json`: an array model or a group model, as its `node_type` declares.""" @@ -875,7 +917,8 @@ def read_group_metadata_v3( Unknown top-level keys are allowed (they map to `extra_fields`). A reader must understand each one that does not declare `must_understand: false`; the model reports these as - `must_understand_fields`. A `consolidated_metadata` member, if + `must_understand_fields`, and `read_openable_node_metadata_v3` reports + each one as a problem. A `consolidated_metadata` member, if present and claimed by the scope, as `CORE_CONTEXT` claims it, is checked against the envelope the core spec fixes. The documents it lists are not read; `read_consolidated_metadata_v3` reads them. This is the diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_json_writer.py b/packages/zarr-metadata/src/zarr_metadata/model/_json_writer.py new file mode 100644 index 0000000000..59ca10a6e1 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/model/_json_writer.py @@ -0,0 +1,143 @@ +"""`JSONWriter`, the one object that turns a document into the JSON bytes a store holds.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, fields +from typing import Final, cast, final + +from typing_extensions import TypedDict + + +class JSONWriterSchema(TypedDict, total=False, closed=True): + """The settings of a `JSONWriter` as a mapping, each by name and every one optional. + + A setting left out takes its default. `to_key_value` accepts this mapping + wherever it accepts a `JSONWriter`, so `model.to_key_value({"indent": 2})` + writes what `model.to_key_value(JSONWriter(indent=2))` writes. + """ + + indent: int | str | None + """`JSONWriter.indent`.""" + sort_keys: bool + """`JSONWriter.sort_keys`.""" + ensure_ascii: bool + """`JSONWriter.ensure_ascii`.""" + trailing_newline: bool + """`JSONWriter.trailing_newline`.""" + + +@final +@dataclass(frozen=True, slots=True, kw_only=True) +class JSONWriter: + """How a document is written as JSON bytes. `encode` does the writing. + + The defaults are those of `json.dumps`: everything on one line, keys + in document order, non-ASCII characters escaped, and no trailing + newline. zarr-python indents its metadata by its `json_indent` + setting, which is 2 unless configured otherwise, so it writes what + `JSONWriter(indent=2)` writes. + + A setting that would change the bytes silently is refused when the + writer is built: `json.dumps` reads `True` as an indent of 1 and a + negative indent as 0, and writes a `str` indent verbatim, so one that + is not whitespace would write bytes no JSON reader parses. + + The class is final: `encode` is the only way the models write JSON, so + a subclass cannot change how a model is written behind its settings. + """ + + indent: int | str | None = None + """`None` to write one line, an `int` to nest by that many spaces, or a `str` of JSON whitespace (spaces, tabs, line feeds and carriage returns) to nest by that string.""" + sort_keys: bool = False + """Write the keys of every object in sorted order instead of document order.""" + ensure_ascii: bool = True + """Escape every non-ASCII character (`"\\u00b5"`). Off, the bytes are UTF-8 text.""" + trailing_newline: bool = False + """End the bytes with `"\\n"`, for stores whose files are read by line-oriented tools.""" + + def __post_init__(self) -> None: + indent = cast("object", self.indent) + if isinstance(indent, bool) or not isinstance(indent, int | str | None): + msg = f"indent must be an int, a str or None, got {type(indent).__name__}" + raise TypeError(msg) + if isinstance(indent, int) and indent < 0: + msg = f"indent must not be negative, got {indent}" + raise ValueError(msg) + if isinstance(indent, str) and indent.strip(" \t\n\r") != "": + msg = f"indent must hold only JSON whitespace, got {indent!r}" + raise ValueError(msg) + for name in ("sort_keys", "ensure_ascii", "trailing_newline"): + flag = cast("object", getattr(self, name)) + if not isinstance(flag, bool): + msg = f"{name} must be a bool, got {type(flag).__name__}" + raise TypeError(msg) + + @classmethod + def from_dict(cls, data: JSONWriterLike) -> JSONWriter: + """The writer that `data` describes: `data` itself when it is a `JSONWriter`, else one built from its settings. + + A key that is not a setting raises `TypeError`, and a value the + setting does not allow raises as the constructor does. + """ + if isinstance(data, JSONWriter): + return data + given = cast("object", data) + if not isinstance(given, Mapping): + msg = f"expected a JSONWriter or a mapping of its settings, got {type(given).__name__}" + raise TypeError(msg) + settings = cast("Mapping[object, object]", given) + unknown = sorted(repr(key) for key in settings if key not in _SETTINGS) + if unknown: + msg = f"unknown JSONWriter setting(s) {', '.join(unknown)}; the settings are {', '.join(_SETTINGS)}" + raise TypeError(msg) + return cls(**cast("JSONWriterSchema", settings)) + + def to_dict(self) -> JSONWriterSchema: + """Every setting of this writer by name, so that `JSONWriter.from_dict(writer.to_dict()) == writer`.""" + return { + "indent": self.indent, + "sort_keys": self.sort_keys, + "ensure_ascii": self.ensure_ascii, + "trailing_newline": self.trailing_newline, + } + + def encode(self, value: object) -> bytes: + """`value`, a document its reader accepted, as UTF-8 JSON bytes spelled by this writer's settings. + + The separators are those of Python's `json` module: `", "` between + items and `": "` after a key, with the item separator becoming + `","` when `indent` is set. A non-finite number is written as a + bare token (`NaN`, `Infinity`, `-Infinity`), as zarr-python writes + attributes; the readers keep such numbers out of every other place + in a document. With `ensure_ascii` off, a lone surrogate (a string + a reader accepts from an escape such as `"\\ud800"`, but which + UTF-8 cannot encode) is still written as that escape. + """ + text = json.dumps( + value, + indent=self.indent, + sort_keys=self.sort_keys, + ensure_ascii=self.ensure_ascii, + allow_nan=True, + ) + if self.trailing_newline: + text += "\n" + # `json.dumps` writes non-ASCII characters only inside strings, and the + # only ones UTF-8 cannot encode are lone surrogates, so the replacement + # writes each as the JSON escape `ensure_ascii` would have written. + return text.encode("utf-8", "backslashreplace") + + +JSONWriterLike = JSONWriter | JSONWriterSchema +"""What `to_key_value` accepts as its writer: a `JSONWriter`, or a `JSONWriterSchema` mapping of the settings to change.""" + +_SETTINGS: Final = tuple(each.name for each in fields(JSONWriter)) +"""The name of each setting, in declaration order.""" + +DEFAULT_JSON_WRITER: Final = JSONWriter() +"""The writer `to_key_value` uses when given none: the spelling of `json.dumps` with its own defaults.""" + + +__all__ = ["DEFAULT_JSON_WRITER", "JSONWriter", "JSONWriterLike", "JSONWriterSchema"] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py index 1a5ab0156a..831a4b5be2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py @@ -310,17 +310,6 @@ def load_store_json(mapping: Mapping[StoreKey, bytes], key: str) -> object: ) from exc -def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: - """Encode a document that its reader has accepted as JSON bytes. - - A non-finite number is written as Python's `json` module writes it - (`NaN`, `Infinity`, `-Infinity`), which matches how zarr-python - writes attributes. The reader keeps such numbers out of every - other place in the document. - """ - return json.dumps(value, indent=indent, allow_nan=True).encode("utf-8") - - def located_conflicts( declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], conflicts: Sequence[Conflict] ) -> tuple[Conflict, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 4434660d8f..308519c0ef 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -29,9 +29,11 @@ For example: - TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(document, context=CONTEXT) - ArrayManifest.model_validate(data, context={"zarr_metadata_context": CONTEXT}) - Mixed.model_validate(data, context={"zarr_metadata_context": V3, "zarr_metadata_context_v2": V2}) +```python +TypeAdapter(zmp.ZarrV3ArrayMetadataField).validate_python(document, context=CONTEXT) +ArrayManifest.model_validate(data, context={"zarr_metadata_context": CONTEXT}) +Mixed.model_validate(data, context={"zarr_metadata_context": V3, "zarr_metadata_context_v2": V2}) +``` An existing model instance passes through unchanged when no scope is given. When one is, the instance is read again in it with `refined_in`, so @@ -63,11 +65,13 @@ Usage: - import zarr_metadata.pydantic as zmp +```python +import zarr_metadata.pydantic as zmp - class ArrayManifest(BaseModel): - path: str - metadata: zmp.ZarrV3ArrayMetadataField +class ArrayManifest(BaseModel): + path: str + metadata: zmp.ZarrV3ArrayMetadataField +``` Static type checkers see each field type as its core model class, so `manifest.metadata` is a `zarr_metadata.model.ZarrV3ArrayMetadata`. diff --git a/packages/zarr-metadata/src/zarr_metadata/typed_json.py b/packages/zarr-metadata/src/zarr_metadata/typed_json.py index 30ed95f695..982a3d4868 100644 --- a/packages/zarr-metadata/src/zarr_metadata/typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/typed_json.py @@ -6,14 +6,16 @@ value against the TypedDict and returns a value of the TypedDict (or None) and every problem, each located in the value: - import json +```python +import json - from zarr_metadata import ZarrV3ArrayMetadataJSON - from zarr_metadata.typed_json import check +from zarr_metadata import ZarrV3ArrayMetadataJSON +from zarr_metadata.typed_json import check - document, problems = check(json.loads(raw), ZarrV3ArrayMetadataJSON) - for problem in problems: - print(problem.loc, problem.kind, problem.message) +document, problems = check(json.loads(raw), ZarrV3ArrayMetadataJSON) +for problem in problems: + print(problem.loc, problem.kind, problem.message) +``` `check` reads a TypedDict as the typing spec defines it, regardless of how its module writes annotations: diff --git a/packages/zarr-metadata/tests/model/test_json_writer.py b/packages/zarr-metadata/tests/model/test_json_writer.py new file mode 100644 index 0000000000..a2c39f1691 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_json_writer.py @@ -0,0 +1,181 @@ +"""`JSONWriter` decides how `to_key_value` spells the bytes it writes. + +The writer changes only the spelling of the JSON text, never the document: +whatever its settings, the bytes decode to the same document and +`from_key_value` reads them back to an equal model. The default writer +spells the document exactly as `json.dumps` does with its own defaults. +`to_key_value` takes a `JSONWriter` or a mapping of its settings. +""" + +from __future__ import annotations + +import json +from dataclasses import fields +from typing import TYPE_CHECKING, Any, cast + +import pytest + +from zarr_metadata.model import ( + JSONWriter, + JSONWriterLike, + JSONWriterSchema, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3GroupMetadata, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.typed_json import JSONValue + +# Keys out of sorted order, and a non-ASCII value, so key order and ASCII +# escaping are each visible in the written bytes. +ATTRIBUTES: dict[str, JSONValue] = {"unit": "µm", "axis": [1, 2]} + +MODELS = [ + ZarrV3ArrayMetadata.create_default(shape=(4,), attributes=ATTRIBUTES), + ZarrV3GroupMetadata.create_default(attributes=ATTRIBUTES), + ZarrV2ArrayMetadata.create_default(shape=(4,), attributes=ATTRIBUTES), + ZarrV2GroupMetadata.create_default(attributes=ATTRIBUTES), + ZarrV2ConsolidatedMetadata.from_json( + { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}, ".zattrs": ATTRIBUTES}, + } + ), +] +MODEL_IDS = ["v3-array", "v3-group", "v2-array", "v2-group", "v2-consolidated"] + +# Each writer, given as a `JSONWriter` or as a mapping of its settings. +WRITERS: list[JSONWriterLike] = [ + JSONWriter(), + JSONWriter(indent=2), + JSONWriter(indent=0), + JSONWriter(indent=""), + JSONWriter(indent="\t", trailing_newline=True), + JSONWriter(sort_keys=True), + JSONWriter(ensure_ascii=False), + JSONWriter(trailing_newline=True), + JSONWriter(indent=4, sort_keys=True, ensure_ascii=False, trailing_newline=True), + {}, + {"indent": 2}, + {"indent": 4, "sort_keys": True, "ensure_ascii": False, "trailing_newline": True}, +] + + +def _decoded(written: Mapping[str, bytes]) -> dict[str, object]: + return {key: json.loads(raw) for key, raw in written.items()} + + +@pytest.mark.parametrize("given", WRITERS, ids=repr) +@pytest.mark.parametrize("model", MODELS, ids=MODEL_IDS) +def test_writer_governs_spelling_and_round_trips(model: object, given: JSONWriterLike) -> None: + """`to_key_value(writer)` writes each key's bytes as `json.dumps` spells the same document under the same settings, whether the writer is a `JSONWriter` or a mapping of its settings. The written store holds the documents a default write holds, and reads back to an equal model.""" + held = cast("Any", model) + writer = JSONWriter.from_dict(given) + written: Mapping[str, bytes] = held.to_key_value(given) + assert written == held.to_key_value(writer) + for key, raw in written.items(): + text = raw.decode("utf-8") + if writer.trailing_newline: + assert text.endswith("\n"), key + text = text.removesuffix("\n") + expected = json.dumps( + json.loads(text), + indent=writer.indent, + sort_keys=writer.sort_keys, + ensure_ascii=writer.ensure_ascii, + ) + assert text == expected, key + assert _decoded(written) == _decoded(held.to_key_value()) + assert type(held).from_key_value(written) == model + + +@pytest.mark.parametrize("model", MODELS, ids=MODEL_IDS) +def test_default_writer_writes_what_json_dumps_writes(model: object) -> None: + """A bare `to_key_value()` writes each document with the defaults of Python's `json` module, byte for byte. `ensure_ascii` escapes the non-ASCII attribute.""" + written: Mapping[str, bytes] = cast("Any", model).to_key_value() + for key, raw in written.items(): + assert raw == json.dumps(json.loads(raw)).encode("utf-8"), key + assert any(b"\\u00b5m" in raw for raw in written.values()) + + +@pytest.mark.parametrize( + "writer", + [ + JSONWriter(), + JSONWriter(indent=" ", sort_keys=True, ensure_ascii=False, trailing_newline=True), + ], + ids=repr, +) +def test_from_dict_returns_a_writer_and_round_trips_its_settings(writer: JSONWriter) -> None: + """`from_dict` hands back a `JSONWriter` unchanged, and builds an equal one from the mapping `to_dict` returns.""" + assert JSONWriter.from_dict(writer) is writer + assert JSONWriter.from_dict(writer.to_dict()) == writer + + +def test_schema_names_every_setting_of_the_writer() -> None: + """`JSONWriterSchema` and `JSONWriter` declare the same settings, so every setting can be given as a mapping.""" + assert set(JSONWriterSchema.__annotations__) == {each.name for each in fields(JSONWriter)} + + +def test_ensure_ascii_off_escapes_only_a_lone_surrogate() -> None: + """With `ensure_ascii` off, a lone surrogate the reader accepted from an escape is written back as that escape, since UTF-8 cannot encode it, while every other non-ASCII character is written as UTF-8. The store reads back to an equal model.""" + stored = { + "zarr.json": json.dumps( + {"zarr_format": 3, "node_type": "group", "attributes": {"x": "\ud800", "y": "µ"}} + ).encode("utf-8") + } + model = ZarrV3GroupMetadata.from_key_value(stored) + written = model.to_key_value({"ensure_ascii": False})["zarr.json"] + assert b'"\\ud800"' in written + assert "µ".encode() in written + assert ZarrV3GroupMetadata.from_key_value({"zarr.json": written}) == model + + +def test_error_from_dict_refuses_an_unknown_setting() -> None: + """A key that is not a setting is refused, so a misspelled setting is not dropped.""" + with pytest.raises(TypeError, match="unknown JSONWriter setting\\(s\\) 'indnet'"): + JSONWriter.from_dict(cast("Any", {"indnet": 2})) + + +def test_error_from_dict_refuses_a_value_that_is_not_a_mapping() -> None: + """Only a `JSONWriter` or a mapping of its settings describes a writer.""" + with pytest.raises( + TypeError, match="expected a JSONWriter or a mapping of its settings, got int" + ): + JSONWriter.from_dict(cast("Any", 2)) + + +def test_error_writer_refuses_a_bool_indent() -> None: + """A `bool` is not an indent, though `json.dumps` would read `True` as an indent of 1.""" + with pytest.raises(TypeError, match="indent must be an int, a str or None, got bool"): + JSONWriter(indent=cast("Any", True)) + + +def test_error_writer_refuses_an_indent_of_another_type() -> None: + """An indent that is not an `int`, a `str` or `None` is refused.""" + with pytest.raises(TypeError, match="indent must be an int, a str or None, got float"): + JSONWriter(indent=cast("Any", 2.0)) + + +def test_error_writer_refuses_a_negative_indent() -> None: + """A negative indent is refused, though `json.dumps` would read it as 0.""" + with pytest.raises(ValueError, match="indent must not be negative, got -3"): + JSONWriter(indent=-3) + + +def test_error_writer_refuses_an_indent_that_is_not_json_whitespace() -> None: + """A `str` indent must hold only JSON whitespace: `json.dumps` writes it verbatim, so any other character writes bytes no JSON reader parses.""" + with pytest.raises(ValueError, match="indent must hold only JSON whitespace, got 'x'"): + JSONWriter(indent="x") + + +@pytest.mark.parametrize("name", ["sort_keys", "ensure_ascii", "trailing_newline"]) +def test_error_writer_refuses_a_flag_that_is_not_a_bool(name: str) -> None: + """Each flag takes only a `bool`: a truthy value such as `1` is refused.""" + with pytest.raises(TypeError, match=f"{name} must be a bool, got int"): + JSONWriter(**{name: cast("Any", 1)}) diff --git a/packages/zarr-metadata/tests/model/test_store_json.py b/packages/zarr-metadata/tests/model/test_store_json.py index 411e215613..5a8d135e41 100644 --- a/packages/zarr-metadata/tests/model/test_store_json.py +++ b/packages/zarr-metadata/tests/model/test_store_json.py @@ -33,6 +33,8 @@ if TYPE_CHECKING: from collections.abc import Iterable + from zarr_metadata.model import JSONWriterLike + class _Written(Protocol): """The output of a model write: store keys, each with its bytes.""" @@ -45,7 +47,7 @@ class _Stored(Protocol): def to_json(self) -> Mapping[str, object]: ... - def to_key_value(self, *, indent: int | str | None = None) -> _Written: ... + def to_key_value(self, writer: JSONWriterLike = ...) -> _Written: ... class _Reading(Protocol): diff --git a/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py b/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py index 02b5521882..e66914b6d2 100644 --- a/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py +++ b/packages/zarr-metadata/tests/model/test_unclaimed_declarations.py @@ -7,6 +7,7 @@ import pytest from typing_extensions import TypedDict +from zarr_metadata import UNSET from zarr_metadata.definition import UnclaimedDeclaration from zarr_metadata.model import ( ZarrV2ArrayMetadata, @@ -15,6 +16,8 @@ ZarrV3ArrayMetadata, ZarrV3GroupMetadata, ZarrV3NodeMetadata, + read_node_metadata_v3, + read_openable_node_metadata_v3, ) from zarr_metadata.v3 import CORE_CONTEXT from zarr_metadata.v3.definition import ( @@ -190,3 +193,78 @@ def test_node_type_tells_a_type_checker_which_model_a_node_is() -> None: for node in nodes: if node.zarr_format == 3 and node.node_type == "group": assert node.consolidated() is None + + +@pytest.mark.parametrize( + ("document", "context", "unclaimed", "problems"), + [ + (ARRAY, CORE_CONTEXT, set(), set()), + ({**ARRAY, "codecs": ("bytes", ZSTD)}, CORE_CONTEXT, {("codecs", 1)}, set()), + ( + {**ARRAY, "codecs": (SHARD,)}, + CORE_CONTEXT, + {("codecs", 0, "configuration", "codecs", 1)}, + set(), + ), + ({**ARRAY, "acme": 1}, CORE_CONTEXT, {("acme",)}, set()), + ({**ARRAY, "acme": {"must_understand": False}}, CORE_CONTEXT, set(), set()), + ({**GROUP, "acme": {"name": "acme"}}, CORE_CONTEXT, {("acme",)}, set()), + ({**GROUP, "acme": {"must_understand": False}}, CORE_CONTEXT, set(), set()), + ( + {**ARRAY, "acme.mirror": {"codec": {"name": "acme.zip"}}}, + MIRRORING, + {("acme.mirror", "codec")}, + set(), + ), + ( + {**ARRAY, "acme.mirror": {"codec": {"name": "acme.zip"}, "must_understand": False}}, + MIRRORING, + set(), + set(), + ), + ( + {**ARRAY, "fill_value": "x", "codecs": ("bytes", ZSTD)}, + CORE_CONTEXT, + {("codecs", 1)}, + {("fill_value",)}, + ), + ({**GROUP, "node_type": "acme"}, CORE_CONTEXT, set(), {("node_type",)}), + ("not a node", CORE_CONTEXT, set(), {()}), + ], + ids=[ + "all-claimed", + "a-codec", + "a-codec-inside-a-shard", + "an-extra-member", + "a-waived-extra-member", + "a-group-member", + "a-waived-group-member", + "a-codec-inside-a-claimed-member", + "a-codec-inside-a-claimed-member-that-waives", + "with-other-problems", + "an-unknown-node-type", + "not-an-object", + ], +) +def test_read_openable_node_metadata_v3_reports_what_a_reader_must_understand( + document: object, + context: Any, # noqa: ANN401 - a v3 scope + unclaimed: set[tuple[str | int, ...]], + problems: set[tuple[str | int, ...]], +) -> None: + """`read_openable_node_metadata_v3` keeps every problem `read_node_metadata_v3` finds and adds one of kind `unclaimed_declaration` at the location of each declaration `unclaimed_declarations` lists, with what the document writes there as its input, whether or not the document has other problems. It gives a model exactly when there are no problems, and the model is the one `read_node_metadata_v3` gives.""" + plain = read_node_metadata_v3(document, context=context) + reading = read_openable_node_metadata_v3(document, context=context) + assert reading.problems[: len(plain.problems)] == plain.problems + assert {problem.loc for problem in plain.problems} == problems + added = reading.problems[len(plain.problems) :] + assert {problem.loc for problem in added} == unclaimed + assert all(problem.kind == "unclaimed_declaration" for problem in added) + assert all(problem.input is not UNSET for problem in added) + if plain.metadata is not None: + assert set(plain.metadata.unclaimed_declarations) == unclaimed + if len(reading.problems) == 0: + assert reading.metadata == plain.metadata + assert reading.metadata is not None + else: + assert reading.metadata is None diff --git a/packages/zarr-metadata/tests/test_docs.py b/packages/zarr-metadata/tests/test_docs.py index 7726151126..a05596075d 100644 --- a/packages/zarr-metadata/tests/test_docs.py +++ b/packages/zarr-metadata/tests/test_docs.py @@ -8,12 +8,14 @@ from __future__ import annotations +import textwrap from collections import defaultdict from pathlib import Path import pytest from pytest_examples import CodeExample, EvalExample, find_examples +import zarr_metadata.definition from zarr_metadata.model import ZarrV3ArrayMetadata PACKAGE_ROOT = Path(__file__).parent.parent @@ -71,3 +73,31 @@ def test_every_python_block_says_whether_it_runs() -> None: ) ] assert unmarked == [] + + +def test_the_extensions_page_example_is_the_definition_module_docstring_example() -> None: + """The acme.lz4 example on the extensions page is, line for line, the example in the `zarr_metadata.definition` module docstring. Only the page's copy runs under these tests, so this is what keeps the docstring's copy from drifting.""" + page_blocks = [ + example + for example in _examples() + if example.path == PACKAGE_ROOT / "docs" / "extensions.md" + and example.prefix_settings().get("session") == "extensions" + ] + assert len(page_blocks) == 1, ( + "expected exactly one extensions-session block on docs/extensions.md" + ) + docstring = zarr_metadata.definition.__doc__ + assert docstring is not None, "zarr_metadata.definition has no module docstring" + lines = docstring.splitlines() + marker = "from collections.abc import Iterator" + starts = [i for i, line in enumerate(lines) if line.strip() == marker] + assert len(starts) == 1, f"expected one line {marker!r} in the docstring, found {len(starts)}" + assert lines[starts[0] - 1].strip() == "```python", ( + f"expected {marker!r} to open a ```python fence in the docstring" + ) + end = starts[0] + while end < len(lines) and lines[end].strip() != "```": + end += 1 + assert end < len(lines), "the docstring's ```python fence is not closed" + docstring_example = textwrap.dedent("\n".join(lines[starts[0] : end])).strip() + assert page_blocks[0].source.strip().splitlines() == docstring_example.splitlines() diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 392c61a7a8..926b7753d4 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -60,6 +60,10 @@ def _group_rank(s: str) -> int: "MetadataValidationError", "ProblemKind", "UNSET", + # How `to_key_value` encodes a document as JSON bytes. + "JSONWriter", + "JSONWriterSchema", + "JSONWriterLike", # Store keys — the names the documents are persisted under. Defined in the # v2/v3 spec modules, re-exported through `zarr_metadata.model`. "ZARR_V2_ARRAY_METADATA_STORE_KEY", @@ -339,6 +343,10 @@ def test_all_is_grouped_and_unique() -> None: "HexFloat64", "JSONSchema", "JSONValue", + # How `to_key_value` encodes a document as JSON bytes. + "JSONWriter", + "JSONWriterSchema", + "JSONWriterLike", "UserJSONValue", # A pydantic field's scope, fixed: `Annotated[Model, InScope(CONTEXT)]`. "InScope", diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index f305245f4e..2eb8cb5597 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -30,7 +30,6 @@ save, save_array, save_group, - tree, zeros, zeros_like, ) @@ -179,7 +178,6 @@ def set_format(log_format: str) -> None: "save", "save_array", "save_group", - "tree", "zeros", "zeros_like", ] diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index f162c13a7b..3993ce35cc 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -7,7 +7,6 @@ import numpy as np import numpy.typing as npt -from typing_extensions import deprecated from zarr.abc.store import Store from zarr.core.array import ( @@ -44,7 +43,6 @@ GroupNotFoundError, NodeTypeValidationError, ZarrDeprecationWarning, - ZarrRuntimeWarning, ZarrUserWarning, ) from zarr.storage import StorePath @@ -91,7 +89,6 @@ "save", "save_array", "save_group", - "tree", "zeros", "zeros_like", ] @@ -109,9 +106,11 @@ def _infer_overwrite(mode: AccessModeLiteral) -> bool: return mode in _OVERWRITE_MODES -def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: +def _warn_ignored_kwargs(kwargs: dict[str, Any]) -> None: """ - Emit a "not yet implemented" warning for each provided keyword argument that is not None. + Emit a deprecation warning for each provided keyword argument that is not None. + + These are Zarr-Python 2 keyword arguments that have no effect in Zarr-Python 3. `kwargs` maps a keyword argument name to its supplied value. The `stacklevel` is chosen so the warning points at the caller of the public API function (the same location as an @@ -119,7 +118,11 @@ def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: """ for name, value in kwargs.items(): if value is not None: - warnings.warn(f"{name} is not yet implemented", ZarrRuntimeWarning, stacklevel=3) + warnings.warn( + f"{name} is ignored, and will be removed in a release after February 2027", + ZarrDeprecationWarning, + stacklevel=3, + ) def _get_shape_chunks(a: ArrayLike | Any) -> tuple[tuple[int, ...] | None, tuple[int, ...] | None]: @@ -632,31 +635,6 @@ async def save_group( await asyncio.gather(*aws) -@deprecated("Use AsyncGroup.tree instead.", category=ZarrDeprecationWarning) -async def tree(grp: AsyncGroup, expand: bool | None = None, level: int | None = None) -> Any: - """Provide a rich display of the hierarchy. - - !!! warning "Deprecated" - `zarr.tree()` is deprecated since v3.0.0 and will be removed in a future release. - Use `group.tree()` instead. - - Parameters - ---------- - grp : Group - Zarr or h5py group. - expand : bool, optional - Only relevant for HTML representation. If True, tree will be fully expanded. - level : int, optional - Maximum depth to descend into hierarchy. - - Returns - ------- - TreeRepr - A pretty-printable object displaying the hierarchy. - """ - return await grp.tree(expand=expand, level=level) - - async def array(data: npt.ArrayLike | AnyArray, **kwargs: Any) -> AnyAsyncArray: """Create an array filled with `data`. @@ -734,18 +712,15 @@ async def group( If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : StoreLike or None, default=None - Separate storage for chunks. Not implemented. + Ignored. Deprecated and will be removed in a release after February 2027. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. path : str, optional Group path within store. meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. + Ignored. Deprecated and will be removed in a release after February 2027. zarr_format : {2, 3, None}, optional The zarr format to use when saving. storage_options : dict @@ -858,23 +833,18 @@ async def open_group( (fail if exists). On a store that cannot delete keys, 'w' raises an error instead of replacing an existing node. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. path : str, optional Group path within store. chunk_store : StoreLike or None, default=None - Separate storage for chunks. See the - [storage documentation in the user guide][user-guide-store-like] - for a description of all valid StoreLike values. + Ignored. Deprecated and will be removed in a release after February 2027. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. + Ignored. Deprecated and will be removed in a release after February 2027. attributes : dict A dictionary of JSON-serializable values with user-defined attributes. use_consolidated : bool or str, default None @@ -900,7 +870,7 @@ async def open_group( The new group. """ - _warn_unimplemented_kwargs( + _warn_ignored_kwargs( { "cache_attrs": cache_attrs, "synchronizer": synchronizer, @@ -1005,15 +975,14 @@ async def create( [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. overwrite : bool, optional If True, delete all pre-existing data in `store` at `path` before creating the array. path : str, optional Path under which array is stored. chunk_store : StoreLike or None, default=None - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. + Ignored. Deprecated and will be removed in a release after February 2027. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. @@ -1034,18 +1003,13 @@ async def create( To create an array with no filters, provide an empty iterable or the value `None`. cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). + Ignored. Deprecated and will be removed in a release after February 2027. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. read_only : bool, optional - True if array should be protected against modification. + Ignored. Deprecated and will be removed in a release after February 2027. object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. + Ignored. Deprecated and will be removed in a release after February 2027. dimension_separator : {'.', '/'}, optional Separator placed between the dimensions of a chunk. Zarr format 2 only. Zarr format 3 arrays should use `chunk_key_encoding` instead. @@ -1062,7 +1026,7 @@ async def create( which instructs Zarr to choose the default Zarr format value defined in the runtime configuration. meta_array : array-like, optional - Not implemented. + Ignored. Deprecated and will be removed in a release after February 2027. attributes : dict[str, JSON], optional A dictionary of user attributes to store with the array. chunk_shape : ChunksLike, optional @@ -1128,7 +1092,7 @@ async def create( if zarr_format is None: zarr_format = _default_zarr_format() - _warn_unimplemented_kwargs( + _warn_ignored_kwargs( { "synchronizer": synchronizer, "chunk_store": chunk_store, diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 86b72dedaf..461c5b800b 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -2,15 +2,12 @@ from typing import TYPE_CHECKING, Any, Literal -from typing_extensions import deprecated - import zarr.api.asynchronous as async_api import zarr.core.array from zarr.core.array import DEFAULT_FILL_VALUE, Array, AsyncArray, CompressorLike from zarr.core.group import Group from zarr.core.sync import sync from zarr.core.sync_group import create_hierarchy -from zarr.errors import ZarrDeprecationWarning if TYPE_CHECKING: from collections.abc import Iterable @@ -69,7 +66,6 @@ "save", "save_array", "save_group", - "tree", "zeros", "zeros_like", ] @@ -389,31 +385,6 @@ def save_group( ) -@deprecated("Use Group.tree instead.", category=ZarrDeprecationWarning) -def tree(grp: Group, expand: bool | None = None, level: int | None = None) -> Any: - """Provide a rich display of the hierarchy. - - !!! warning "Deprecated" - `zarr.tree()` is deprecated since v3.0.0 and will be removed in a future release. - Use `group.tree()` instead. - - Parameters - ---------- - grp : Group - Zarr or h5py group. - expand : bool, optional - Only relevant for HTML representation. If True, tree will be fully expanded. - level : int, optional - Maximum depth to descend into hierarchy. - - Returns - ------- - TreeRepr - A pretty-printable object displaying the hierarchy. - """ - return sync(async_api.tree(grp._async_group, expand=expand, level=level)) - - # TODO: add type annotations for kwargs def array(data: npt.ArrayLike | AnyArray, **kwargs: Any) -> AnyArray: """Create an array filled with `data`. @@ -459,18 +430,15 @@ def group( If True, delete any pre-existing data in `store` at `path` before creating the group. chunk_store : StoreLike or None, default=None - Separate storage for chunks. Not implemented. + Ignored. Deprecated and will be removed in a release after February 2027. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. path : str, optional Group path within store. meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. + Ignored. Deprecated and will be removed in a release after February 2027. zarr_format : {2, 3, None}, optional The zarr format to use when saving. storage_options : dict @@ -529,23 +497,18 @@ def open_group( (fail if exists). On a store that cannot delete keys, 'w' raises an error instead of replacing an existing node. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. path : str, optional Group path within store. chunk_store : StoreLike or None, default=None - Separate storage for chunks. See the - [storage documentation in the user guide][user-guide-store-like] - for a description of all valid StoreLike values. + Ignored. Deprecated and will be removed in a release after February 2027. storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. + Ignored. Deprecated and will be removed in a release after February 2027. attributes : dict A dictionary of JSON-serializable values with user-defined attributes. use_consolidated : bool or str, default None @@ -710,15 +673,14 @@ def create( [storage documentation in the user guide][user-guide-store-like] for a description of all valid StoreLike values. synchronizer : object, optional - Array synchronizer. + Ignored. Deprecated and will be removed in a release after February 2027. overwrite : bool, optional If True, delete all pre-existing data in `store` at `path` before creating the array. path : str, optional Path under which array is stored. chunk_store : StoreLike or None, default=None - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. + Ignored. Deprecated and will be removed in a release after February 2027. filters : Iterable[Codec] | Literal["auto"], optional Iterable of filters to apply to each chunk of the array, in order, before serializing that chunk to bytes. @@ -739,18 +701,13 @@ def create( To create an array with no filters, provide an empty iterable or the value `None`. cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). + Ignored. Deprecated and will be removed in a release after February 2027. cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. + Ignored. Deprecated and will be removed in a release after February 2027. read_only : bool, optional - True if array should be protected against modification. + Ignored. Deprecated and will be removed in a release after February 2027. object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. + Ignored. Deprecated and will be removed in a release after February 2027. dimension_separator : {'.', '/'}, optional Separator placed between the dimensions of a chunk. Zarr format 2 only. Zarr format 3 arrays should use `chunk_key_encoding` instead. @@ -767,7 +724,7 @@ def create( which instructs Zarr to choose the default Zarr format value defined in the runtime configuration. meta_array : array-like, optional - Not implemented. + Ignored. Deprecated and will be removed in a release after February 2027. attributes : dict[str, JSON], optional A dictionary of user attributes to store with the array. chunk_shape : ChunksLike, optional diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 622161c458..621dca78e9 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -3,9 +3,12 @@ import sys import warnings from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, ClassVar, Final, Literal +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal -from zarr.abc.codec import ArrayBytesCodec +import numpy as np + +from zarr.abc.codec import ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin +from zarr.abc.store import RangeByteRequest from zarr.codecs._deprecated_enum import _coerce_enum_input, _DeprecatedStrEnumMeta from zarr.core.common import JSON, parse_named_configuration from zarr.core.dtype.common import HasEndianness @@ -14,8 +17,10 @@ if TYPE_CHECKING: from typing import Self + from zarr.abc.store import ByteGetter from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer + from zarr.core.indexing import Selector, SelectorTuple EndianLiteral = Literal["little", "big"] @@ -40,7 +45,7 @@ def _parse_endian(data: object) -> EndianLiteral: @dataclass(frozen=True) -class BytesCodec(ArrayBytesCodec): +class BytesCodec(ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin): """bytes codec""" is_fixed_size = True @@ -137,6 +142,73 @@ async def _decode_single( ) -> NDBuffer: return self._decode_sync(chunk_bytes, chunk_spec) + async def _decode_partial_single( + self, + byte_getter: ByteGetter, + selection: SelectorTuple, + chunk_spec: ArraySpec, + ) -> NDBuffer | None: + """Read only the part of an uncompressed chunk that a selection needs. + + The chunk is stored in C order, so each row along its first axis is a + contiguous run of bytes. The rows from the first to the last one the + selection touches are fetched with a single range request, and the + selection is applied to them. A selection that touches every row reads + the whole chunk, as before. + """ + window = _row_window(selection, chunk_spec.shape) + chunk_bytes = await byte_getter.get( + prototype=chunk_spec.prototype, byte_range=_row_byte_range(window, chunk_spec) + ) + return self._decode_rows(chunk_bytes, selection, window, chunk_spec) + + def _decode_partial_sync( + self, + byte_getter: Any, + selection: SelectorTuple, + chunk_spec: ArraySpec, + ) -> NDBuffer | None: + """Sync equivalent of `_decode_partial_single`.""" + window = _row_window(selection, chunk_spec.shape) + chunk_bytes = byte_getter.get_sync( + prototype=chunk_spec.prototype, byte_range=_row_byte_range(window, chunk_spec) + ) + return self._decode_rows(chunk_bytes, selection, window, chunk_spec) + + def _decode_rows( + self, + chunk_bytes: Buffer | None, + selection: SelectorTuple, + window: tuple[int, int, SelectorTuple] | None, + chunk_spec: ArraySpec, + ) -> NDBuffer | None: + """Decode the bytes fetched for a row window and apply the selection to them.""" + if chunk_bytes is None: + return None + if window is None: + return self._decode_sync(chunk_bytes, chunk_spec)[selection] + first, stop, rows_selection = window + row_bytes = _row_bytes(chunk_spec) + start, end = first * row_bytes, stop * row_bytes + chunk_size = chunk_spec.shape[0] * row_bytes + # A store may not honor the byte range it was given. The length of what + # it sent tells which bytes those are. + if len(chunk_bytes) == end - start: + pass + elif len(chunk_bytes) == chunk_size: + # The whole chunk, as an HTTP server that ignores the Range header sends. + chunk_bytes = chunk_bytes[start:end] + elif len(chunk_bytes) == chunk_size - start: + # Everything from the start of the range to the end of the chunk. + chunk_bytes = chunk_bytes[: end - start] + else: + raise ValueError( + f"Requested bytes {start} to {end} of a {chunk_size} byte chunk, " + f"but the store returned {len(chunk_bytes)} bytes." + ) + rows_spec = replace(chunk_spec, shape=(stop - first, *chunk_spec.shape[1:])) + return self._decode_sync(chunk_bytes, rows_spec)[rows_selection] + def _encode_sync( self, chunk_array: NDBuffer, @@ -165,3 +237,60 @@ async def _encode_single( def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) -> int: return input_byte_length + + +def _row_window( + selection: SelectorTuple, shape: tuple[int, ...] +) -> tuple[int, int, SelectorTuple] | None: + """The rows along axis 0 that a selection touches, and the selection relative to them. + + Returns the first row, one past the last row, and the selection shifted so + that it indexes an array holding only those rows. Returns None when the + selection touches every row or the rows cannot be determined, in which case + the whole chunk is read. + """ + if len(shape) == 0 or not isinstance(selection, tuple) or len(selection) == 0: + return None + first_axis = selection[0] + shifted: Selector + if isinstance(first_axis, slice): + rows = range(*first_axis.indices(shape[0])) + if len(rows) == 0 or rows.step < 0: + return None + first, stop = rows[0], rows[-1] + 1 + shifted = slice(0, stop - first, rows.step) + elif isinstance(first_axis, int | np.integer): + first = int(first_axis) % shape[0] + stop = first + 1 + shifted = 0 + elif isinstance(first_axis, np.ndarray): + if first_axis.dtype == bool: + indices = np.nonzero(first_axis)[0] + else: + indices = first_axis % shape[0] + if indices.size == 0: + return None + first, stop = int(indices.min()), int(indices.max()) + 1 + shifted = first_axis[first:stop] if first_axis.dtype == bool else indices - first + else: + return None + if (first, stop) == (0, shape[0]): + return None + return first, stop, (shifted, *selection[1:]) + + +def _row_bytes(chunk_spec: ArraySpec) -> int: + """The number of bytes in one row along axis 0 of a chunk.""" + row_items = int(np.prod(chunk_spec.shape[1:])) + return chunk_spec.dtype.to_native_dtype().itemsize * row_items + + +def _row_byte_range( + window: tuple[int, int, SelectorTuple] | None, chunk_spec: ArraySpec +) -> RangeByteRequest | None: + """The byte range holding a row window, or None to read the whole chunk.""" + if window is None: + return None + first, stop, _ = window + row_bytes = _row_bytes(chunk_spec) + return RangeByteRequest(first * row_bytes, stop * row_bytes) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 8c473f3223..3735a4da18 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -1011,20 +1011,6 @@ def serializer(self) -> ArrayBytesCodec | None: codec for codec in self.metadata.inner_codecs if isinstance(codec, ArrayBytesCodec) ) - @property - @deprecated("Use AsyncArray.compressors instead.", category=ZarrDeprecationWarning) - def compressor(self) -> Numcodec | None: - """ - Compressor that is applied to each chunk of the array. - - !!! warning "Deprecated" - `Array.compressor` is deprecated since v3.0.0 and will be removed in a future release. - Use [`Array.compressors`][zarr.AsyncArray.compressors] instead. - """ - if self.metadata.zarr_format == 2: - return self.metadata.compressor - raise TypeError("`compressor` is not available for Zarr format 3 arrays.") - @property def compressors(self) -> tuple[Numcodec, ...] | tuple[BytesBytesCodec, ...]: """ @@ -2309,18 +2295,6 @@ def serializer(self) -> ArrayBytesCodec | None: """ return self.async_array.serializer - @property - @deprecated("Use Array.compressors instead.", category=ZarrDeprecationWarning) - def compressor(self) -> Numcodec | None: - """ - Compressor that is applied to each chunk of the array. - - !!! warning "Deprecated" - `array.compressor` is deprecated since v3.0.0 and will be removed in a future release. - Use [`array.compressors`][zarr.Array.compressors] instead. - """ - return self.async_array.compressor - @property def compressors(self) -> tuple[Numcodec, ...] | tuple[BytesBytesCodec, ...]: """ diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 136267fc58..806401d2b4 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -134,7 +134,7 @@ def chunk_grid(self) -> ChunkGrid: "ArrayV2Metadata.chunk_grid is deprecated. " "Use ChunkGrid.from_metadata(metadata) instead.", DeprecationWarning, - stacklevel=2, + stacklevel=3, # skip the `cached_property.__get__` frame ) return ChunkGrid.from_sizes(self.shape, tuple(self.chunks)) diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index 074463af85..fc04a79057 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -42,7 +42,7 @@ def __setattr__(self, attr: str, value: Any) -> None: "zarr.config to configure array.v2_default_compressor " "e.g. config.set({'codecs.zstd':'numcodecs.Zstd', 'array.v2_default_compressor.numeric': 'zstd'})", ZarrDeprecationWarning, - stacklevel=1, + stacklevel=2, ) else: super().__setattr__(attr, value) diff --git a/tests/test_api.py b/tests/test_api.py index e162ddfeea..53d8353b7f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1188,18 +1188,6 @@ def test_load_local(tmp_path: Path, path: str | None, load_read_only: bool) -> N assert np.array_equal(result, data) -def test_tree() -> None: - g1 = zarr.group() - g1.create_group("foo") - g3 = g1.create_group("bar") - g3.create_group("baz") - g5 = g3.create_group("qux") - g5.create_array("baz", shape=(100,), chunks=(10,), dtype="float64") - with pytest.warns(ZarrDeprecationWarning, match=r"Group\.tree instead\."): # noqa: PT031 - assert repr(zarr.tree(g1)) == repr(g1.tree()) - assert str(zarr.tree(g1)) == str(g1.tree()) - - # @pytest.mark.parametrize("stores_from_path", [False, True]) # @pytest.mark.parametrize( # "with_chunk_store,listable", @@ -2133,8 +2121,26 @@ def test_auto_chunks(f: Callable[..., AnyArray]) -> None: assert a.chunks == (500, 500) -@pytest.mark.parametrize("kwarg_name", ["synchronizer", "chunk_store", "cache_attrs", "meta_array"]) -def test_unimplemented_kwarg_warnings(kwarg_name: str) -> None: +@pytest.mark.parametrize( + "kwarg_name", + [ + "synchronizer", + "chunk_store", + "cache_metadata", + "cache_attrs", + "object_codec", + "read_only", + "meta_array", + ], +) +def test_ignored_kwarg_warnings(kwarg_name: str) -> None: kwargs = {kwarg_name: 1} - with pytest.warns(RuntimeWarning, match=".* is not yet implemented"): + with pytest.warns(ZarrDeprecationWarning, match=f"{kwarg_name} is ignored"): zarr.create(shape=(1,), **kwargs) # type: ignore[arg-type] + + +@pytest.mark.parametrize("kwarg_name", ["synchronizer", "chunk_store", "cache_attrs", "meta_array"]) +def test_ignored_kwarg_warnings_open_group(kwarg_name: str) -> None: + kwargs = {kwarg_name: 1} + with pytest.warns(ZarrDeprecationWarning, match=f"{kwarg_name} is ignored"): + zarr.open_group(store=MemoryStore(), **kwargs) # type: ignore[arg-type] diff --git a/tests/test_codecs/test_bytes.py b/tests/test_codecs/test_bytes.py index ead778f526..8ac4df7ba1 100644 --- a/tests/test_codecs/test_bytes.py +++ b/tests/test_codecs/test_bytes.py @@ -13,11 +13,13 @@ import zarr from tests.conftest import Expect, ExpectFail from zarr.abc.codec import SupportsSyncCodec +from zarr.abc.store import OffsetByteRequest, RangeByteRequest from zarr.codecs.bytes import ( ENDIAN, BytesCodec, Endian, EndianLiteral, + _row_window, ) from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import NDBuffer, default_buffer_prototype @@ -29,6 +31,8 @@ from .test_codecs import _AsyncArrayProxy if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from zarr.abc.store import Store @@ -338,3 +342,211 @@ def test_bytes_codec_evolve_structured_single_byte_fields_clears_endian() -> Non spec = _make_array_spec(dtype) evolved = codec.evolve_from_array_spec(spec) assert evolved.endian is None + + +@pytest.fixture( + params=[ + "zarr.core.codec_pipeline.BatchedCodecPipeline", + "zarr.core.codec_pipeline.FusedCodecPipeline", + ] +) +def codec_pipeline(request: pytest.FixtureRequest) -> Iterator[None]: + """Run a test once with each codec pipeline.""" + with zarr.config.set({"codec_pipeline.path": request.param}): + yield + + +class _RangeLoggingStore(zarr.storage.MemoryStore): + """A memory store that records the byte ranges it serves for chunk keys.""" + + def __init__(self) -> None: + super().__init__() + self.reads: list[tuple[str, Any, int]] = [] + + def _log(self, key: str, byte_range: Any, buf: Any) -> None: + if buf is not None and not key.endswith("zarr.json"): + self.reads.append((key, byte_range, len(buf))) + + async def get(self, key: str, prototype: Any = None, byte_range: Any = None) -> Any: + buf = await super().get(key, prototype, byte_range) + self._log(key, byte_range, buf) + return buf + + def get_sync(self, key: str, *, prototype: Any = None, byte_range: Any = None) -> Any: + buf = super().get_sync(key, prototype=prototype, byte_range=byte_range) + self._log(key, byte_range, buf) + return buf + + +def _single_chunk_array( + data: np.ndarray[Any, Any], **kwargs: Any +) -> tuple[Any, _RangeLoggingStore]: + store = _RangeLoggingStore() + arr = zarr.create_array( + store, shape=data.shape, dtype=data.dtype, chunks=data.shape, fill_value=0, **kwargs + ) + arr[...] = data + store.reads.clear() + return arr, store + + +@pytest.mark.usefixtures("codec_pipeline") +@pytest.mark.parametrize( + ("selection", "rows_read"), + [ + (np.s_[500:600, 3], 100), + (np.s_[500:600], 100), + (np.s_[777, 5], 1), + (np.s_[-1], 1), + (np.s_[10:20:3, ::2], 10), # rows 10 to 19 are fetched + (np.s_[...], 10_000), + ], +) +def test_uncompressed_partial_read(selection: Any, rows_read: int) -> None: + """Reading part of an uncompressed chunk fetches only the rows it touches.""" + data = np.arange(100_000, dtype="int16").reshape(10_000, 10) + arr, store = _single_chunk_array(data, compressors=None) + np.testing.assert_array_equal(arr[selection], data[selection]) + assert sum(n for *_, n in store.reads) == rows_read * 10 * 2 + + +@pytest.mark.usefixtures("codec_pipeline") +@pytest.mark.parametrize("dtype", [">u2", " None: + """Partial reads return the same values as numpy for several dtypes, byte orders, and shapes.""" + shape = {1: (1000,), 2: (100, 7), 3: (20, 5, 3)}[ndim] + data = (np.arange(int(np.prod(shape))) % 200).astype(dtype).reshape(shape) + arr, _ = _single_chunk_array(data, compressors=None) + selections: list[Any] = [np.s_[3:9], np.s_[4], np.s_[-3:], np.s_[1:15:4]] + for selection in selections: + np.testing.assert_array_equal(arr[selection], data[selection]) + np.testing.assert_array_equal(arr.oindex[[1, 5, 2]], data[[1, 5, 2]]) + mask = np.zeros(shape[0], dtype=bool) + mask[[2, 5, 11]] = True + np.testing.assert_array_equal(arr.oindex[mask], data[mask]) + coords = tuple(np.array([0, 6, 3]) % n for n in shape) + np.testing.assert_array_equal(arr.vindex[coords], data[coords]) + + +@pytest.mark.usefixtures("codec_pipeline") +def test_uncompressed_partial_read_across_chunks() -> None: + """A selection spanning several chunks reads only the touched rows of each.""" + data = np.arange(40_000, dtype="int32").reshape(4000, 10) + store = _RangeLoggingStore() + arr = zarr.create_array( + store, shape=data.shape, dtype="int32", chunks=(1000, 10), compressors=None + ) + arr[...] = data + store.reads.clear() + np.testing.assert_array_equal(arr[990:1010, 2], data[990:1010, 2]) + assert sorted((key, n) for key, _, n in store.reads) == [("c/0/0", 400), ("c/1/0", 400)] + + +@pytest.mark.usefixtures("codec_pipeline") +def test_compressed_chunks_are_read_whole() -> None: + """With a compressor the chunk bytes cannot be split, so the whole chunk is fetched.""" + data = np.arange(100_000, dtype="int16").reshape(10_000, 10) + arr, store = _single_chunk_array(data) # default compressor + np.testing.assert_array_equal(arr[500:600, 3], data[500:600, 3]) + ((_, byte_range, _),) = store.reads + assert byte_range is None + + +@pytest.mark.usefixtures("codec_pipeline") +@pytest.mark.parametrize("selection", [np.s_[10:20], np.s_[...]]) +def test_uncompressed_partial_read_missing_chunk(selection: Any) -> None: + """An unwritten chunk reads as the fill value, for a partial and a whole-chunk read.""" + store = _RangeLoggingStore() + arr = zarr.create_array( + store, shape=(100, 4), dtype="int16", chunks=(100, 4), compressors=None, fill_value=7 + ) + expected = np.full((100, 4), 7, dtype="int16")[selection] + np.testing.assert_array_equal(arr[selection], expected) + + +class _RewritesRangeStore(zarr.storage.MemoryStore): + """A memory store that does not serve the byte range it is asked for. + + `rewrite` maps the requested range to the one that is served instead. + """ + + def __init__(self, rewrite: Callable[[RangeByteRequest], Any]) -> None: + super().__init__() + self.rewrite = rewrite + + def _served(self, byte_range: Any) -> Any: + if isinstance(byte_range, RangeByteRequest): + return self.rewrite(byte_range) + return byte_range + + async def get(self, key: str, prototype: Any = None, byte_range: Any = None) -> Any: + return await super().get(key, prototype, self._served(byte_range)) + + def get_sync(self, key: str, *, prototype: Any = None, byte_range: Any = None) -> Any: + return super().get_sync(key, prototype=prototype, byte_range=self._served(byte_range)) + + +def _ignore_range(byte_range: RangeByteRequest) -> None: + """Send the whole value, as an HTTP server that ignores the Range header does.""" + + +def _ignore_range_end(byte_range: RangeByteRequest) -> OffsetByteRequest: + """Send everything from the start of the range to the end of the value.""" + return OffsetByteRequest(byte_range.start) + + +def _one_byte_short(byte_range: RangeByteRequest) -> RangeByteRequest: + return RangeByteRequest(byte_range.start, byte_range.end - 1) + + +def _rewriting_array( + rewrite: Callable[[RangeByteRequest], Any], +) -> tuple[Any, np.ndarray[Any, Any]]: + data = np.arange(400, dtype=" None: + """A store that sends more than the requested range still reads correctly.""" + arr, data = _rewriting_array(rewrite) + np.testing.assert_array_equal(arr[selection], data[selection]) + + +@pytest.mark.usefixtures("codec_pipeline") +def test_uncompressed_partial_read_store_sends_unexpected_length() -> None: + """A response whose length matches no known byte range behavior is an error.""" + arr, _ = _rewriting_array(_one_byte_short) + with pytest.raises(ValueError, match="the store returned 15 bytes"): + arr[50:52, 0] + + +@pytest.mark.parametrize( + ("selection", "shape"), + [ + ((slice(5, 5), slice(None)), (10, 4)), # no rows + ((slice(8, 2, -1), slice(None)), (10, 4)), # negative step + ((np.array([], dtype=np.intp), slice(None)), (10, 4)), # no rows + ((slice(None), 2), (10, 4)), # every row + ((np.array([0, 9]), slice(None)), (10, 4)), # first and last row + ((..., 2), (10, 4)), # rows not determined + (slice(2, 4), (10,)), # not a tuple + ((), ()), # zero-dimensional chunk + ], +) +def test_row_window_reads_whole_chunk(selection: Any, shape: tuple[int, ...]) -> None: + """Selections with no narrower row window than the whole chunk give None.""" + assert _row_window(selection, shape) is None diff --git a/tests/test_fused_pipeline.py b/tests/test_fused_pipeline.py index 06e79d3de7..ea44eda324 100644 --- a/tests/test_fused_pipeline.py +++ b/tests/test_fused_pipeline.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import multiprocessing as mp +import sys from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any from unittest.mock import patch @@ -37,6 +39,7 @@ if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable, Iterable + from pathlib import Path from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, NDBuffer @@ -543,6 +546,48 @@ def read_row_block(i: int, handle: zarr.Array[Any] = fresh) -> np.ndarray: np.testing.assert_array_equal(fut.result(), data[i * 4 : (i + 1) * 4, :]) +def _read_region(arr: zarr.Array[Any], selection: Any) -> np.ndarray: + return np.asarray(arr[selection]) + + +@pytest.mark.skipif(sys.platform == "win32", reason="fork is not available on Windows") +@pytest.mark.filterwarnings( + r"ignore:This process \(pid=\d+\) is multi-threaded, use of fork\(\):DeprecationWarning" +) +def test_forked_child_does_not_reuse_parent_pool(tmp_path: Path) -> None: + """A forked child must not reuse the parent's codec-pipeline thread pool. + + Regression test for https://github.com/zarr-developers/zarr-python/issues/4478. + Executor threads do not survive `fork()`, but the inherited executor keeps the + idle-semaphore permits its parent's workers released. Each submission in the + child consumes a permit instead of starting a thread, so a read of no more + chunks than there are permits queues work that nothing runs. + + `max_workers` is pinned rather than left to `os.cpu_count()`: on a 1-CPU host + the default resolves to 1, the parent never creates the pool, and the test + would pass without exercising the bug. + """ + import zarr.core.codec_pipeline as cp_mod + + data = np.arange(100).reshape(10, 10) + with zarr_config.set(_FUSED_POOL_CONFIG): + arr = zarr.create_array( + store=str(tmp_path / "a.zarr"), shape=data.shape, chunks=(5, 5), dtype=data.dtype + ) + arr[:] = data + # Each multi-chunk read releases one idle permit per completed task, so after + # two 4-chunk reads the pool holds more permits than the child read submits. + arr[:] + arr[:] + assert cp_mod._pool is not None, "the parent read did not create the pool" + with mp.get_context("fork").Pool(1) as pool: + # `[:5]` spans 2 chunks. Without the after-fork reset, both submissions + # are swallowed by the dead pool and `get` times out. + result = pool.starmap_async(_read_region, [(arr, np.s_[:5])]) + (read_back,) = result.get(timeout=30) + np.testing.assert_array_equal(read_back, data[:5]) + + def test_shared_transform_decode_alternating_specs() -> None: """A single ChunkTransform must decode chunks of DIFFERENT specs correctly when calls alternate, exercising eviction/refill of its single-entry diff --git a/tests/test_v2.py b/tests/test_v2.py index c6842c4902..52068724fd 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -147,13 +147,13 @@ def test_create_array_defaults(store: Store) -> None: g = zarr.open(store, mode="w", zarr_format=2) assert isinstance(g, Group) arr = g.create_array("one", dtype="i8", shape=(1,), chunks=(1,), compressor=None) - assert arr.async_array.compressor is None + assert arr.compressors == () assert not (arr.filters) arr = g.create_array("two", dtype="i8", shape=(1,), chunks=(1,)) - assert arr.async_array.compressor is not None + assert arr.compressors != () assert not (arr.filters) arr = g.create_array("three", dtype="i8", shape=(1,), chunks=(1,), compressor=Zstd()) - assert arr.async_array.compressor is not None + assert arr.compressors != () assert not (arr.filters) with pytest.raises(ValueError): g.create_array( @@ -229,8 +229,18 @@ def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal[" def test_default_compressor_deprecation_warning() -> None: - with pytest.warns(ZarrDeprecationWarning, match="default_compressor is deprecated"): + with pytest.warns(ZarrDeprecationWarning, match="default_compressor is deprecated") as caught: zarr.storage.default_compressor = "zarr.codecs.zstd.ZstdCodec()" # type: ignore[attr-defined] + assert caught[0].filename == __file__ + + +def test_v2_metadata_chunk_grid_deprecation_warning() -> None: + metadata = zarr.create(shape=(2,), chunks=(1,), zarr_format=2).metadata + with pytest.warns( + DeprecationWarning, match="ArrayV2Metadata.chunk_grid is deprecated" + ) as caught: + metadata.chunk_grid # noqa: B018 + assert caught[0].filename == __file__ @pytest.mark.parametrize("fill_value", [None, (b"", 0, 0.0)], ids=["no_fill", "fill"])