feat(zarr-metadata): draw nested fields and codec pipelines

packages/zarr-metadata/changes/4597.bugfix.md

@@ -0,0 +1 @@
+`cast_value` and `scale_offset` reject `fixed_length_utf32`, `null_terminated_bytes`, `raw_bytes` and `variable_length_bytes`, whose values are not numbers. The strategies no longer draw them for either codec. The `scale_offset` strategy now draws `offset` and `scale` in every spelling a fill value of the data type takes, including integers, `NaN`, the infinities and the hex strings of their bits; a scale is still never zero.

packages/zarr-metadata/changes/4597.feature.md

@@ -0,0 +1 @@
+Added strategies to `zarr_metadata.testing.strategies` for the definitions that hold other declarations: `struct`, `cast_value` and `sharding_indexed`. Their nested declarations are drawn from the same `Strategies` mapping, so a struct's field types, a cast's target data type and a shard's codecs come from every definition the mapping holds. A new `pipelines` strategy draws a whole codec pipeline for a `Chunk`, drawing each codec for the chunk the codec before it hands on, as that codec's `transition` gives it. Shards are drawn at most one inside another, and structs likewise. Every function a strategy record holds now also takes the `Strategies` mapping as its last argument, and `declarations` reads what it draws in the scope of every definition the mapping holds.

packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py

@@ -14,86 +14,122 @@
 from hypothesis import strategies as st
 
 from zarr_metadata.testing._strategy import MAX_RANK, ChunkGridStrategy
-from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID
-from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID
+from zarr_metadata.v3.chunk_grid.rectilinear import (
+    RECTILINEAR_CHUNK_GRID,
+    RectilinearChunkGridConfiguration,
+    RectilinearDimSpec,
+)
+from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID, RegularChunkGridConfiguration
 
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
-    from zarr_metadata._common import JSONValue
+    from zarr_metadata.testing._strategy import Strategies
     from zarr_metadata.v3._definition import ChunkGridDefinition
 
 _MAX_LENGTH: Final = 16
-"""The longest chunk drawn along a dimension whose length is unknown."""
+"""The longest chunk drawn along a dimension whose length is unknown, before any factor its edges share."""
+
+_MAX_CHUNK_LENGTH: Final = 2**63 - 1
+"""The longest chunk drawn at all: what a signed 64-bit size holds, as readers index with."""
 
 _MAX_RUNS: Final = 5
 """The most runs of equal lengths a rectilinear dimension is drawn with."""
 
-_MAX_COUNT: Final = 4
-"""The most chunks a run is drawn with."""
+_FEW_CHUNKS: Final = 4
+"""The most chunks a short run is drawn with."""
+
+_MAX_WRITTEN_OUT: Final = 64
+"""The most chunks a run is written out with, one length for each; a longer run is always a pair."""
+
+_MAX_FACTOR: Final = 16
+"""The largest factor all of a dimension's edges are drawn to share."""
+
+_MANY_CHUNKS: Final = 2**20
+"""The most chunks a run is drawn with along a dimension whose length is unknown or 0, into which it grows when the array is resized."""
 
 
 def _length(extent: int | None) -> st.SearchStrategy[int]:
-    """A chunk length along a dimension of `extent`: at least 1, and up to twice the extent when it is known and not 0."""
+    """A chunk length along a dimension of `extent`: at least 1, and up to twice the extent when it is known and not 0, within a signed 64-bit size."""
     if extent is None or extent == 0:
         return st.integers(1, _MAX_LENGTH)
-    return st.integers(1, 2 * extent)
+    return st.integers(1, min(2 * extent, _MAX_CHUNK_LENGTH))
 
 
-def _regular(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]:
+def _regular(
+    shape: tuple[int, ...] | None, _strategies: Strategies
+) -> st.SearchStrategy[RegularChunkGridConfiguration]:
     """A chunk length for each dimension."""
     if shape is None:
         lengths = st.lists(_length(None), max_size=MAX_RANK).map(tuple)
     else:
         lengths = st.tuples(*(_length(extent) for extent in shape))
-    return lengths.map(lambda chunk_shape: {"chunk_shape": chunk_shape})
+    return lengths.map(lambda chunk_shape: RegularChunkGridConfiguration(chunk_shape=chunk_shape))
+
 
+def _count(extent: int | None, size: int) -> st.SearchStrategy[int]:
+    """How many chunks of `size` a run holds: a few, or as many as it takes to cover `extent`."""
+    many = _MANY_CHUNKS if extent is None or extent == 0 else max(extent // size, _FEW_CHUNKS)
+    return st.integers(1, _FEW_CHUNKS) | st.integers(1, many)
 
-def _spelled_run(draw: st.DrawFn, size: int, count: int) -> list[JSONValue]:
-    """`count` chunks of `size`: one `[size, count]` pair, or the size written once for each."""
-    if draw(st.booleans()):
+
+def _spelled_run(draw: st.DrawFn, size: int, count: int) -> list[int | tuple[int, int]]:
+    """`count` chunks of `size`: one `[size, count]` pair, or, for a run short enough, the size written once for each."""
+    if count > _MAX_WRITTEN_OUT or draw(st.booleans()):
         return [(size, count)]
     return [size] * count
 
 
 @st.composite
-def _edges(draw: st.DrawFn, extent: int | None) -> JSONValue:
+def _edges(draw: st.DrawFn, extent: int | None) -> tuple[int | tuple[int, int], ...]:
     """Edge lengths along a dimension, covering `extent`, in runs spelled as pairs or written out.
 
     Runs are drawn apart, so two of one size can sit side by side and a
-    pair can count one chunk: spellings the canonical form merges. A
+    pair can count one chunk: spellings the canonical form merges. A run
+    can be long, since a pair costs the same however many chunks it
+    counts, which is what the spelling is for. Its edges often share a
+    factor, as they must for a shard to divide them into inner chunks
+    longer than 1; edges drawn apart rarely do. A
     dimension of length zero, or of a length not known, may be covered by
     no edges at all.
     """
     min_runs = 0 if extent is None or extent == 0 else 1
-    runs = draw(
-        st.lists(
-            st.tuples(st.integers(1, _MAX_LENGTH), st.integers(1, _MAX_COUNT)),
-            min_size=min_runs,
-            max_size=_MAX_RUNS,
+    factor = draw(st.just(1) | st.integers(2, _MAX_FACTOR))
+    sizes = [
+        factor * size
+        for size in draw(
+            st.lists(st.integers(1, _MAX_LENGTH), min_size=min_runs, max_size=_MAX_RUNS)
         )
-    )
+    ]
+    runs = [(size, draw(_count(extent, size))) for size in sizes]
     covered = sum(size * count for size, count in runs)
     if extent is not None and covered < extent:
-        runs.append((extent - covered, 1))
-    spelled: list[JSONValue] = []
+        # Rounded up to the factor, so the last edge shares it, and past the dimension if need be.
+        runs.append((-(-(extent - covered) // factor) * factor, 1))
+    spelled: list[int | tuple[int, int]] = []
     for size, count in runs:
         spelled.extend(_spelled_run(draw, size, count))
     return tuple(spelled)
 
 
-def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[JSONValue]:
+def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[RectilinearDimSpec]:
     """A bare length, which repeats until it covers the dimension, or edges that cover it."""
     return _length(extent) | _edges(extent)
 
 
-def _rectilinear(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]:
+def _rectilinear(
+    shape: tuple[int, ...] | None, _strategies: Strategies
+) -> st.SearchStrategy[RectilinearChunkGridConfiguration]:
     """Inline chunk shapes, one entry for each dimension."""
     if shape is None:
         dimensions = st.lists(_rectilinear_dimension(None), max_size=MAX_RANK).map(tuple)
     else:
         dimensions = st.tuples(*(_rectilinear_dimension(extent) for extent in shape))
-    return dimensions.map(lambda chunk_shapes: {"kind": "inline", "chunk_shapes": chunk_shapes})
+    return dimensions.map(
+        lambda chunk_shapes: RectilinearChunkGridConfiguration(
+            kind="inline", chunk_shapes=chunk_shapes
+        )
+    )
 
 
 CHUNK_GRID_STRATEGIES: Final[Mapping[ChunkGridDefinition[Any], ChunkGridStrategy]] = {

packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py

@@ -10,29 +10,35 @@
 from zarr_metadata.v3.chunk_key_encoding.default import (
     DEFAULT_CHUNK_KEY_ENCODING,
     DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR,
+    DefaultChunkKeyEncodingConfiguration,
 )
 from zarr_metadata.v3.chunk_key_encoding.v2 import (
     V2_CHUNK_KEY_ENCODING,
     V2_CHUNK_KEY_ENCODING_SEPARATOR,
+    V2ChunkKeyEncodingConfiguration,
 )
 
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
     from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition
 
+_DEFAULT_SEPARATORS: Final = st.builds(DefaultChunkKeyEncodingConfiguration) | st.sampled_from(
+    DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR
+).map(lambda separator: DefaultChunkKeyEncodingConfiguration(separator=separator))
+
+_V2_SEPARATORS: Final = st.builds(V2ChunkKeyEncodingConfiguration) | st.sampled_from(
+    V2_CHUNK_KEY_ENCODING_SEPARATOR
+).map(lambda separator: V2ChunkKeyEncodingConfiguration(separator=separator))
+
 CHUNK_KEY_ENCODING_STRATEGIES: Final[
     Mapping[ChunkKeyEncodingDefinition[Any], ChunkKeyEncodingStrategy]
 ] = {
     DEFAULT_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy(
-        configurations=st.fixed_dictionaries(
-            {}, optional={"separator": st.sampled_from(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)}
-        )
+        configurations=lambda _strategies: _DEFAULT_SEPARATORS
     ),
     V2_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy(
-        configurations=st.fixed_dictionaries(
-            {}, optional={"separator": st.sampled_from(V2_CHUNK_KEY_ENCODING_SEPARATOR)}
-        )
+        configurations=lambda _strategies: _V2_SEPARATORS
     ),
 }
 """A strategy for each chunk key encoding: a `separator`, or none, which means the encoding's default."""