diff --git a/packages/zarr-metadata/changes/4597.bugfix.md b/packages/zarr-metadata/changes/4597.bugfix.md new file mode 100644 index 0000000000..374ccaff92 --- /dev/null +++ b/packages/zarr-metadata/changes/4597.bugfix.md @@ -0,0 +1 @@ +`cast_value` and `scale_offset` reject `fixed_length_utf32`, `null_terminated_bytes`, `raw_bytes` and `variable_length_bytes`, whose values are not numbers. The strategies no longer draw them for either codec. The `scale_offset` strategy now draws `offset` and `scale` in every spelling a fill value of the data type takes, including integers, `NaN`, the infinities and the hex strings of their bits; a scale is still never zero. diff --git a/packages/zarr-metadata/changes/4597.feature.md b/packages/zarr-metadata/changes/4597.feature.md new file mode 100644 index 0000000000..bda7cebc08 --- /dev/null +++ b/packages/zarr-metadata/changes/4597.feature.md @@ -0,0 +1 @@ +Added strategies to `zarr_metadata.testing.strategies` for the definitions that hold other declarations: `struct`, `cast_value` and `sharding_indexed`. Their nested declarations are drawn from the same `Strategies` mapping, so a struct's field types, a cast's target data type and a shard's codecs come from every definition the mapping holds. A new `pipelines` strategy draws a whole codec pipeline for a `Chunk`, drawing each codec for the chunk the codec before it hands on, as that codec's `transition` gives it. Shards are drawn at most one inside another, and structs likewise. Every function a strategy record holds now also takes the `Strategies` mapping as its last argument, and `declarations` reads what it draws in the scope of every definition the mapping holds. diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py index b7528a3907..a987f241c8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -14,86 +14,122 @@ from hypothesis import strategies as st from zarr_metadata.testing._strategy import MAX_RANK, ChunkGridStrategy -from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID -from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID +from zarr_metadata.v3.chunk_grid.rectilinear import ( + RECTILINEAR_CHUNK_GRID, + RectilinearChunkGridConfiguration, + RectilinearDimSpec, +) +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID, RegularChunkGridConfiguration if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata._common import JSONValue + from zarr_metadata.testing._strategy import Strategies from zarr_metadata.v3._definition import ChunkGridDefinition _MAX_LENGTH: Final = 16 -"""The longest chunk drawn along a dimension whose length is unknown.""" +"""The longest chunk drawn along a dimension whose length is unknown, before any factor its edges share.""" + +_MAX_CHUNK_LENGTH: Final = 2**63 - 1 +"""The longest chunk drawn at all: what a signed 64-bit size holds, as readers index with.""" _MAX_RUNS: Final = 5 """The most runs of equal lengths a rectilinear dimension is drawn with.""" -_MAX_COUNT: Final = 4 -"""The most chunks a run is drawn with.""" +_FEW_CHUNKS: Final = 4 +"""The most chunks a short run is drawn with.""" + +_MAX_WRITTEN_OUT: Final = 64 +"""The most chunks a run is written out with, one length for each; a longer run is always a pair.""" + +_MAX_FACTOR: Final = 16 +"""The largest factor all of a dimension's edges are drawn to share.""" + +_MANY_CHUNKS: Final = 2**20 +"""The most chunks a run is drawn with along a dimension whose length is unknown or 0, into which it grows when the array is resized.""" def _length(extent: int | None) -> st.SearchStrategy[int]: - """A chunk length along a dimension of `extent`: at least 1, and up to twice the extent when it is known and not 0.""" + """A chunk length along a dimension of `extent`: at least 1, and up to twice the extent when it is known and not 0, within a signed 64-bit size.""" if extent is None or extent == 0: return st.integers(1, _MAX_LENGTH) - return st.integers(1, 2 * extent) + return st.integers(1, min(2 * extent, _MAX_CHUNK_LENGTH)) -def _regular(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: +def _regular( + shape: tuple[int, ...] | None, _strategies: Strategies +) -> st.SearchStrategy[RegularChunkGridConfiguration]: """A chunk length for each dimension.""" if shape is None: lengths = st.lists(_length(None), max_size=MAX_RANK).map(tuple) else: lengths = st.tuples(*(_length(extent) for extent in shape)) - return lengths.map(lambda chunk_shape: {"chunk_shape": chunk_shape}) + return lengths.map(lambda chunk_shape: RegularChunkGridConfiguration(chunk_shape=chunk_shape)) + +def _count(extent: int | None, size: int) -> st.SearchStrategy[int]: + """How many chunks of `size` a run holds: a few, or as many as it takes to cover `extent`.""" + many = _MANY_CHUNKS if extent is None or extent == 0 else max(extent // size, _FEW_CHUNKS) + return st.integers(1, _FEW_CHUNKS) | st.integers(1, many) -def _spelled_run(draw: st.DrawFn, size: int, count: int) -> list[JSONValue]: - """`count` chunks of `size`: one `[size, count]` pair, or the size written once for each.""" - if draw(st.booleans()): + +def _spelled_run(draw: st.DrawFn, size: int, count: int) -> list[int | tuple[int, int]]: + """`count` chunks of `size`: one `[size, count]` pair, or, for a run short enough, the size written once for each.""" + if count > _MAX_WRITTEN_OUT or draw(st.booleans()): return [(size, count)] return [size] * count @st.composite -def _edges(draw: st.DrawFn, extent: int | None) -> JSONValue: +def _edges(draw: st.DrawFn, extent: int | None) -> tuple[int | tuple[int, int], ...]: """Edge lengths along a dimension, covering `extent`, in runs spelled as pairs or written out. Runs are drawn apart, so two of one size can sit side by side and a - pair can count one chunk: spellings the canonical form merges. A + pair can count one chunk: spellings the canonical form merges. A run + can be long, since a pair costs the same however many chunks it + counts, which is what the spelling is for. Its edges often share a + factor, as they must for a shard to divide them into inner chunks + longer than 1; edges drawn apart rarely do. A dimension of length zero, or of a length not known, may be covered by no edges at all. """ min_runs = 0 if extent is None or extent == 0 else 1 - runs = draw( - st.lists( - st.tuples(st.integers(1, _MAX_LENGTH), st.integers(1, _MAX_COUNT)), - min_size=min_runs, - max_size=_MAX_RUNS, + factor = draw(st.just(1) | st.integers(2, _MAX_FACTOR)) + sizes = [ + factor * size + for size in draw( + st.lists(st.integers(1, _MAX_LENGTH), min_size=min_runs, max_size=_MAX_RUNS) ) - ) + ] + runs = [(size, draw(_count(extent, size))) for size in sizes] covered = sum(size * count for size, count in runs) if extent is not None and covered < extent: - runs.append((extent - covered, 1)) - spelled: list[JSONValue] = [] + # Rounded up to the factor, so the last edge shares it, and past the dimension if need be. + runs.append((-(-(extent - covered) // factor) * factor, 1)) + spelled: list[int | tuple[int, int]] = [] for size, count in runs: spelled.extend(_spelled_run(draw, size, count)) return tuple(spelled) -def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[JSONValue]: +def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[RectilinearDimSpec]: """A bare length, which repeats until it covers the dimension, or edges that cover it.""" return _length(extent) | _edges(extent) -def _rectilinear(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: +def _rectilinear( + shape: tuple[int, ...] | None, _strategies: Strategies +) -> st.SearchStrategy[RectilinearChunkGridConfiguration]: """Inline chunk shapes, one entry for each dimension.""" if shape is None: dimensions = st.lists(_rectilinear_dimension(None), max_size=MAX_RANK).map(tuple) else: dimensions = st.tuples(*(_rectilinear_dimension(extent) for extent in shape)) - return dimensions.map(lambda chunk_shapes: {"kind": "inline", "chunk_shapes": chunk_shapes}) + return dimensions.map( + lambda chunk_shapes: RectilinearChunkGridConfiguration( + kind="inline", chunk_shapes=chunk_shapes + ) + ) CHUNK_GRID_STRATEGIES: Final[Mapping[ChunkGridDefinition[Any], ChunkGridStrategy]] = { diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py index 9c525118c9..84487174ad 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py @@ -10,10 +10,12 @@ from zarr_metadata.v3.chunk_key_encoding.default import ( DEFAULT_CHUNK_KEY_ENCODING, DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, + DefaultChunkKeyEncodingConfiguration, ) from zarr_metadata.v3.chunk_key_encoding.v2 import ( V2_CHUNK_KEY_ENCODING, V2_CHUNK_KEY_ENCODING_SEPARATOR, + V2ChunkKeyEncodingConfiguration, ) if TYPE_CHECKING: @@ -21,18 +23,22 @@ from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition +_DEFAULT_SEPARATORS: Final = st.builds(DefaultChunkKeyEncodingConfiguration) | st.sampled_from( + DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR +).map(lambda separator: DefaultChunkKeyEncodingConfiguration(separator=separator)) + +_V2_SEPARATORS: Final = st.builds(V2ChunkKeyEncodingConfiguration) | st.sampled_from( + V2_CHUNK_KEY_ENCODING_SEPARATOR +).map(lambda separator: V2ChunkKeyEncodingConfiguration(separator=separator)) + CHUNK_KEY_ENCODING_STRATEGIES: Final[ Mapping[ChunkKeyEncodingDefinition[Any], ChunkKeyEncodingStrategy] ] = { DEFAULT_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( - configurations=st.fixed_dictionaries( - {}, optional={"separator": st.sampled_from(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)} - ) + configurations=lambda _strategies: _DEFAULT_SEPARATORS ), V2_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( - configurations=st.fixed_dictionaries( - {}, optional={"separator": st.sampled_from(V2_CHUNK_KEY_ENCODING_SEPARATOR)} - ) + configurations=lambda _strategies: _V2_SEPARATORS ), } """A strategy for each chunk key encoding: a `separator`, or none, which means the encoding's default.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py index bd6c0927d9..5d2d34f90e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -1,15 +1,22 @@ -"""Strategies for each codec this package defines that holds no other declaration. +"""Strategies for each codec this package defines. A codec handed an array draws what fits the chunk: a `bytes` codec an `endian` for a type whose values take several bytes, a `transpose` an -`order` with an entry for each axis. Handed `Chunk()`, nothing is known -of the chunk, and what is drawn is what the codec's rules accept. +`order` with an entry for each axis, a shard inner chunks that divide +it. Handed `Chunk()`, nothing is known of the chunk, and what is drawn +is what the codec's rules accept. The declarations a codec holds are drawn +from the strategies: the data type a `cast_value` casts to, and a +shard's two pipelines, each drawn for the chunk the shard's definition +says it is handed. """ from __future__ import annotations +import math +from functools import partial from typing import TYPE_CHECKING, Any, Final +from hypothesis import reject from hypothesis import strategies as st from zarr_metadata.definition._definition import AcceptedDeclaration, EmptyConfiguration @@ -17,22 +24,53 @@ COMPLEX_WIDTHS, FLOAT_WIDTHS, INTEGER_RANGES, - FloatWidth, - finite_floats, - float_spellings, + float_fill_values, + is_zero_float, +) +from zarr_metadata.testing._draw import ( + briefly, + data_type_declarations, + data_type_names, + fill_values_of, + pipelines_for, + read, ) from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy -from zarr_metadata.v3._definition import Chunk, storage_of -from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS -from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE -from zarr_metadata.v3.codec.bytes import BYTES_CODEC, ENDIANNESS +from zarr_metadata.v3._definition import Chunk, DataTypeDefinition, storage_of +from zarr_metadata.v3.codec._arithmetic import FLOATING_POINT, NO_REAL_NUMBERS, NOT_NUMBERS +from zarr_metadata.v3.codec.blosc import ( + BLOSC_CNAME, + BLOSC_CODEC, + BLOSC_NO_SHUFFLE, + BLOSC_SHUFFLE, + BloscCodecConfiguration, +) +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, ENDIANNESS, BytesCodecConfiguration +from zarr_metadata.v3.codec.cast_value import ( + CAST_OUT_OF_RANGE_MODE, + CAST_ROUNDING_MODE, + CAST_VALUE_CODEC, + CastOutOfRangeMode, + CastValueCodecConfiguration, + ScalarMap, +) from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC -from zarr_metadata.v3.codec.gzip import GZIP_CODEC -from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC -from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC, GzipCodecConfiguration +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC, ScaleOffsetCodecConfiguration +from zarr_metadata.v3.codec.sharding_indexed import ( + SHARDING_INDEX_LOCATION, + SHARDING_INDEXED_CODEC, + ShardingIndexedCodecConfiguration, +) +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC, TransposeCodecConfiguration from zarr_metadata.v3.codec.vlen_bytes import VLEN_BYTES_CODEC from zarr_metadata.v3.codec.vlen_utf8 import VLEN_UTF8_CODEC -from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL +from zarr_metadata.v3.codec.zstd import ( + ZSTD_CODEC, + ZSTD_MAX_LEVEL, + ZSTD_MIN_LEVEL, + ZstdCodecConfiguration, +) from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE @@ -40,7 +78,9 @@ from collections.abc import Mapping from zarr_metadata._common import JSONValue - from zarr_metadata.v3._definition import CodecDefinition + from zarr_metadata.definition._definition import ResolvedDeclaration + from zarr_metadata.testing._strategy import Configurations, Strategies + from zarr_metadata.v3._definition import CodecDefinition, CodecSize, Lengths _BLOSC_MAX_TYPESIZE: Final = 255 """The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" @@ -48,8 +88,26 @@ _BLOSC_MAX_BLOCKSIZE: Final = 2**31 - 1 """The largest block c-blosc takes, a signed 32-bit size; 0 lets it choose.""" +_MAX_DIVISOR_SEARCH: Final = 2**12 +"""The largest divisor of a shard's length searched for: one of an axis longer than its square is still found by its pair.""" + +_MAX_INNER_LENGTH: Final = 16 +"""The longest inner chunk drawn along an axis whose shard lengths are unknown.""" + +_MAX_SHARD_DEPTH: Final = 1 +"""How deep shards are drawn inside a shard: one inside another, and none inside that.""" + +_MAX_SCALARS: Final = 3 +"""The most entries each direction of a `cast_value` `scalar_map` is drawn with.""" + +_STATIC: Final[frozenset[CodecSize]] = frozenset({"static"}) +"""What a shard index's codecs are: of static size, so a reader finds the index before reading it.""" + _ENDIAN: Final = st.sampled_from(ENDIANNESS) +_SMALL_NUMBERS: Final = st.integers(0, 127) +"""Values every integral and floating-point data type holds: for a chunk whose data type is not known.""" + _NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) """The numpy time types, each stored as a signed 64-bit integer.""" @@ -57,73 +115,84 @@ """The signed 64-bit integers but the least, which numpy reads as `NaT`: the finite numpy times.""" -def _bytes(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: +@st.composite +def _endians(draw: st.DrawFn, *, required: bool) -> BytesCodecConfiguration: + configuration = BytesCodecConfiguration() + if required or draw(st.booleans()): + configuration["endian"] = draw(_ENDIAN) + return configuration + + +def _bytes( + chunk: Chunk, _strategies: Strategies +) -> st.SearchStrategy[BytesCodecConfiguration] | None: """An `endian`, required for a data type whose values take several bytes; None for one whose values vary in size.""" storage = None if chunk.data_type is None else storage_of(chunk.data_type) if storage == "variable_length": return None - if storage == "multi_byte": - return st.fixed_dictionaries({"endian": _ENDIAN}) - return st.fixed_dictionaries({}, optional={"endian": _ENDIAN}) + return _endians(required=storage == "multi_byte") @st.composite -def _blosc(draw: st.DrawFn) -> Mapping[str, JSONValue]: +def _blosc(draw: st.DrawFn) -> BloscCodecConfiguration: """A compressor, a level, a shuffle and a block size; and a `typesize`, which only `noshuffle` may leave out. Under `noshuffle` the spec says `typesize` "is ignored", so it is drawn there too, a spelling that is not the simplest. """ shuffle = draw(st.sampled_from(BLOSC_SHUFFLE)) - typesize = st.integers(1, _BLOSC_MAX_TYPESIZE) - required: dict[str, st.SearchStrategy[JSONValue]] = { - "cname": st.sampled_from(BLOSC_CNAME), - "clevel": st.integers(0, 9), - "shuffle": st.just(shuffle), - "blocksize": st.integers(0, _BLOSC_MAX_BLOCKSIZE), - } - if shuffle == BLOSC_NO_SHUFFLE: - return draw(st.fixed_dictionaries(required, optional={"typesize": typesize})) - return draw(st.fixed_dictionaries({**required, "typesize": typesize})) + configuration = BloscCodecConfiguration( + cname=draw(st.sampled_from(BLOSC_CNAME)), + clevel=draw(st.integers(0, 9)), + shuffle=shuffle, + blocksize=draw(st.integers(0, _BLOSC_MAX_BLOCKSIZE)), + ) + if shuffle != BLOSC_NO_SHUFFLE or draw(st.booleans()): + configuration["typesize"] = draw(st.integers(1, _BLOSC_MAX_TYPESIZE)) + return configuration -def _order(rank: int) -> st.SearchStrategy[Mapping[str, JSONValue]]: - return st.permutations(range(rank)).map(lambda order: {"order": tuple(order)}) +def _order(rank: int) -> st.SearchStrategy[TransposeCodecConfiguration]: + return st.permutations(range(rank)).map( + lambda order: TransposeCodecConfiguration(order=tuple(order)) + ) -def _transpose(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]]: +def _transpose( + chunk: Chunk, _strategies: Strategies +) -> st.SearchStrategy[TransposeCodecConfiguration]: """A permutation of the chunk's axes; of some number of axes, when the chunk does not say.""" if chunk.rank is not None: return _order(chunk.rank) return st.integers(0, MAX_RANK).flatmap(_order) -def _finite_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: - """A finite float of `width` bits, as a number or the hex string of its bits.""" - return finite_floats(width).flatmap(lambda value: float_spellings(value, width)) - - -def _non_zero_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: - """A finite, non-zero float of `width` bits, as a number or the hex string of its bits.""" - return ( - finite_floats(width) - .filter(lambda value: value != 0) - .flatmap(lambda value: float_spellings(value, width)) - ) - - -def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: - """An `offset` and a `scale`, each a finite value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. +@st.composite +def _offsets_and_scales( + draw: st.DrawFn, offsets: st.SearchStrategy[JSONValue], scales: st.SearchStrategy[JSONValue] +) -> ScaleOffsetCodecConfiguration: + configuration = ScaleOffsetCodecConfiguration() + if draw(st.booleans()): + configuration["offset"] = draw(offsets) + if draw(st.booleans()): + configuration["scale"] = draw(scales) + return configuration + + +def _scale_offset( + chunk: Chunk, _strategies: Strategies +) -> st.SearchStrategy[ScaleOffsetCodecConfiguration] | None: + """An `offset` and a `scale`, each a value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. Each is a fill value of the chunk's data type (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), - a float's as a number or the hex string of its bits, a complex - number's as a pair of them. Each is drawn finite, and the scale - non-zero, so that decoding never divides by zero; a scale small - enough that decoding overflows is still drawn. A numpy time is drawn - as the signed 64-bit integer it is stored as, never `NaT`, since it - "is compatible with any codec that supports arrays of signed 64-bit - integers" + so a float's is in every spelling a fill value takes, a `NaN` and an + infinity among them, and a complex number's is a pair of them. The + scale is drawn non-zero in any spelling, so that decoding never divides + by zero; a scale small enough that decoding overflows is still drawn. A + numpy time is drawn as the signed 64-bit integer it is stored as, never + `NaT`, since it "is compatible with any codec that supports arrays of + signed 64-bit integers" (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). A data type this package does not define, whose values it cannot draw, takes only the configuration with neither, the identity. @@ -133,7 +202,7 @@ def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | data_type = chunk.data_type definition = data_type.definition if isinstance(data_type, AcceptedDeclaration) else None if definition is None: - offset: st.SearchStrategy[JSONValue] = st.integers(0, 127) + offset: st.SearchStrategy[JSONValue] = _SMALL_NUMBERS scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) elif definition.name in NOT_NUMBERS: return None @@ -142,25 +211,33 @@ def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | offset = st.integers(low, high) scale = st.integers(low, high).filter(lambda value: value != 0) elif definition in FLOAT_WIDTHS: - offset = _finite_float_values(FLOAT_WIDTHS[definition]) - scale = _non_zero_float_values(FLOAT_WIDTHS[definition]) + offset = float_fill_values(FLOAT_WIDTHS[definition]) + scale = offset.filter(lambda value: not is_zero_float(value)) elif definition in COMPLEX_WIDTHS: - width = COMPLEX_WIDTHS[definition] - offset = st.tuples(_finite_float_values(width), _finite_float_values(width)) + part = float_fill_values(COMPLEX_WIDTHS[definition]) + offset = st.tuples(part, part) # Non-zero in either part is a non-zero complex number. - scale = st.one_of( - st.tuples(_non_zero_float_values(width), _finite_float_values(width)), - st.tuples(_finite_float_values(width), _non_zero_float_values(width)), + scale = st.tuples(part, part).filter( + lambda pair: not (is_zero_float(pair[0]) and is_zero_float(pair[1])) ) else: - return EMPTY_CONFIGURATIONS - return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) + return st.builds(ScaleOffsetCodecConfiguration) + return _offsets_and_scales(offset, scale) + + +@st.composite +def _zstd(draw: st.DrawFn) -> ZstdCodecConfiguration: + """A level, and a checksum or none, which means no checksum.""" + configuration = ZstdCodecConfiguration(level=draw(st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL))) + if draw(st.booleans()): + configuration["checksum"] = draw(st.booleans()) + return configuration def _vlen(definition: CodecDefinition[Any]) -> CodecStrategy: """A codec with nothing to configure that takes only some data types: none for a chunk of another, which its own chunk rules say.""" return CodecStrategy( - configurations=lambda chunk: ( + configurations=lambda chunk, _strategies: ( None if any(definition.chunk_rules(EmptyConfiguration(), {}, chunk)) else EMPTY_CONFIGURATIONS @@ -168,27 +245,193 @@ def _vlen(definition: CodecDefinition[Any]) -> CodecStrategy: ) -def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> CodecStrategy: +def _unchunked(configurations: Configurations) -> CodecStrategy: """A codec whose configurations do not depend on the chunk it is handed.""" - return CodecStrategy(configurations=lambda _chunk: configurations) + return CodecStrategy(configurations=lambda _chunk, _strategies: configurations) + + +def _name_read(data_type: ResolvedDeclaration[DataTypeDefinition[Any]] | None) -> str | None: + """The name of the definition that accepted `data_type`; None when none did.""" + return data_type.definition.name if isinstance(data_type, AcceptedDeclaration) else None + + +def _source_scalars(chunk: Chunk, strategies: Strategies) -> st.SearchStrategy[JSONValue]: + """Values of the chunk's data type: its fill values, or small numbers when no scope read it or no strategy draws for it.""" + source = chunk.data_type + if not isinstance(source, AcceptedDeclaration) or source.definition not in strategies: + return _SMALL_NUMBERS + return fill_values_of(source, strategies) + + +@st.composite +def _scalar_maps( + draw: st.DrawFn, sources: st.SearchStrategy[JSONValue], targets: st.SearchStrategy[JSONValue] +) -> ScalarMap: + """Entries pairing a value of each type, the one an entry maps from first.""" + scalar_map = ScalarMap() + if draw(st.booleans()): + scalar_map["encode"] = tuple( + draw(st.lists(st.tuples(sources, targets), max_size=_MAX_SCALARS)) + ) + if draw(st.booleans()): + scalar_map["decode"] = tuple( + draw(st.lists(st.tuples(targets, sources), max_size=_MAX_SCALARS)) + ) + return scalar_map + + +@st.composite +def _cast_value_configuration( + draw: st.DrawFn, + targets: list[str], + sources: st.SearchStrategy[JSONValue], + strategies: Strategies, +) -> CastValueCodecConfiguration: + """A data type to cast to, a rounding, an out-of-range mode it allows, and a scalar map between the two types.""" + declaration = draw(data_type_declarations(targets, strategies)) + target: ResolvedDeclaration[DataTypeDefinition[Any]] = read( + declaration, DataTypeDefinition, strategies + ) + modes: tuple[CastOutOfRangeMode, ...] = CAST_OUT_OF_RANGE_MODE + if _name_read(target) in FLOATING_POINT: + modes = tuple(mode for mode in modes if mode != "wrap") + configuration = CastValueCodecConfiguration(data_type=declaration) + if draw(st.booleans()): + configuration["rounding"] = draw(st.sampled_from(CAST_ROUNDING_MODE)) + if draw(st.booleans()): + configuration["out_of_range"] = draw(st.sampled_from(modes)) + if draw(st.booleans()): + configuration["scalar_map"] = draw( + _scalar_maps(sources, fill_values_of(target, strategies)) + ) + return configuration + + +def _cast_value( + chunk: Chunk, strategies: Strategies +) -> st.SearchStrategy[CastValueCodecConfiguration] | None: + """A cast to a data type that models real numbers, from one that does; None for a chunk of one that does not. + + The codec "is only defined for data types that model real numbers" + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md?plain=1#L5-L9), + on either side of the cast. Each `scalar_map` entry pairs a value of + the type it casts from with one of the type it casts to, in the order + of its direction; `wrap` is drawn only for a type that is not + floating-point. + """ + if _name_read(chunk.data_type) in NO_REAL_NUMBERS: + return None + targets = [name for name in data_type_names(strategies) if name not in NO_REAL_NUMBERS] + if len(targets) == 0: + return None + return _cast_value_configuration(targets, _source_scalars(chunk, strategies), strategies) + + +def _divisors(lengths: frozenset[int] | None) -> st.SearchStrategy[int]: + """An inner chunk length that divides each of `lengths`; any, when they are unknown or none but 0.""" + common = 0 if lengths is None else math.gcd(*lengths) + if common == 0: + return st.integers(1, _MAX_INNER_LENGTH) + # Divisors come in pairs, one of each no greater than the square root. + small = [ + n for n in range(1, min(math.isqrt(common), _MAX_DIVISOR_SEARCH) + 1) if common % n == 0 + ] + # 1 and the length itself go last: Hypothesis draws the first of several + # options far more often when a draw of variable length comes before it. + return st.sampled_from( + sorted({*small, *(common // n for n in small)}, key=lambda n: (n in (1, common), n)) + ) + + +def _inner_chunk_shapes(lengths: Lengths | None) -> st.SearchStrategy[tuple[int, ...]]: + """An inner chunk length for each axis of the shard, dividing the lengths shards take along it.""" + if lengths is None: + return st.lists(st.integers(1, _MAX_INNER_LENGTH), max_size=MAX_RANK).map(tuple) + return st.tuples(*(_divisors(axis) for axis in lengths)) + + +@st.composite +def _sharding_configuration( + draw: st.DrawFn, chunk: Chunk, strategies: Strategies, inner: Strategies +) -> ShardingIndexedCodecConfiguration: + """Inner chunks that divide the shard, and the two pipelines, each drawn for what the shard's definition says it is handed. + + The inner codecs are drawn from `inner`, which says how a shard + inside this one is drawn, if at all. + """ + chunk_shape = draw(_inner_chunk_shapes(chunk.lengths)) + sized: ShardingIndexedCodecConfiguration = { + "chunk_shape": chunk_shape, + "codecs": (), + "index_codecs": (), + } + handed = SHARDING_INDEXED_CODEC.pipelines(sized, {}, chunk) + codecs = pipelines_for(handed["codecs"], inner) + index_codecs = pipelines_for(handed["index_codecs"], strategies, _STATIC) + if codecs is None or index_codecs is None: + reject() # a codec that takes a chunk of the data type, but not of these lengths + configuration = ShardingIndexedCodecConfiguration( + chunk_shape=chunk_shape, codecs=draw(codecs), index_codecs=draw(index_codecs) + ) + if draw(st.booleans()): + configuration["index_location"] = draw(st.sampled_from(SHARDING_INDEX_LOCATION)) + return configuration + + +_NO_SHAPE: Final[ShardingIndexedCodecConfiguration] = { + "chunk_shape": (), + "codecs": (), + "index_codecs": (), +} +"""A shard configuration that says nothing but what its definition's `pipelines` reads: the data type each pipeline is handed.""" + + +def _sharding( + chunk: Chunk, strategies: Strategies, *, depth: int +) -> st.SearchStrategy[ShardingIndexedCodecConfiguration] | None: + """A shard of the chunk, holding shards `depth` deep at most; None when no pipeline is drawn for its inner chunks or its index. + + Each pipeline needs a codec that holds no pipelines and takes the + data type its definition hands it -- the shard's own for the inner + chunks, and for the index one of static size; whether one does is + asked of a chunk of that data type and nothing else known, since the + inner lengths are drawn after. Its inner codecs are drawn from the + same strategies but for a shard, which holds one level fewer, or is + not drawn: shards drawn without end would nest without end. So is any + definition named as a shard, which would otherwise claim the name in the + scope the inner codecs are read in. + """ + handed = SHARDING_INDEXED_CODEC.pipelines(_NO_SHAPE, {}, Chunk(None, chunk.data_type)) + if pipelines_for(Chunk(None, handed["codecs"].data_type), strategies) is None: + return None + if pipelines_for(Chunk(None, handed["index_codecs"].data_type), strategies, _STATIC) is None: + return None + inner = {d: s for d, s in strategies.items() if d.name != SHARDING_INDEXED_CODEC.name} + if depth > 0: + inner[SHARDING_INDEXED_CODEC] = _shards(depth - 1) + return _sharding_configuration(chunk, strategies, briefly(inner)) + + +def _shards(depth: int) -> CodecStrategy: + """Shards that hold shards `depth` deep at most.""" + return CodecStrategy(configurations=partial(_sharding, depth=depth)) CODEC_STRATEGIES: Final[Mapping[CodecDefinition[Any], CodecStrategy]] = { BLOSC_CODEC: _unchunked(_blosc()), BYTES_CODEC: CodecStrategy(configurations=_bytes), + CAST_VALUE_CODEC: CodecStrategy(configurations=_cast_value), CRC32C_CODEC: _unchunked(EMPTY_CONFIGURATIONS), - GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), + GZIP_CODEC: _unchunked( + st.integers(0, 9).map(lambda level: GzipCodecConfiguration(level=level)) + ), SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), + SHARDING_INDEXED_CODEC: _shards(_MAX_SHARD_DEPTH), TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), VLEN_BYTES_CODEC: _vlen(VLEN_BYTES_CODEC), VLEN_UTF8_CODEC: _vlen(VLEN_UTF8_CODEC), - ZSTD_CODEC: _unchunked( - st.fixed_dictionaries( - {"level": st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL)}, - optional={"checksum": st.booleans()}, - ) - ), + ZSTD_CODEC: _unchunked(_zstd()), } -"""A strategy for each codec that holds no other declaration.""" +"""A strategy for each codec.""" __all__ = ["CODEC_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py index be68ce7b10..0149ea392f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -1,20 +1,32 @@ -"""Strategies for each data type this package defines that holds no other declaration. +"""Strategies for each data type this package defines. A fill value is drawn in every spelling the data type's spec gives it: a float's as a number, a named non-finite value or a hex string of its -bits, a `bytes` value's as an array of byte values or base64. +bits, a `bytes` value's as an array of byte values or base64. A struct's +field types are drawn from the strategies, and its fill value is one +for each field, drawn by that field's own type. """ from __future__ import annotations import base64 +import math import struct +from functools import partial from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args from hypothesis import strategies as st from zarr_metadata._numpy import NUMPY_TIME_MAX_SCALE_FACTOR +from zarr_metadata.testing._draw import ( + briefly, + data_type_declarations, + data_type_names, + fill_values_of, + read, +) from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, DataTypeStrategy +from zarr_metadata.v3._definition import DataTypeDefinition, storage_of from zarr_metadata.v3.data_type._float import FloatSpecialFillValue from zarr_metadata.v3.data_type._numpy_time import NUMPY_TIME_UNIT from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE @@ -36,10 +48,17 @@ NULL_TERMINATED_BYTES_DATA_TYPE, NullTerminatedBytesConfiguration, ) -from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE -from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_datetime64 import ( + NUMPY_DATETIME64_DATA_TYPE, + NumpyDatetime64Configuration, +) +from zarr_metadata.v3.data_type.numpy_timedelta64 import ( + NUMPY_TIMEDELTA64_DATA_TYPE, + NumpyTimedelta64Configuration, +) from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE +from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE, StructConfiguration, StructField from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE @@ -54,7 +73,8 @@ from collections.abc import Mapping from zarr_metadata._common import JSONValue - from zarr_metadata.v3._definition import DataTypeDefinition + from zarr_metadata.testing._strategy import Strategies + from zarr_metadata.v3 import ZarrV3DeclarationJSON FloatWidth = Literal[16, 32, 64] @@ -89,10 +109,11 @@ _MAX_BYTES: Final = 16 """The longest `bytes` fill value drawn.""" +_MAX_STRUCT_FIELDS: Final = 4 +"""The most fields a struct is drawn with.""" -def finite_floats(width: FloatWidth) -> st.SearchStrategy[float]: - """Finite numbers a float of `width` bits holds exactly.""" - return st.floats(width=width, allow_nan=False, allow_infinity=False) +_MAX_STRUCT_DEPTH: Final = 1 +"""How deep structs are drawn inside a struct: one inside another, and none inside that.""" _PACKED: Final[Mapping[FloatWidth, str]] = {16: ">e", 32: ">f", 64: ">d"} @@ -104,12 +125,42 @@ def _hex_spellings(digits: str) -> st.SearchStrategy[str]: return st.sampled_from((f"0x{digits}", f"0x{digits.upper()}")) -def float_spellings(value: float, width: FloatWidth) -> st.SearchStrategy[JSONValue]: - """`value`, a float of `width` bits, as a number or as the hex string of its bits.""" - return st.just(value) | _hex_spellings(struct.pack(_PACKED[width], value).hex()) +_EXACT_INTEGERS: Final[Mapping[FloatWidth, int]] = {16: 2**11, 32: 2**24, 64: 2**53} +"""The magnitude up to which a float of each width holds every integer.""" + +_MANTISSA_BITS: Final[Mapping[FloatWidth, int]] = {16: 10, 32: 23, 64: 52} +"""How many of a float's bits, at the low end, are its mantissa; those above it are the exponent and the sign.""" + + +def _bits(value: int, width: FloatWidth) -> str: + """`value`, a float's `width` bits, as hex digits.""" + return f"{value:0{width // 4}x}" + +def _nan_digits(width: FloatWidth) -> st.SearchStrategy[str]: + """The bits of a NaN of any sign and payload: the exponent all ones, and the mantissa not zero.""" + mantissa_bits = _MANTISSA_BITS[width] + exponent = (2 ** (width - 1 - mantissa_bits) - 1) << mantissa_bits + return st.tuples(st.sampled_from((0, 1)), st.integers(1, 2**mantissa_bits - 1)).map( + lambda parts: _bits((parts[0] << (width - 1)) | exponent | parts[1], width) + ) + + +def _float_digits(width: FloatWidth) -> st.SearchStrategy[str]: + """The bits of a float in hex digits: of a number, an infinity or a NaN, or any bits at all. + + Bits drawn as an integer alone are nearly all of small numbers, since + Hypothesis favors small integers: a high exponent is rarely drawn, and a + `float64` is then subnormal four times in five. + """ + floats = (st.floats(width=width) | st.sampled_from((math.inf, -math.inf))).map( + lambda value: struct.pack(_PACKED[width], value).hex() + ) + any_bits = st.integers(0, 2**width - 1).map(lambda value: _bits(value, width)) + return st.one_of(floats, _nan_digits(width), any_bits) -def _float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + +def float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: """A number, an integer the type holds exactly, a named non-finite value, or the bits in hex. A float's fill value is any of these @@ -118,19 +169,28 @@ def _float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: `float16` takes `1e300` by the letter of the spec, and a reader is free to make of it what it will. """ - # Every integer of magnitude up to 2**11 is a float16, the narrowest type. - numbers = finite_floats(width) | st.integers(-(2**11), 2**11) - named = st.sampled_from(get_args(FloatSpecialFillValue)) - bits = st.integers(0, 2**width - 1).flatmap( - lambda value: _hex_spellings(f"{value:0{width // 4}x}") + exact = _EXACT_INTEGERS[width] + numbers = st.floats(width=width, allow_nan=False, allow_infinity=False) | st.integers( + -exact, exact ) - return st.one_of(numbers, named, bits) + named = st.sampled_from(get_args(FloatSpecialFillValue)) + return st.one_of(numbers, named, _float_digits(width).flatmap(_hex_spellings)) + + +def is_zero_float(value: JSONValue) -> bool: + """Whether `value`, a float fill value, is zero of either sign, as a number or as the hex string of its bits.""" + if isinstance(value, str): + digits = value.removeprefix("0x") + # A hex string of zero has no bit set but perhaps the sign, the first. + return digits != value and int(digits, 16) in (0, 8 * 16 ** (len(digits) - 1)) + return value == 0 def _fill_values_of(values: st.SearchStrategy[JSONValue]) -> DataTypeStrategy: """A data type with nothing to configure, whose fill values are `values`.""" return DataTypeStrategy( - configurations=EMPTY_CONFIGURATIONS, fill_values=lambda _configuration: values + configurations=lambda _strategies: EMPTY_CONFIGURATIONS, + fill_values=lambda _configuration, _strategies: values, ) @@ -141,7 +201,9 @@ def _byte_values(size: int) -> st.SearchStrategy[JSONValue]: ) -def _raw_bytes_fill_values(configuration: Mapping[str, JSONValue]) -> st.SearchStrategy[JSONValue]: +def _raw_bytes_fill_values( + configuration: Mapping[str, JSONValue], _strategies: Strategies +) -> st.SearchStrategy[JSONValue]: """One byte value for each 8 bits of the size.""" return _byte_values(cast("RawBytesConfiguration", configuration)["bits"] // 8) @@ -152,7 +214,7 @@ def _base64(value: bytes) -> JSONValue: def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: """A pair of float fill values of `width` bits, real then imaginary.""" - component = _float_fill_values(width) + component = float_fill_values(width) return st.tuples(component, component).map(lambda pair: cast("JSONValue", pair)) @@ -161,43 +223,115 @@ def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: ).map(_base64) """A `bytes` fill value: byte values, or the bytes in standard-alphabet base64.""" +_CHARACTERS: Final = st.characters(exclude_categories=("Cs",)) +"""Characters a JSON document can hold: no lone surrogate, which is no Unicode scalar value.""" + +_TEXT: Final = st.text(_CHARACTERS, max_size=16) + _LENGTH_BYTES: Final = st.integers(1, _MAX_BYTES) """The `length_bytes` of zarr-python's fixed-length byte types.""" def _base64_fill_values( - configuration: Mapping[str, JSONValue], + configuration: Mapping[str, JSONValue], _strategies: Strategies ) -> st.SearchStrategy[JSONValue]: """At most `length_bytes` bytes in standard-alphabet base64, as zarr-python's fixed-length byte types take them.""" length_bytes = cast("ZarrPythonRawBytesConfiguration", configuration)["length_bytes"] return st.binary(max_size=length_bytes).map(_base64) -_TEXT: Final = st.text(st.characters(exclude_categories=("Cs",)), max_size=16) -"""Text a JSON document can hold: no lone surrogate, which is no Unicode scalar value.""" - _MAX_CODE_POINTS: Final = 16 """The most code points a drawn `fixed_length_utf32` value holds.""" def _fixed_length_utf32_fill_values( - configuration: Mapping[str, JSONValue], + configuration: Mapping[str, JSONValue], _strategies: Strategies ) -> st.SearchStrategy[JSONValue]: """Text of at most one code point for each 4 bytes of the size, some of them trailing `U+0000` padding.""" capacity = cast("FixedLengthUtf32Configuration", configuration)["length_bytes"] // 4 - return st.text(st.characters(exclude_categories=("Cs",)) | st.just("\x00"), max_size=capacity) + return st.text(_CHARACTERS | st.just("\x00"), max_size=capacity) -_NUMPY_TIME: Final = DataTypeStrategy( - configurations=st.fixed_dictionaries( - { - "unit": st.sampled_from(NUMPY_TIME_UNIT), - "scale_factor": st.integers(1, NUMPY_TIME_MAX_SCALE_FACTOR), - } +_TICKS: Final = st.integers(-(2**63), 2**63 - 1) | st.just("NaT") +"""A numpy time's fill value: a signed 64-bit count of ticks, or `"NaT"`.""" + +_UNITS: Final = st.sampled_from(NUMPY_TIME_UNIT) +_SCALE_FACTORS: Final = st.integers(1, NUMPY_TIME_MAX_SCALE_FACTOR) + +_NUMPY_DATETIME64: Final = DataTypeStrategy( + configurations=lambda _strategies: st.tuples(_UNITS, _SCALE_FACTORS).map( + lambda pair: NumpyDatetime64Configuration(unit=pair[0], scale_factor=pair[1]) + ), + fill_values=lambda _configuration, _strategies: _TICKS, +) +"""A unit, and how many of it one tick is.""" + +_NUMPY_TIMEDELTA64: Final = DataTypeStrategy( + configurations=lambda _strategies: st.tuples(_UNITS, _SCALE_FACTORS).map( + lambda pair: NumpyTimedelta64Configuration(unit=pair[0], scale_factor=pair[1]) ), - fill_values=lambda _configuration: st.integers(-(2**63), 2**63 - 1) | st.just("NaT"), + fill_values=lambda _configuration, _strategies: _TICKS, ) -"""Both numpy time types: a unit, and how many of it one tick is; a signed 64-bit count of ticks, or `"NaT"`.""" +"""A unit, and how many of it one tick is.""" + + +def _fixed_size(strategies: Strategies) -> st.SearchStrategy[ZarrV3DeclarationJSON]: + """A data type declaration whose values are of a fixed size, which a struct's field types must be. + + "Variable-length data types (e.g. "string") MUST NOT be used as field + types" + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md?plain=1#L42-L49). + """ + return data_type_declarations(data_type_names(strategies), strategies).filter( + lambda declaration: ( + storage_of(read(declaration, DataTypeDefinition, strategies)) != "variable_length" + ) + ) + + +def _struct(strategies: Strategies, *, depth: int) -> st.SearchStrategy[StructConfiguration]: + """Fields with distinct names, none empty, each of a data type of fixed size; structs among them `depth` deep at most. + + The field types are drawn from the same strategies but for a struct, + which holds one level fewer, or is not drawn: structs drawn without + end would nest without end. So is any definition named as a struct, + which would otherwise claim the name in the scope the fields are read in. + """ + inner = {d: s for d, s in strategies.items() if d.name != STRUCT_DATA_TYPE.name} + if depth > 0: + inner[STRUCT_DATA_TYPE] = _structs(depth - 1) + # The type is drawn before the name: Hypothesis draws the first of many + # options far more often when a draw of variable length comes before it. + member = st.tuples( + _fixed_size(briefly(inner)), st.text(_CHARACTERS, min_size=1, max_size=16) + ).map(lambda pair: StructField(name=pair[1], data_type=pair[0])) + members = st.lists( + member, min_size=1, max_size=_MAX_STRUCT_FIELDS, unique_by=lambda entry: entry["name"] + ) + return members.map(lambda entries: StructConfiguration(fields=tuple(entries))) + + +def _struct_fill_values( + configuration: Mapping[str, JSONValue], strategies: Strategies +) -> st.SearchStrategy[JSONValue]: + """A fill value for each field, by its name, drawn by the field's own data type.""" + members = cast("StructConfiguration", configuration)["fields"] + return st.fixed_dictionaries( + { + member["name"]: fill_values_of( + read(member["data_type"], DataTypeDefinition, strategies), strategies + ) + for member in members + } + ).map(lambda value: cast("JSONValue", value)) + + +def _structs(depth: int) -> DataTypeStrategy: + """Structs that hold structs `depth` deep at most.""" + return DataTypeStrategy( + configurations=partial(_struct, depth=depth), fill_values=_struct_fill_values + ) + DATA_TYPE_STRATEGIES: Final[Mapping[DataTypeDefinition[Any], DataTypeStrategy]] = { BOOL_DATA_TYPE: _fill_values_of(st.booleans()), @@ -206,7 +340,7 @@ def _fixed_length_utf32_fill_values( for definition, (low, high) in INTEGER_RANGES.items() }, **{ - definition: _fill_values_of(_float_fill_values(width)) + definition: _fill_values_of(float_fill_values(width)) for definition, width in FLOAT_WIDTHS.items() }, **{ @@ -214,7 +348,7 @@ def _fixed_length_utf32_fill_values( for definition, width in COMPLEX_WIDTHS.items() }, RAW_BYTES_DATA_TYPE: DataTypeStrategy( - configurations=st.integers(1, _MAX_RAW_BYTES).map( + configurations=lambda _strategies: st.integers(1, _MAX_RAW_BYTES).map( lambda size: RawBytesConfiguration(bits=8 * size) ), fill_values=_raw_bytes_fill_values, @@ -222,28 +356,29 @@ def _fixed_length_utf32_fill_values( BYTES_DATA_TYPE: _fill_values_of(_BYTES_FILL_VALUES), STRING_DATA_TYPE: _fill_values_of(_TEXT), NULL_TERMINATED_BYTES_DATA_TYPE: DataTypeStrategy( - configurations=_LENGTH_BYTES.map( + configurations=lambda _strategies: _LENGTH_BYTES.map( lambda length_bytes: NullTerminatedBytesConfiguration(length_bytes=length_bytes) ), fill_values=_base64_fill_values, ), VARIABLE_LENGTH_BYTES_DATA_TYPE: _fill_values_of(st.binary(max_size=_MAX_BYTES).map(_base64)), ZARR_PYTHON_RAW_BYTES_DATA_TYPE: DataTypeStrategy( - configurations=_LENGTH_BYTES.map( + configurations=lambda _strategies: _LENGTH_BYTES.map( lambda length_bytes: ZarrPythonRawBytesConfiguration(length_bytes=length_bytes) ), fill_values=_base64_fill_values, ), FIXED_LENGTH_UTF32_DATA_TYPE: DataTypeStrategy( - configurations=st.integers(1, _MAX_CODE_POINTS).map( + configurations=lambda _strategies: st.integers(1, _MAX_CODE_POINTS).map( lambda count: FixedLengthUtf32Configuration(length_bytes=4 * count) ), fill_values=_fixed_length_utf32_fill_values, ), - NUMPY_DATETIME64_DATA_TYPE: _NUMPY_TIME, - NUMPY_TIMEDELTA64_DATA_TYPE: _NUMPY_TIME, + NUMPY_DATETIME64_DATA_TYPE: _NUMPY_DATETIME64, + NUMPY_TIMEDELTA64_DATA_TYPE: _NUMPY_TIMEDELTA64, + STRUCT_DATA_TYPE: _structs(_MAX_STRUCT_DEPTH), } -"""A strategy for each data type that holds no other declaration.""" +"""A strategy for each data type.""" __all__ = [ "COMPLEX_WIDTHS", @@ -251,6 +386,6 @@ def _fixed_length_utf32_fill_values( "FLOAT_WIDTHS", "INTEGER_RANGES", "FloatWidth", - "finite_floats", - "float_spellings", + "float_fill_values", + "is_zero_float", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py new file mode 100644 index 0000000000..6c8c2094d2 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py @@ -0,0 +1,319 @@ +"""Drawing declarations from `Strategies`: alone, inside a configuration, and as a codec pipeline. + +What a configuration that holds other declarations draws them with, and +what the public strategies are built from: a declaration in an envelope +the spec allows, a data type declaration of any definition the +strategies hold, and a pipeline whose each codec is drawn for the chunk +the one before it hands on. Each drawn declaration is read in the scope +of the strategies' own definitions, so a nested declaration reads as +what it was drawn for. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final, cast + +from hypothesis import reject +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import AcceptedDeclaration, resolve +from zarr_metadata.testing._strategy import NOTHING_KNOWN +from zarr_metadata.v3._context import ZarrV3Context +from zarr_metadata.v3._core import CORE_CONTEXT +from zarr_metadata.v3._definition import CodecDefinition, DataTypeDefinition, no_pipelines + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator, Sequence + + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.definition._definition import D, Definition, ResolvedDeclaration + from zarr_metadata.testing._strategy import ( + CodecStrategy, + Configurations, + DataTypeStrategy, + DefinitionStrategy, + Strategies, + ) + from zarr_metadata.v3 import ZarrV3DeclarationJSON + from zarr_metadata.v3._context import ZarrV3ContextSlots + from zarr_metadata.v3._definition import Chunk, CodecKind, CodecSize + +_MAX_ARRAY_ARRAY: Final = 2 +"""The most array -> array codecs a pipeline is drawn with.""" + +_MAX_BYTES_BYTES: Final = 2 +"""The most bytes -> bytes codecs a pipeline is drawn with.""" + +EVERY_SIZE: Final[frozenset[CodecSize]] = frozenset({"static", "dynamic"}) +"""Codecs of any size: what a pipeline but a shard's index takes.""" + + +class _Brief(Mapping["Definition[Any]", "DefinitionStrategy"]): + """`Strategies` whose repr says how many definitions they hold, not what each one is. + + Strategies are handed to the composite strategies that draw nested + declarations, and Hypothesis writes the repr of a strategy's arguments into + the note on any error raised while drawing from it -- among them the + rejections it raises itself, as `floats(width=16)` does -- warning + when that repr runs past 30,000 characters. Every definition's + strategy, in full, runs past that, which fails a test suite that + turns warnings into errors. It also holds the scope drawn declarations are + read in, built once rather than for every declaration read. + """ + + __slots__ = ("_strategies", "scope") + + def __init__(self, strategies: Strategies) -> None: + self._strategies = strategies + self.scope = _scope(strategies) + + def __getitem__(self, definition: Definition[Any]) -> DefinitionStrategy: + return self._strategies[definition] + + def __iter__(self) -> Iterator[Definition[Any]]: + return iter(self._strategies) + + def __len__(self) -> int: + return len(self._strategies) + + def __repr__(self) -> str: + return f"" + + +def _slot_of(definition: Definition[Any]) -> str: + """The slot of a Zarr v3 scope that holds `definition`; a `TypeError` for a definition none holds, such as a v2 codec.""" + for slot, kind in ZarrV3Context.kinds.items(): + if isinstance(definition, kind): + return slot + msg = ( + f"{definition.name!r} is a {type(definition).__name__}, " + "which no slot of a Zarr v3 scope holds" + ) + raise TypeError(msg) + + +def _scope(definitions: Iterable[Definition[Any]]) -> ZarrV3Context: + """The scope that claims `definitions` and nothing else, each filed in the slot of its kind; of two with one name, the later.""" + slots: dict[str, dict[str, Definition[Any]]] = {} + for definition in definitions: + slots.setdefault(_slot_of(definition), {})[definition.name] = definition + return ZarrV3Context(**cast("ZarrV3ContextSlots", slots)) + + +def briefly(strategies: Strategies) -> _Brief: + """`strategies`, with a repr short enough to go in Hypothesis's notes.""" + return strategies if isinstance(strategies, _Brief) else _Brief(strategies) + + +def strategy_for(definition: Definition[Any], strategies: Strategies) -> DefinitionStrategy: + """The strategy `strategies` holds for `definition`, which draws configurations of its kind.""" + found = strategies.get(definition) + if found is None: + msg = f"no strategy for {definition.name!r}; pass one in `strategies`" + raise LookupError(msg) + if not isinstance(definition, found.kind): + msg = ( + f"the strategy for {definition.name!r}, a {type(definition).__name__}, draws " + f"configurations of a {found.kind.__name__}: {found!r}" + ) + raise TypeError(msg) + return found + + +def scope_of(strategies: Strategies) -> ZarrV3Context: + """The scope a drawn declaration is read in: every definition the strategies draw for.""" + return briefly(strategies).scope + + +def scope_with(definition: Definition[Any], strategies: Strategies) -> ZarrV3Context: + """The scope a declaration of `definition` is read in: the strategies', with `definition` claiming its name whether or not they hold it.""" + return _scope((*strategies, definition)) + + +def read(declaration: object, kind: type[D], strategies: Strategies) -> ResolvedDeclaration[D]: + """`declaration`, drawn from `strategies` or held by a configuration handed to one, read in their scope; a `ValueError` if it is refused. + + A strategy draws what its definition accepts, so a refused declaration is a + fault in a strategy or in the configuration it was handed, not + something to draw again. + """ + resolved, problems = resolve(declaration, kind, scope_of(strategies)) + if len(problems) != 0: + msg = f"{declaration!r}, read in the strategies' scope, is refused: {problems!r}" + raise ValueError(msg) + return resolved + + +def _written_bare(definition: Definition[Any]) -> bool: + """Whether the spec has the declaration written as its bare name: a data type it defines itself. + + "If the data type is defined in this specification, then the value + must be the data type identifier provided as a string" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L509-L512). + """ + if not isinstance(definition, DataTypeDefinition): + return False + return CORE_CONTEXT.claimant(DataTypeDefinition, definition.name) is definition + + +@st.composite +def any_spelling( + draw: st.DrawFn, definition: Definition[Any], configurations: SearchStrategy[Any] +) -> ZarrV3DeclarationJSON: + """A configuration of `definition` in one of the envelopes the spec allows it. + + A configuration with nothing in it is written as the bare name, an + object without one, or an object with an empty one; any object may + say `must_understand` is `true`, which is what absence means. A name + that carries the configuration, as `r16` does, and a data type the + spec defines itself, are written as the bare name. + """ + configuration = cast("Mapping[str, JSONValue]", draw(configurations)) + carried = definition._carrying_name(configuration) # pyright: ignore[reportPrivateUsage] + if carried is not None: + return carried + if _written_bare(definition): + return definition.name + empty = len(configuration) == 0 + if empty and draw(st.booleans()): + return definition.name + declaration: dict[str, JSONValue] = {"name": definition.name} + if not empty or draw(st.booleans()): + declaration["configuration"] = configuration + if draw(st.booleans()): + declaration["must_understand"] = True + return cast("ZarrV3DeclarationJSON", declaration) + + +def data_type_names(strategies: Strategies) -> list[str]: + """The names of the data types `strategies` draw for, in order. + + Names, not definitions, are what the strategies built from them hold: + Hypothesis prints a definition with every member it has, so a list of + them would make the strategy's repr as long as the scope is wide. + """ + return sorted(scope_of(strategies).data_types) + + +@st.composite +def data_type_declarations( + draw: st.DrawFn, among: Sequence[str], strategies: Strategies +) -> ZarrV3DeclarationJSON: + """A data type declaration, of one of the definitions named `among`, drawn from `strategies`.""" + definition = scope_of(strategies).data_types[draw(st.sampled_from(among))] + found = cast("DataTypeStrategy", strategy_for(definition, strategies)) + return draw(any_spelling(definition, found.configurations(strategies))) + + +def fill_values_of( + data_type: ResolvedDeclaration[DataTypeDefinition[Any]], strategies: Strategies +) -> SearchStrategy[JSONValue]: + """Fill values of `data_type`, a data type declaration read in the strategies' scope; a `LookupError` if no definition in it read the declaration.""" + if isinstance(data_type, AcceptedDeclaration): + found = cast("DataTypeStrategy", strategy_for(data_type.definition, strategies)) + return found.fill_values(data_type.configuration, strategies) + msg = f"no strategy draws fill values of {data_type.json!r}, which no definition in `strategies` reads" + raise LookupError(msg) + + +def _taking( + chunk: Chunk, + kind: CodecKind, + sizes: frozenset[CodecSize], + strategies: Strategies, + *, + holding: bool = True, +) -> list[tuple[CodecDefinition[Any], Configurations]]: + """Each codec of `kind` and one of `sizes` that `strategies` draw for, that takes `chunk`, with its configurations. + + Without `holding`, a codec that holds pipelines of its own is left + out: what one of those takes depends on what its pipelines do. + """ + taking: list[tuple[CodecDefinition[Any], Configurations]] = [] + for definition in sorted(scope_of(strategies).codecs.values(), key=lambda d: d.name): + if definition.kind != kind or definition.size not in sizes: + continue + if not holding and _holds_pipelines(definition): + continue + found = cast("CodecStrategy", strategy_for(definition, strategies)) + configurations = found.configurations(chunk, strategies) + if configurations is not None: + taking.append((definition, configurations)) + return taking + + +def _holds_pipelines(definition: CodecDefinition[Any]) -> bool: + return definition.pipelines is not no_pipelines + + +def pipelines_for( + chunk: Chunk, strategies: Strategies, sizes: frozenset[CodecSize] = EVERY_SIZE +) -> SearchStrategy[tuple[ZarrV3DeclarationJSON, ...]] | None: + """Pipelines handed `chunk`, of codecs of one of `sizes`; None when no array -> bytes codec that holds no pipelines takes the chunk as it is. + + One that does is enough for a pipeline: one of only that codec. One + that holds pipelines of its own takes a chunk only if what it holds + does, so it settles nothing. + """ + if len(_taking(chunk, "array_bytes", sizes, strategies, holding=False)) == 0: + return None + return _walk(chunk, strategies, sizes) + + +@st.composite +def _codec( + draw: st.DrawFn, options: Sequence[tuple[CodecDefinition[Any], Configurations]] +) -> tuple[CodecDefinition[Any], ZarrV3DeclarationJSON]: + definition, configurations = draw(st.sampled_from(options)) + return definition, draw(any_spelling(definition, configurations)) + + +@st.composite +def _walk( + draw: st.DrawFn, chunk: Chunk, strategies: Strategies, sizes: frozenset[CodecSize] +) -> tuple[ZarrV3DeclarationJSON, ...]: + """Codecs for a pipeline handed `chunk`, each drawn for the chunk the one before it hands on. + + Array -> array codecs, then one array -> bytes codec, then bytes -> + bytes codecs, each of one of `sizes`. What an array -> array codec + hands on is what its definition's `transition` says of the codec as + drawn: a `cast_value` hands the next codec the data type it casts to. + """ + codecs: list[ZarrV3DeclarationJSON] = [] + handed = chunk + for _ in range(draw(st.integers(0, _MAX_ARRAY_ARRAY))): + options = _taking(handed, "array_array", sizes, strategies) + if len(options) == 0: + break + definition, declaration = draw(_codec(options)) + codec = read(declaration, CodecDefinition, strategies) + assert isinstance(codec, AcceptedDeclaration) + handed = definition.transition(codec.configuration, codec.nested, handed) + codecs.append(declaration) + options = _taking(handed, "array_bytes", sizes, strategies) + if len(options) == 0: + reject() # what the array -> array codecs handed on, no array -> bytes codec takes + codecs.append(draw(_codec(options))[1]) + options = _taking(NOTHING_KNOWN, "bytes_bytes", sizes, strategies) + if len(options) != 0: + count = draw(st.integers(0, _MAX_BYTES_BYTES)) + codecs.extend(draw(_codec(options))[1] for _ in range(count)) + return tuple(codecs) + + +__all__ = [ + "EVERY_SIZE", + "any_spelling", + "briefly", + "data_type_declarations", + "data_type_names", + "fill_values_of", + "pipelines_for", + "read", + "scope_of", + "scope_with", + "strategy_for", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py index 6c526a042d..8f0528f368 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -4,6 +4,10 @@ each kind of definition does: a data type's fill values depend on its configuration, a chunk grid's on the shape of the array, a codec's on the chunk it is handed. A chunk key encoding's depend on nothing. + +Every function a record holds is also handed `Strategies`, every +definition's strategy: a configuration that holds other declarations -- +a struct's field types, a shard's codecs -- draws them from it. """ from __future__ import annotations @@ -11,7 +15,7 @@ from abc import ABC, abstractmethod from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, cast +from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias from hypothesis import strategies as st @@ -52,7 +56,7 @@ class DefinitionStrategy(ABC): """The kind of definition whose configurations it draws.""" @abstractmethod - def unconstrained(self) -> Configurations: + def unconstrained(self, strategies: Strategies) -> Configurations: """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" @@ -63,14 +67,14 @@ class DataTypeStrategy(DefinitionStrategy): kind: ClassVar[type[Definition[Any]]] = DataTypeDefinition """The kind of definition it draws configurations of: a data type.""" - configurations: Configurations + configurations: Callable[[Strategies], Configurations] """Configurations its definition accepts.""" - fill_values: Callable[[Mapping[str, JSONValue]], SearchStrategy[JSONValue]] + fill_values: Callable[[Mapping[str, JSONValue], Strategies], SearchStrategy[JSONValue]] """Fill values of the data type a configuration describes.""" - def unconstrained(self) -> Configurations: + def unconstrained(self, strategies: Strategies) -> Configurations: """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" - return self.configurations + return self.configurations(strategies) @dataclass(frozen=True, kw_only=True, slots=True) @@ -80,16 +84,16 @@ class ChunkGridStrategy(DefinitionStrategy): kind: ClassVar[type[Definition[Any]]] = ChunkGridDefinition """The kind of definition it draws configurations of: a chunk grid.""" - configurations: Callable[[tuple[int, ...] | None], Configurations] + configurations: Callable[[tuple[int, ...] | None, Strategies], Configurations] """Configurations of grids that fit an array of the shape. Handed None, no shape is known, and what is drawn is what the grid's rules accept, whatever shape it is put over. """ - def unconstrained(self) -> Configurations: + def unconstrained(self, strategies: Strategies) -> Configurations: """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" - return self.configurations(None) + return self.configurations(None, strategies) @dataclass(frozen=True, kw_only=True, slots=True) @@ -99,12 +103,12 @@ class ChunkKeyEncodingStrategy(DefinitionStrategy): kind: ClassVar[type[Definition[Any]]] = ChunkKeyEncodingDefinition """The kind of definition it draws configurations of: a chunk key encoding.""" - configurations: Configurations + configurations: Callable[[Strategies], Configurations] """Configurations its definition accepts.""" - def unconstrained(self) -> Configurations: + def unconstrained(self, strategies: Strategies) -> Configurations: """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" - return self.configurations + return self.configurations(strategies) @dataclass(frozen=True, kw_only=True, slots=True) @@ -112,27 +116,26 @@ class CodecStrategy(DefinitionStrategy): """How to draw a codec's configurations, handed a chunk. Every codec takes `Chunk()`, a chunk nothing is known of, so a - strategy that draws nothing for it is refused when it is built. + strategy that draws nothing for it is refused when it is asked to. """ kind: ClassVar[type[Definition[Any]]] = CodecDefinition """The kind of definition it draws configurations of: a codec.""" - configurations: Callable[[Chunk], Configurations | None] + configurations: Callable[[Chunk, Strategies], Configurations | None] """Configurations of the codec handed the chunk; None when no configuration takes it. A `bytes` codec handed a `string` chunk is None, since the spec gives that data type a codec of its own. """ - def __post_init__(self) -> None: - if self.configurations(NOTHING_KNOWN) is None: + def unconstrained(self, strategies: Strategies) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + found = self.configurations(NOTHING_KNOWN, strategies) + if found is None: msg = "a codec strategy draws configurations for Chunk(), a chunk nothing is known of; got None" raise ValueError(msg) - - def unconstrained(self) -> Configurations: - """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" - return cast("Configurations", self.configurations(NOTHING_KNOWN)) + return found Strategies: TypeAlias = Mapping[Any, DefinitionStrategy] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py index 683959c4ea..213bef3305 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -21,13 +21,19 @@ def test_my_reader_reads_every_blosc_codec(declaration): the shape of the array, `codec_configurations` the `Chunk` the codec is handed. Left out, nothing is known of it, and what is drawn is what the definition's rules accept. `declarations` wraps configurations in the -envelopes a document writes them in. +envelopes a document writes them in. `pipelines` draws a whole codec +pipeline for a chunk, each codec drawn for what the one before it hands +on. How each definition's configurations are drawn is a value, `Strategies`, keyed by the definition: `DEFAULT_STRATEGIES` holds one -for each definition this package defines that holds no other declaration. An -extension author adds their own with -`DEFAULT_STRATEGIES | {MY_CODEC: CodecStrategy(...)}`. +for each definition this package defines. A configuration that holds +other declarations -- a struct's field types, the data type a +`cast_value` casts to, a shard's codecs -- draws them from the same +value, so they are drawn from every definition it holds, and read in a +scope of those definitions. An extension author adds their own with +`DEFAULT_STRATEGIES | {MY_CODEC: CodecStrategy(...)}`; one under a name +the defaults hold replaces theirs, as the later of two with one name. Configurations are drawn at the data level the TypedDicts declare, JSON arrays as tuples; `json.dumps` takes them as they are. @@ -41,14 +47,19 @@ def test_my_reader_reads_every_blosc_codec(declaration): from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast -from hypothesis import strategies as st - from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._definition import Definition, as_kind, canonicalize, kind_of from zarr_metadata.testing._chunk_grid import CHUNK_GRID_STRATEGIES from zarr_metadata.testing._chunk_key_encoding import CHUNK_KEY_ENCODING_STRATEGIES from zarr_metadata.testing._codec import CODEC_STRATEGIES from zarr_metadata.testing._data_type import DATA_TYPE_STRATEGIES +from zarr_metadata.testing._draw import ( + any_spelling, + briefly, + pipelines_for, + scope_with, + strategy_for, +) from zarr_metadata.testing._strategy import ( NOTHING_KNOWN, ChunkGridStrategy, @@ -59,9 +70,6 @@ def test_my_reader_reads_every_blosc_codec(declaration): DefinitionStrategy, Strategies, ) -from zarr_metadata.v3._context import ZarrV3Context -from zarr_metadata.v3._core import CORE_CONTEXT -from zarr_metadata.v3._definition import Chunk, DataTypeDefinition if TYPE_CHECKING: from collections.abc import Mapping @@ -70,11 +78,13 @@ def test_my_reader_reads_every_blosc_codec(declaration): from zarr_metadata._common import JSONValue from zarr_metadata.v3 import ZarrV3DeclarationJSON - from zarr_metadata.v3._context import ZarrV3ContextSlots + from zarr_metadata.v3._context import ZarrV3Context from zarr_metadata.v3._definition import ( + Chunk, ChunkGridDefinition, ChunkKeyEncodingDefinition, CodecDefinition, + DataTypeDefinition, ) C = TypeVar("C") @@ -98,30 +108,16 @@ def test_my_reader_reads_every_blosc_codec(declaration): **CODEC_STRATEGIES, } ) -"""A strategy for each definition this package defines that holds no other declaration.""" - - -def _strategy_for(definition: Definition[Any], strategies: Strategies) -> DefinitionStrategy: - """The strategy `strategies` holds for `definition`, which draws configurations of its kind.""" - found = strategies.get(definition) - if found is None: - msg = f"no strategy for {definition.name!r}; pass one in `strategies`" - raise LookupError(msg) - if not isinstance(definition, found.kind): - msg = ( - f"the strategy for {definition.name!r}, a {type(definition).__name__}, draws " - f"configurations of a {found.kind.__name__}: {found!r}" - ) - raise TypeError(msg) - return found +"""A strategy for each definition this package defines.""" def data_type_configurations( definition: DataTypeDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES ) -> SearchStrategy[C]: """Configurations of the data type `definition` defines.""" - found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) - return cast("SearchStrategy[C]", found.configurations) + strategies = briefly(strategies) + found = cast("DataTypeStrategy", strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations(strategies)) def fill_values( @@ -131,8 +127,9 @@ def fill_values( strategies: Strategies = DEFAULT_STRATEGIES, ) -> SearchStrategy[JSONValue]: """Fill values of the data type `definition` and `configuration` describe, in every spelling the spec gives them.""" - found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) - return found.fill_values(cast("Mapping[str, JSONValue]", configuration)) + strategies = briefly(strategies) + found = cast("DataTypeStrategy", strategy_for(definition, strategies)) + return found.fill_values(cast("Mapping[str, JSONValue]", configuration), strategies) def chunk_grid_configurations( @@ -145,16 +142,18 @@ def chunk_grid_configurations( With no shape, what is drawn is what the grid's rules accept. """ - found = cast("ChunkGridStrategy", _strategy_for(definition, strategies)) - return cast("SearchStrategy[C]", found.configurations(shape)) + strategies = briefly(strategies) + found = cast("ChunkGridStrategy", strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations(shape, strategies)) def chunk_key_encoding_configurations( definition: ChunkKeyEncodingDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES ) -> SearchStrategy[C]: """Configurations of the chunk key encoding `definition` defines.""" - found = cast("ChunkKeyEncodingStrategy", _strategy_for(definition, strategies)) - return cast("SearchStrategy[C]", found.configurations) + strategies = briefly(strategies) + found = cast("ChunkKeyEncodingStrategy", strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations(strategies)) def codec_configurations( @@ -168,61 +167,27 @@ def codec_configurations( With no chunk, nothing is known of it, and what is drawn is what the codec's rules accept. """ - found = cast("CodecStrategy", _strategy_for(definition, strategies)) - return cast("SearchStrategy[C] | None", found.configurations(chunk)) - - -def _written_bare(definition: Definition[Any]) -> bool: - """Whether the spec has the declaration written as its bare name: a data type it defines itself. - - "If the data type is defined in this specification, then the value - must be the data type identifier provided as a string" - (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L509-L512). + strategies = briefly(strategies) + found = cast("CodecStrategy", strategy_for(definition, strategies)) + return cast("SearchStrategy[C] | None", found.configurations(chunk, strategies)) + + +def pipelines( + chunk: Chunk = NOTHING_KNOWN, *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[tuple[ZarrV3DeclarationJSON, ...]] | None: + """Codec pipelines handed `chunk`, as an array's `codecs` holds them; None when none is drawn for it. + + Array -> array codecs, then one array -> bytes codec, then bytes -> + bytes codecs, each drawn for the chunk the codec before it hands on, + as its definition's `transition` says: a `transpose` hands on the + chunk's axes permuted, a `cast_value` the data type it casts to. A + codec that holds pipelines of its own, as a shard does, has each drawn + the same way, handed the chunk its definition's `pipelines` says. + None when no array -> bytes codec takes the chunk, as none takes a + `string` one when the strategies hold no `vlen-utf8`. """ - if not isinstance(definition, DataTypeDefinition): - return False - return CORE_CONTEXT.claimant(DataTypeDefinition, definition.name) is definition - - -@st.composite -def _any_spelling( - draw: st.DrawFn, definition: Definition[Any], configurations: SearchStrategy[Any] -) -> ZarrV3DeclarationJSON: - """A configuration of `definition` in one of the envelopes the spec allows it. - - A configuration with nothing in it is written as the bare name, an - object without one, or an object with an empty one; any object may - say `must_understand` is `true`, which is what absence means. A name - that carries the configuration, as `r16` does, and a data type the - spec defines itself, are written as the bare name. - """ - configuration = cast("Mapping[str, JSONValue]", draw(configurations)) - carried = definition._carrying_name(configuration) # pyright: ignore[reportPrivateUsage] - if carried is not None: - return carried - if _written_bare(definition): - return definition.name - empty = len(configuration) == 0 - if empty and draw(st.booleans()): - return definition.name - declaration: dict[str, JSONValue] = {"name": definition.name} - if not empty or draw(st.booleans()): - declaration["configuration"] = configuration - if draw(st.booleans()): - declaration["must_understand"] = True - return cast("ZarrV3DeclarationJSON", declaration) - - -def _scope_of(definition: Definition[Any]) -> ZarrV3Context: - """The scope that claims `definition` and nothing else; a `TypeError` for a definition no slot of a Zarr v3 scope holds.""" - for slot, kind in ZarrV3Context.kinds.items(): - if isinstance(definition, kind): - return ZarrV3Context(**cast("ZarrV3ContextSlots", {slot: (definition,)})) - msg = ( - f"{definition.name!r} is a {type(definition).__name__}, " - "which no slot of a Zarr v3 scope holds" - ) - raise TypeError(msg) + strategies = briefly(strategies) + return pipelines_for(chunk, strategies) def _checked( @@ -258,14 +223,16 @@ def declarations( once it is known not to be None. `spellings` says whether every spelling the spec allows is drawn, or only the simplest. - Each declaration is read in a scope of its definition alone, with no chunk - known, and one its definition's rules refuse, as they refuse one - drawn from `configurations` they do not accept, raises a - `ValueError`. A codec's chunk rules are not checked: what fits the - chunk is what `codec_configurations` draws. + Each declaration is read in the strategies' scope, with `definition` + in it whether or not they hold it, and with no chunk known; one its + definition's rules refuse, as they refuse one drawn from + `configurations` they do not accept, raises a `ValueError`. A + codec's chunk rules are not checked: what fits the chunk is what + `codec_configurations` draws. """ + strategies = briefly(strategies) drawn: Configurations | SearchStrategy[C] = ( - _strategy_for(definition, strategies).unconstrained() + strategy_for(definition, strategies).unconstrained(strategies) if configurations is UNSET else configurations ) @@ -273,10 +240,10 @@ def declarations( _checked, definition=definition, kind=as_kind(kind_of(definition)), - scope=_scope_of(definition), + scope=scope_with(definition, strategies), spellings=spellings, ) - return _any_spelling(definition, drawn).map(checked) + return any_spelling(definition, drawn).map(checked) __all__ = [ @@ -296,4 +263,5 @@ def declarations( "data_type_configurations", "declarations", "fill_values", + "pipelines", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py index 9c7c8f1874..cd31374272 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/_arithmetic.py @@ -25,8 +25,12 @@ from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE_NAME from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.fixed_length_utf32 import FIXED_LENGTH_UTF32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.null_terminated_bytes import NULL_TERMINATED_BYTES_DATA_TYPE_NAME from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.variable_length_bytes import VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.zarr_python_raw_bytes import ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME FLOATING_POINT: Final = frozenset( { @@ -55,6 +59,10 @@ STRING_DATA_TYPE_NAME, BYTES_DATA_TYPE_NAME, STRUCT_DATA_TYPE_NAME, + FIXED_LENGTH_UTF32_DATA_TYPE_NAME, + NULL_TERMINATED_BYTES_DATA_TYPE_NAME, + VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME, + ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME, } ) """Data types whose values are not numbers (truth values, raw bits, text, byte strings, records). Neither codec accepts them.""" @@ -68,4 +76,8 @@ """ -__all__ = ["COMPLEX", "FLOATING_POINT", "NOT_NUMBERS"] +NO_REAL_NUMBERS: Final = NOT_NUMBERS | COMPLEX +"""The data types that model no real number, which `cast_value` takes neither from nor to.""" + + +__all__ = ["COMPLEX", "FLOATING_POINT", "NOT_NUMBERS", "NO_REAL_NUMBERS"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 873a987f9a..114d31fddf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -21,7 +21,7 @@ Chunk, CodecDefinition, ) -from zarr_metadata.v3.codec._arithmetic import COMPLEX, FLOATING_POINT, NOT_NUMBERS +from zarr_metadata.v3.codec._arithmetic import FLOATING_POINT, NO_REAL_NUMBERS CAST_VALUE_CODEC_NAME: Final = "cast_value" """The `name` field value of the `cast_value` codec.""" @@ -145,10 +145,6 @@ class CastValueCodecObject(TypedDict, closed=True): """ -_NO_REAL_NUMBERS: Final = NOT_NUMBERS | COMPLEX -"""The data types the codec accepts neither as input nor as output, because none of them models real numbers.""" - - def _scalars( configuration: CastValueCodecConfiguration, side: Literal["source", "target"] ) -> Iterator[tuple[tuple[str | int, ...], JSONValue]]: @@ -190,7 +186,7 @@ def _rules( if not isinstance(target, AcceptedDeclaration): return name, written = target.definition.name, target.name - if name in _NO_REAL_NUMBERS: + if name in NO_REAL_NUMBERS: yield ValidationProblem( ("data_type",), f"expected a data type that models real numbers, got {shown(written)}", @@ -221,7 +217,7 @@ def _chunk_rules( source = chunk.data_type if not isinstance(source, AcceptedDeclaration): return - if source.definition.name in _NO_REAL_NUMBERS: + if source.definition.name in NO_REAL_NUMBERS: yield ValidationProblem( (), f"expected a chunk of a data type that models real numbers, got {shown(source.name)}", diff --git a/packages/zarr-metadata/tests/testing/_shares.py b/packages/zarr-metadata/tests/testing/_shares.py new file mode 100644 index 0000000000..27517e2653 --- /dev/null +++ b/packages/zarr-metadata/tests/testing/_shares.py @@ -0,0 +1,37 @@ +"""How often a strategy draws something, in a few hundred examples. + +A reach test asks whether a target can be drawn at all, in as many examples +as it takes, which says nothing of whether a test, at the 100 examples it +runs by default, ever meets it. Runs are derandomized, which keeps most shares +fixed, but a share can still move a few points between processes, so a bound +needs room: this is no place to bound what is drawn a few percent of the time. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, TypeVar + +from hypothesis import HealthCheck, given, settings + +if TYPE_CHECKING: + from collections.abc import Callable + + from hypothesis.strategies import SearchStrategy + +T = TypeVar("T") + + +def share_of( + strategy: SearchStrategy[T], counts: Callable[[T], bool], *, examples: int = 300 +) -> float: + """The share of the examples of `strategy` that `counts` holds of.""" + tally: list[bool] = [] + run = settings( + max_examples=examples, + database=None, + derandomize=True, + deadline=None, + suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], + )(given(strategy)(lambda value: tally.append(counts(value)))) + run() + return sum(tally) / len(tally) diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 0d28ba6632..42b859fbb9 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -3,7 +3,8 @@ Every draw is read in `ZARR_PYTHON_CONTEXT` and must read without a problem, by the definition it was drawn for: the declaration alone, a fill value beside its data type, a chunk grid over the shape it was drawn -for, a codec handed the chunk it was drawn for. A strategy that only +for, a codec handed the chunk it was drawn for, and a pipeline handed +the chunks of a grid, as `read_pipeline` reads one. A strategy that only drew valid metadata could draw almost none, so reach is checked too, with targets worked out from each configuration's TypedDict rather than from the strategies: every optional key present and absent, every @@ -14,12 +15,15 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, Any, Literal, cast, get_args, get_origin +import math +from collections import Counter +from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast, get_args, get_origin import pytest from hypothesis import event, find, given, settings from hypothesis import strategies as st +from tests.testing._shares import share_of from zarr_metadata._typed_json import typeddict_keys from zarr_metadata.definition import ( AcceptedDeclaration, @@ -29,20 +33,32 @@ resolve, ) from zarr_metadata.definition._definition import as_kind, canonicalize, kind_of, unchanged +from zarr_metadata.testing._draw import scope_with +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS from zarr_metadata.testing.strategies import ( DEFAULT_STRATEGIES, ChunkKeyEncodingStrategy, CodecStrategy, + DataTypeStrategy, Spellings, + Strategies, chunk_grid_configurations, codec_configurations, data_type_configurations, declarations, fill_values, + pipelines, ) from zarr_metadata.v2.definition import CORE_V2_CONTEXT -from zarr_metadata.v3._definition import chunk_grid_lengths +from zarr_metadata.v3._definition import ( + chunk_grid_lengths, + no_pipelines, + storage_of, + variable_length, +) +from zarr_metadata.v3._pipeline import read_pipeline from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC from zarr_metadata.v3.codec.gzip import GZIP_CODEC from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC @@ -54,6 +70,7 @@ from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE +from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE @@ -68,13 +85,11 @@ ) if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Callable, Mapping from zarr_metadata._common import JSONValue from zarr_metadata.v3 import ZarrV3DeclarationJSON - -_HOLDING_DECLARATIONS = frozenset({SHARDING_INDEXED_CODEC, CAST_VALUE_CODEC, STRUCT_DATA_TYPE}) -"""The definitions whose configurations hold other declarations, which have no strategy yet.""" + from zarr_metadata.v3._definition import Lengths _IN_SCOPE = frozenset( ( @@ -93,6 +108,8 @@ _CHUNK_GRIDS = [d for d in _DEFINITIONS if isinstance(d, ChunkGridDefinition)] _CODECS = [d for d in _DEFINITIONS if isinstance(d, CodecDefinition)] +T = TypeVar("T") + _SHAPES = st.lists(st.integers(0, 12), max_size=4).map(tuple) @@ -112,9 +129,8 @@ def _read(definition: Definition[Any], declaration: object) -> Any: # noqa: ANN def test_every_definition_has_a_strategy() -> None: - """Each definition in scope has a strategy, but those that hold other declarations; and none is out of scope.""" - assert _IN_SCOPE - frozenset(DEFAULT_STRATEGIES) == _HOLDING_DECLARATIONS - assert frozenset(DEFAULT_STRATEGIES) <= _IN_SCOPE + """Each definition in scope has a strategy, and none is out of scope.""" + assert frozenset(DEFAULT_STRATEGIES) == _IN_SCOPE @pytest.mark.parametrize("spellings", get_args(Spellings)) @@ -172,31 +188,163 @@ def test_chunk_grids_fit_their_shape( @st.composite def _chunks(draw: st.DrawFn) -> Chunk: - """A chunk of a data type drawn from the strategies, or of none known, with lengths known along some axes, all, or none.""" + """A chunk of a data type drawn from the strategies, or of none known, of lengths known along some axes, all, or none, or of a grid drawn over an array of some shape. + + A rectilinear grid's chunks take several lengths along an axis, which + a codec drawn for them must take each of. + """ + lengths: Lengths | None + if draw(st.booleans()): + shape = draw(_SHAPES) + grid = draw(st.sampled_from(_CHUNK_GRIDS)) + chunk_grid = _read(grid, draw(declarations(grid, chunk_grid_configurations(grid, shape)))) + lengths, _ = chunk_grid_lengths(chunk_grid, shape) + else: + axis = st.none() | st.integers(1, 8).map(lambda length: frozenset({length})) + lengths = draw(st.none() | st.lists(axis, max_size=4).map(tuple)) definition = draw(st.none() | st.sampled_from(_DATA_TYPES)) data_type = None if definition is None else _read(definition, draw(declarations(definition))) - axis = st.none() | st.integers(1, 8).map(lambda length: frozenset({length})) - lengths = draw(st.none() | st.lists(axis, max_size=4).map(tuple)) return Chunk(lengths, data_type) +def _handed_problems(codec: Any, chunk: Chunk) -> list[object]: # noqa: ANN401 + """What is wrong with `codec`, a codec field read, handed `chunk`: by its chunk rules, and in each pipeline it holds.""" + definition = codec.definition + problems: list[object] = list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) + handed = definition.pipelines(codec.configuration, codec.nested, chunk) + for member, inner in handed.items(): + held = [codec.nested[(member, index)] for index in range(len(codec.configuration[member]))] + problems.extend(read_pipeline(held, inner)[1]) + return problems + + +def _holds_pipelines(definition: CodecDefinition[Any]) -> bool: + return definition.pipelines is not no_pipelines + + +def _encoders_refuse(data: st.DataObject, chunk: Chunk) -> bool: + """Whether every array -> bytes codec that holds no pipelines refuses a chunk of `chunk`'s data type.""" + of_data_type = Chunk(None, chunk.data_type) + encoders = [d for d in _CODECS if d.kind == "array_bytes" and not _holds_pipelines(d)] + for encoder in encoders: + codec = _read(encoder, data.draw(declarations(encoder))) + if list(encoder.chunk_rules(codec.configuration, codec.nested, of_data_type)) == []: + return False + return True + + @pytest.mark.parametrize("definition", _CODECS, ids=_id) @given(data=st.data(), chunk=_chunks()) def test_codecs_take_their_chunk( definition: CodecDefinition[Any], data: st.DataObject, chunk: Chunk ) -> None: - """A codec drawn for a chunk takes it; one drawn for none is one the spec does not give it, which needs the data type known.""" + """A codec drawn for a chunk takes it; one drawn for none is one the spec does not give it, which needs the data type known. + + A shard drawn for none is one whose inner chunks no codec encodes. + """ configurations = codec_configurations(definition, chunk) if configurations is None: assert chunk.data_type is not None assert chunk.data_type.definition is not None data_type_name = chunk.data_type.definition.name event(f"takes no {data_type_name}") + if _holds_pipelines(definition): + assert _encoders_refuse(data, chunk) + return codec = _read(definition, data.draw(declarations(definition))) assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) != [] return codec = _read(definition, data.draw(declarations(definition, configurations))) - assert list(definition.chunk_rules(codec.configuration, codec.nested, chunk)) == [] + assert _handed_problems(codec, chunk) == [] + + +_MIXED_LENGTHS = st.lists( + st.frozensets(st.integers(1, 24), min_size=1, max_size=3), max_size=3 +).map(tuple) +"""Several lengths along each axis, as a rectilinear grid gives: one inner length must divide them all.""" + + +@given(data=st.data(), lengths=_MIXED_LENGTHS) +def test_shards_divide_every_length( + data: st.DataObject, lengths: tuple[frozenset[int], ...] +) -> None: + chunk = Chunk(lengths) + configurations = codec_configurations(SHARDING_INDEXED_CODEC, chunk) + assert configurations is not None + codec = _read( + SHARDING_INDEXED_CODEC, data.draw(declarations(SHARDING_INDEXED_CODEC, configurations)) + ) + assert _handed_problems(codec, chunk) == [] + + +def test_shards_divide_a_long_shard() -> None: + """A length of 2**30 has its divisors found without counting to it.""" + chunk = Chunk((frozenset({2**30}),)) + configurations = codec_configurations(SHARDING_INDEXED_CODEC, chunk) + assert configurations is not None + find( + configurations, + lambda configuration: configuration["chunk_shape"][0] > 2**15, + settings=_REACH, + ) + + +def test_a_shard_whose_index_nothing_encodes_is_none() -> None: + """Without a codec of static size to encode the index, no shard is drawn.""" + without_bytes = {d: s for d, s in DEFAULT_STRATEGIES.items() if d is not BYTES_CODEC} + assert codec_configurations(SHARDING_INDEXED_CODEC, strategies=without_bytes) is None + + +def test_a_cast_from_a_data_type_without_a_strategy_draws_numbers() -> None: + """Scalars of a chunk's data type that no strategy draws for are small numbers, which it holds.""" + data_type = _read(INT8_DATA_TYPE, "int8") + strategies: Strategies = { + CAST_VALUE_CODEC: DEFAULT_STRATEGIES[CAST_VALUE_CODEC], + FLOAT32_DATA_TYPE: DEFAULT_STRATEGIES[FLOAT32_DATA_TYPE], + } + configurations = codec_configurations( + CAST_VALUE_CODEC, Chunk(None, data_type), strategies=strategies + ) + assert configurations is not None + find(configurations, lambda configuration: "scalar_map" in configuration, settings=_REACH) + + +@st.composite +def _partly_known_chunks(draw: st.DrawFn) -> Chunk: + """A chunk some of which nothing says: its lengths along an axis or at all, or its data type, read by nothing in scope.""" + chunk = draw(_chunks()) + lengths = draw( + st.sampled_from( + [ + chunk.lengths, + None, + None if chunk.lengths is None else tuple(None for _ in chunk.lengths), + ] + ) + ) + data_type = draw(st.sampled_from([chunk.data_type, None, _UNREAD_DATA_TYPE])) + return Chunk(lengths, data_type) + + +_UNREAD_DATA_TYPE = resolve({"name": "an.unread.type"}, DataTypeDefinition, ZARR_PYTHON_CONTEXT)[0] +"""A data type field that nothing in scope claims: a chunk of it says nothing of its values.""" + + +@given(data=st.data(), chunk=_chunks() | _partly_known_chunks()) +def test_pipelines_read(data: st.DataObject, chunk: Chunk) -> None: + """A pipeline drawn for a chunk reads as one handed it, through JSON too; none is drawn only for a chunk no codec encodes.""" + drawn = pipelines(chunk) + if drawn is None: + assert chunk.data_type is not None + event(f"no pipeline for {chunk.data_type.json!r}") + assert storage_of(chunk.data_type) == "variable_length" + return + pipeline = data.draw(drawn) + for written in (pipeline, json.loads(json.dumps(pipeline))): + codecs = [resolve(field, CodecDefinition, ZARR_PYTHON_CONTEXT) for field in written] + assert [problems for _, problems in codecs] == [()] * len(codecs) + _, problems = read_pipeline([codec for codec, _ in codecs], chunk) + assert problems == () Target = tuple[Literal["present", "absent"], str] | tuple[Literal["value"], str, object] @@ -253,6 +401,7 @@ def _hits( _REACH = settings(max_examples=2000, database=None) +_ONCE = settings(database=None) @pytest.mark.parametrize( @@ -286,7 +435,7 @@ def test_declarations_reach(definition: Definition[Any], form: DeclarationForm) def test_an_added_strategy_is_drawn_from() -> None: """A strategy joined to the defaults, as an extension author adds one, is the one drawn from.""" - level_three = CodecStrategy(configurations=lambda _chunk: st.just({"level": 3})) + level_three = CodecStrategy(configurations=lambda _chunk, _strategies: st.just({"level": 3})) strategies = DEFAULT_STRATEGIES | {GZIP_CODEC: level_three} declaration = find( declarations(GZIP_CODEC, strategies=strategies), lambda _declaration: True, settings=_REACH @@ -294,29 +443,75 @@ def test_an_added_strategy_is_drawn_from() -> None: assert _read(GZIP_CODEC, declaration).configuration == {"level": 3} +_SHORT_REPR = 10_000 +"""Well short of the 30,000 characters past which Hypothesis warns about a strategy's repr. + +It writes that repr into the note on any error raised while drawing, +its own rejections among them, so a longer one fails a suite that turns +warnings into errors. +""" + + +@pytest.mark.parametrize("definition", _DEFINITIONS, ids=_id) +def test_reprs_stay_short(definition: Definition[Any]) -> None: + """However the strategies are passed: a plain mapping is as long as every strategy in it.""" + plain: Strategies = dict(DEFAULT_STRATEGIES) + assert len(repr(declarations(definition, strategies=plain))) < _SHORT_REPR + assert len(repr(pipelines(strategies=plain))) < _SHORT_REPR + + def test_a_definition_without_a_strategy_is_a_lookup_error() -> None: - with pytest.raises(LookupError, match="no strategy for 'sharding_indexed'"): - declarations(SHARDING_INDEXED_CODEC) + with pytest.raises(LookupError, match="no strategy for 'gzip'"): + declarations(GZIP_CODEC, strategies={}) def test_a_strategy_of_another_kind_is_a_type_error() -> None: # Unannotated, as an extension author writes it: a mapping keyed by a # `CodecDefinition` is a `Strategies`. - strategies = {GZIP_CODEC: ChunkKeyEncodingStrategy(configurations=st.just({}))} + strategies = { + GZIP_CODEC: ChunkKeyEncodingStrategy(configurations=lambda _strategies: st.just({})) + } with pytest.raises(TypeError, match="draws configurations of a ChunkKeyEncodingDefinition"): codec_configurations(GZIP_CODEC, strategies=strategies) def test_a_codec_strategy_refusing_an_unknown_chunk_is_a_value_error() -> None: + refusing = CodecStrategy(configurations=lambda _chunk, _strategies: None) with pytest.raises(ValueError, match=r"Chunk\(\), a chunk nothing is known of"): - CodecStrategy(configurations=lambda _chunk: None) + declarations(GZIP_CODEC, strategies=DEFAULT_STRATEGIES | {GZIP_CODEC: refusing}) + + +def test_fill_values_of_a_field_no_definition_reads_is_a_lookup_error() -> None: + """A struct handed a field type that no definition in the strategies reads has no fill values to draw for it.""" + without_float32 = {d: s for d, s in DEFAULT_STRATEGIES.items() if d is not FLOAT32_DATA_TYPE} + configuration: Any = {"fields": ({"name": "a", "data_type": "float32"},)} + with pytest.raises(LookupError, match="no definition in `strategies` reads"): + fill_values(STRUCT_DATA_TYPE, configuration, strategies=without_float32) + + +def test_a_refused_nested_draw_is_a_value_error() -> None: + """A field a configuration holds is read as it is drawn, so a strategy that draws a refused one is caught there.""" + refused = DataTypeStrategy( + configurations=lambda _strategies: st.just({"stray": 1}), + fill_values=lambda _configuration, _strategies: st.just(0.0), + ) + strategies: Strategies = { + CAST_VALUE_CODEC: DEFAULT_STRATEGIES[CAST_VALUE_CODEC], + NUMPY_DATETIME64_DATA_TYPE: refused, + } + with pytest.raises(ValueError, match="read in the strategies. scope, is refused"): + find( + declarations(CAST_VALUE_CODEC, strategies=strategies), + lambda _field: True, + settings=_ONCE, + ) @pytest.mark.parametrize("spellings", get_args(Spellings)) def test_a_refused_draw_is_a_value_error(spellings: Spellings) -> None: refused = declarations(GZIP_CODEC, st.just({"level": 99}), spellings=spellings) with pytest.raises(ValueError, match="which is refused"): - find(refused, lambda _declaration: True, settings=settings(database=None)) + find(refused, lambda _field: True, settings=_ONCE) _BFLOAT16 = DataTypeDefinition(name="bfloat16", configuration=EmptyConfiguration) @@ -344,36 +539,200 @@ def test_a_definition_no_v3_scope_holds_is_a_type_error() -> None: declarations(zlib, st.just({})) -@pytest.mark.parametrize( - "definition", [FLOAT16_DATA_TYPE, FLOAT32_DATA_TYPE, FLOAT64_DATA_TYPE], ids=_id -) +_FLOAT_BITS = {FLOAT16_DATA_TYPE: 16, FLOAT32_DATA_TYPE: 32, FLOAT64_DATA_TYPE: 64} + + +def _bits(value: object) -> int | None: + """The bits of a hex string; None for any other value.""" + return int(value, 16) if isinstance(value, str) and value.startswith("0x") else None + + +def _is_hex_of(value: object, width: int, *, exponent: bool, mantissa: bool) -> bool: + """Whether `value` is a float's bits in hex with its exponent all ones or not, and its mantissa not zero or zero.""" + bits = _bits(value) + if bits is None: + return False + mantissa_bits = {16: 10, 32: 23, 64: 52}[width] + low = bits & (2**mantissa_bits - 1) + high = (bits >> mantissa_bits) & (2 ** (width - 1 - mantissa_bits) - 1) + return (high == 2 ** (width - 1 - mantissa_bits) - 1) == exponent and (low != 0) == mantissa + + +_SCALE_OFFSET_VALUES: dict[str, Callable[[object, int], bool]] = { + "a number": lambda value, _width: isinstance(value, float) and value != 0.0, + "an integer": lambda value, _width: isinstance(value, int) and value != 0, + "NaN by name": lambda value, _width: value == "NaN", + "infinity by name": lambda value, _width: value == "Infinity", + "negative infinity by name": lambda value, _width: value == "-Infinity", + "hex of a number": lambda value, width: _is_hex_of(value, width, exponent=False, mantissa=True), + "hex of an infinity": lambda value, width: _is_hex_of( + value, width, exponent=True, mantissa=False + ), + "hex of a NaN": lambda value, width: _is_hex_of(value, width, exponent=True, mantissa=True), +} +"""Each spelling a fill value of a float takes, which `offset` and `scale` are.""" + + +@pytest.mark.parametrize("definition", _FLOAT_BITS, ids=_id) @pytest.mark.parametrize("member", ["offset", "scale"]) -def test_scale_offset_reaches_a_value_written_in_hex( - definition: DataTypeDefinition[Any], member: str +@pytest.mark.parametrize("spelling", _SCALE_OFFSET_VALUES) +def test_scale_offset_reaches_every_spelling_of_a_float( + definition: DataTypeDefinition[Any], member: str, spelling: str ) -> None: - """A float's `offset` and `scale` are fill values of its type, which the bits in hex spell too.""" + """A float's `offset` and `scale` are fill values of its type, non-finite ones and the bits in hex too.""" chunk = Chunk(None, _read(definition, definition.name)) configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) assert configurations is not None + wanted = _SCALE_OFFSET_VALUES[spelling] + width = _FLOAT_BITS[definition] find( configurations, - lambda configuration: isinstance(configuration.get(member), str), + lambda configuration: member in configuration and wanted(configuration[member], width), settings=_REACH, ) -def _has_pair(configuration: Mapping[str, Any]) -> bool: - return any( - isinstance(entry, tuple) - for dimension in configuration["chunk_shapes"] - if isinstance(dimension, tuple) - for entry in cast("tuple[object, ...]", dimension) +@pytest.mark.parametrize("definition", [COMPLEX64_DATA_TYPE, COMPLEX128_DATA_TYPE], ids=_id) +@pytest.mark.parametrize("member", ["offset", "scale"]) +@pytest.mark.parametrize("part", [0, 1]) +@pytest.mark.parametrize("value", ["NaN", "Infinity", "-Infinity"]) +def test_scale_offset_reaches_a_non_finite_part_of_a_complex( + definition: DataTypeDefinition[Any], member: str, part: int, value: str +) -> None: + """A complex number's `offset` and `scale` are pairs of float fill values, either of them non-finite.""" + chunk = Chunk(None, _read(definition, definition.name)) + configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) + assert configurations is not None + find( + configurations, + lambda configuration: ( + member in configuration + and cast("tuple[object, object]", configuration[member])[part] == value + ), + settings=_REACH, ) -def test_rectilinear_edges_reach_a_run_written_as_a_pair() -> None: - """The `[size, count]` spelling, which no TypedDict target asks for: it is one arm of a union.""" - find(chunk_grid_configurations(RECTILINEAR_CHUNK_GRID), _has_pair, settings=_REACH) +def _is_zero_spelled(value: object, width: int) -> bool: + """Whether a float fill value is zero of either sign, by its value alone.""" + bits = _bits(value) + return value == 0 if bits is None else bits in (0, 2 ** (width - 1)) + + +@pytest.mark.parametrize( + ("definition", "width"), + [ + *_FLOAT_BITS.items(), + (COMPLEX64_DATA_TYPE, 32), + (COMPLEX128_DATA_TYPE, 64), + ], + ids=str, +) +@given(data=st.data()) +def test_scale_offset_scale_is_never_zero( + definition: DataTypeDefinition[Any], width: int, data: st.DataObject +) -> None: + """No spelling of a float, nor of both parts of a complex number, draws a scale of zero: decoding divides by it.""" + chunk = Chunk(None, _read(definition, definition.name)) + configurations = codec_configurations(SCALE_OFFSET_CODEC, chunk) + assert configurations is not None + configuration = data.draw(configurations) + assert list(SCALE_OFFSET_CODEC.chunk_rules(configuration, {}, chunk)) == [] + if "scale" in configuration: + scale = configuration["scale"] + parts = scale if isinstance(scale, tuple) else (scale,) + assert not all(_is_zero_spelled(part, width) for part in parts) + + +def _entries(dimension: object) -> tuple[object, ...]: + return cast("tuple[object, ...]", dimension) if isinstance(dimension, tuple) else () + + +def _is_pair(entry: object) -> bool: + return isinstance(entry, tuple) + + +_RECTILINEAR_SPELLINGS: dict[str, tuple[tuple[int, ...] | None, Callable[[object], bool]]] = { + "a bare length": (None, lambda dimension: isinstance(dimension, int)), + "no edges": (None, lambda dimension: dimension == ()), + "only written-out lengths": ( + None, + lambda dimension: ( + len(_entries(dimension)) > 1 + and not any(_is_pair(entry) for entry in _entries(dimension)) + ), + ), + "only pairs": ( + None, + lambda dimension: ( + len(_entries(dimension)) > 1 and all(_is_pair(entry) for entry in _entries(dimension)) + ), + ), + "pairs among written-out lengths": ( + None, + lambda dimension: len({_is_pair(entry) for entry in _entries(dimension)}) == 2, + ), + "a long run written out": ( + (64,), + lambda dimension: any( + count >= 16 + for count in Counter( + entry for entry in _entries(dimension) if not _is_pair(entry) + ).values() + ), + ), + "a short run written as a pair": ( + None, + lambda dimension: any( + _is_pair(entry) and cast("tuple[int, int]", entry)[1] <= 4 + for entry in _entries(dimension) + ), + ), + "a run too long to write out": ( + (2**16,), + lambda dimension: any( + _is_pair(entry) and cast("tuple[int, int]", entry)[1] > 64 + for entry in _entries(dimension) + ), + ), + "a run that covers a long dimension": ( + (2**16,), + lambda dimension: any( + _is_pair(entry) + and cast("tuple[int, int]", entry)[1] > 1 + and math.prod(cast("tuple[int, int]", entry)) >= 2**15 + for entry in _entries(dimension) + ), + ), + "a run of a million chunks past an unknown length": ( + None, + lambda dimension: any( + _is_pair(entry) and cast("tuple[int, int]", entry)[1] >= 2**16 + for entry in _entries(dimension) + ), + ), +} +"""What each spelling of a rectilinear dimension looks like, and the shape it is drawn over: spellings of one union, which no TypedDict target asks for.""" + + +@pytest.mark.parametrize("spelling", _RECTILINEAR_SPELLINGS) +def test_rectilinear_dimensions_reach(spelling: str) -> None: + shape, reached = _RECTILINEAR_SPELLINGS[spelling] + find( + chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, shape), + lambda configuration: any(reached(d) for d in configuration["chunk_shapes"]), + settings=_REACH, + ) + + +@given(data=st.data(), shape=st.lists(st.integers(0, 2**63 - 1), max_size=3).map(tuple)) +def test_chunk_grids_fit_a_long_shape(data: st.DataObject, shape: tuple[int, ...]) -> None: + """However long an axis is, a grid covers it without a list as long as the axis.""" + configurations = chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, shape) + declaration = data.draw(declarations(RECTILINEAR_CHUNK_GRID, configurations)) + _, problems = chunk_grid_lengths(_read(RECTILINEAR_CHUNK_GRID, declaration), shape) + assert problems == () + assert len(json.dumps(declaration)) < 10_000 FillValueSpelling = Literal[ @@ -436,3 +795,364 @@ def test_fill_values_reach( lambda configuration: fill_values(definition, configuration) ) find(values, lambda value: spelling in _spellings_of(value), settings=_REACH) + + +def _codecs_in(pipeline: tuple[Any, ...]) -> list[tuple[str, tuple[str, ...]]]: + """Each codec's name in `pipeline`, with the members of the shards it sits inside.""" + found: list[tuple[str, tuple[str, ...]]] = [] + for field in pipeline: + if isinstance(field, str): + found.append((field, ())) + continue + found.append((field["name"], ())) + if field["name"] != SHARDING_INDEXED_CODEC.name: + continue + configuration: Mapping[str, Any] = field["configuration"] + for member in ("codecs", "index_codecs"): + found.extend( + (inner, (member, *within)) for inner, within in _codecs_in(configuration[member]) + ) + return found + + +def _drawn(strategy: st.SearchStrategy[T] | None) -> st.SearchStrategy[T]: + """`strategy`, or nothing when there is none.""" + return st.nothing() if strategy is None else strategy + + +def _drawn_pipelines() -> st.SearchStrategy[tuple[Any, ...]]: + return _chunks().flatmap(lambda chunk: _drawn(pipelines(chunk))) + + +def _shards() -> st.SearchStrategy[Any]: + """Shards drawn for the chunks of a grid.""" + return _chunks().flatmap( + lambda chunk: _drawn(codec_configurations(SHARDING_INDEXED_CODEC, chunk)).flatmap( + lambda configuration: declarations(SHARDING_INDEXED_CODEC, st.just(configuration)) + ) + ) + + +@pytest.mark.parametrize("definition", _CODECS, ids=_id) +def test_pipelines_reach_every_codec(definition: CodecDefinition[Any]) -> None: + find( + _drawn_pipelines(), + lambda pipeline: any(name == definition.name for name, _ in _codecs_in(pipeline)), + settings=_REACH, + ) + + +@pytest.mark.parametrize( + ("definition", "member"), + [ + (d, member) + for d in _CODECS + if d is not SHARDING_INDEXED_CODEC + for member in ("codecs", "index_codecs") + if member == "codecs" or d.size == "static" + ], + ids=_id, +) +def test_shards_hold_every_codec_their_pipeline_takes( + definition: CodecDefinition[Any], member: str +) -> None: + """A shard's pipelines reach each codec they take: its index codecs, only those of static size.""" + find( + _shards(), + lambda shard: (definition.name, (member,)) in _codecs_in((shard,)), + settings=_REACH, + ) + + +def test_shards_reach_a_shard_inside() -> None: + find( + _shards(), + lambda shard: (SHARDING_INDEXED_CODEC.name, ("codecs",)) in _codecs_in((shard,)), + settings=_REACH, + ) + + +_VARIABLE_LENGTH = frozenset(d for d in _DATA_TYPES if d.storage is variable_length) +"""The data types whose values vary in size, which a struct's fields may not be.""" + + +@pytest.mark.parametrize( + "definition", [d for d in _DATA_TYPES if d not in _VARIABLE_LENGTH], ids=_id +) +def test_struct_fields_reach_every_fixed_size_data_type( + definition: DataTypeDefinition[Any], +) -> None: + """A struct's field types reach each data type of fixed size, a struct among them.""" + find( + data_type_configurations(STRUCT_DATA_TYPE), + lambda configuration: any( + _read_data_type(member["data_type"]).definition is definition + for member in configuration["fields"] + ), + settings=_REACH, + ) + + +def _read_data_type(field: object) -> Any: # noqa: ANN401 + resolved, problems = resolve(field, DataTypeDefinition, ZARR_PYTHON_CONTEXT) + assert problems == () + return resolved + + +_REAL_NUMBERS = [ + d + for d in _DATA_TYPES + if d.name + in { + *(f"{sign}int{bits}" for sign in ("", "u") for bits in (8, 16, 32, 64)), + *(f"float{bits}" for bits in (16, 32, 64)), + "numpy.datetime64", + "numpy.timedelta64", + } +] +"""The data types `cast_value` casts to: integers and floats, and the numpy times, which are signed 64-bit integers.""" + + +@pytest.mark.parametrize("definition", _REAL_NUMBERS, ids=_id) +def test_cast_value_reaches_every_real_number_data_type( + definition: DataTypeDefinition[Any], +) -> None: + find( + declarations(CAST_VALUE_CODEC).map(lambda field: _read(CAST_VALUE_CODEC, field)), + lambda codec: codec.nested[("data_type",)].definition is definition, + settings=_REACH, + ) + + +@pytest.mark.parametrize("direction", ["encode", "decode"]) +def test_cast_value_reaches_scalars_in_each_direction(direction: str) -> None: + find( + _drawn(codec_configurations(CAST_VALUE_CODEC)), + lambda configuration: len(configuration.get("scalar_map", {}).get(direction, ())) != 0, + settings=_REACH, + ) + + +@given(declaration=declarations(CAST_VALUE_CODEC)) +def test_cast_value_draws_only_real_number_data_types(declaration: ZarrV3DeclarationJSON) -> None: + """The reach test above says each of these is drawn; this says nothing else is.""" + target = _read(CAST_VALUE_CODEC, declaration).nested[("data_type",)] + assert target.definition in _REAL_NUMBERS + + +_PLAIN_GZIP = CodecDefinition( + name=GZIP_CODEC.name, configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" +) +"""A codec named as one the defaults hold, with a configuration of its own.""" + +_WITH_PLAIN_GZIP = DEFAULT_STRATEGIES | { + _PLAIN_GZIP: CodecStrategy(configurations=lambda _chunk, _strategies: EMPTY_CONFIGURATIONS) +} + + +@given(declaration=declarations(_PLAIN_GZIP, st.just({}))) +def test_a_definition_named_as_a_default_is_read_as_itself( + declaration: ZarrV3DeclarationJSON, +) -> None: + """It claims its name in the scope its draw is read in, though the strategies hold another, whose rules refuse an empty configuration.""" + scope = scope_with(_PLAIN_GZIP, DEFAULT_STRATEGIES) + read, problems = resolve(declaration, CodecDefinition, scope) + assert problems == () + assert isinstance(read, AcceptedDeclaration) + assert read.definition is _PLAIN_GZIP + + +def test_a_definition_shadowed_in_the_strategies_is_read_as_itself() -> None: + """Asked for by name, it claims that name over the one that replaced it.""" + declaration = find( + declarations(GZIP_CODEC, strategies=_WITH_PLAIN_GZIP), + lambda _declaration: True, + settings=_ONCE, + ) + assert _read(GZIP_CODEC, declaration).definition is GZIP_CODEC + + +@given(pipeline=_drawn(pipelines(strategies=_WITH_PLAIN_GZIP))) +def test_a_later_definition_replaces_a_default_of_its_name(pipeline: tuple[Any, ...]) -> None: + """A pipeline holds the later of two codecs with one name, so none has the earlier's `level`.""" + gzips = [ + field for field in pipeline if isinstance(field, dict) and field["name"] == GZIP_CODEC.name + ] + assert [field.get("configuration", {}) for field in gzips] == [{}] * len(gzips) + + +_PLAIN_STRUCT = DataTypeDefinition(name=STRUCT_DATA_TYPE.name, configuration=EmptyConfiguration) +"""A data type named as the struct the defaults hold, with a configuration of its own.""" + +_PLAIN_SHARD = CodecDefinition( + name=SHARDING_INDEXED_CODEC.name, + configuration=EmptyConfiguration, + kind="array_bytes", + size="dynamic", +) +"""A codec named as the shard the defaults hold, with a configuration of its own.""" + + +@given( + declaration=declarations( + STRUCT_DATA_TYPE, + strategies=DEFAULT_STRATEGIES + | { + _PLAIN_STRUCT: DataTypeStrategy( + configurations=lambda _strategies: EMPTY_CONFIGURATIONS, + fill_values=lambda _configuration, _strategies: st.just(0), + ) + }, + ) +) +def test_a_struct_holds_no_other_definition_of_its_name( + declaration: ZarrV3DeclarationJSON, +) -> None: + """Its field types are read as the struct asked for, not the one named as it in the strategies.""" + assert _read(STRUCT_DATA_TYPE, declaration).definition is STRUCT_DATA_TYPE + + +@given( + declaration=declarations( + SHARDING_INDEXED_CODEC, + strategies=DEFAULT_STRATEGIES + | {_PLAIN_SHARD: CodecStrategy(configurations=lambda _chunk, _s: EMPTY_CONFIGURATIONS)}, + ) +) +def test_a_shard_holds_no_other_definition_of_its_name( + declaration: ZarrV3DeclarationJSON, +) -> None: + """Its inner codecs are read as the shard asked for, not the one named as it in the strategies.""" + assert _read(SHARDING_INDEXED_CODEC, declaration).definition is SHARDING_INDEXED_CODEC + + +def _float_class(value: object) -> str | None: + """What the bits of a `float64` fill value written in hex are; None for any other spelling.""" + if not isinstance(value, str) or not value.startswith("0x"): + return None + bits = int(value, 16) + exponent, mantissa = (bits >> 52) & 0x7FF, bits & (2**52 - 1) + if exponent == 0x7FF: + return "infinity" if mantissa == 0 else "NaN" + if exponent == 0: + return "zero" if mantissa == 0 else "subnormal" + return "normal" + + +_HEX_FILL_VALUES = fill_values(FLOAT64_DATA_TYPE, {}).filter( + lambda value: _float_class(value) is not None +) + + +@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.1), ("infinity", 0.01)]) +def test_hex_fill_values_are_not_only_small_numbers(kind: str, at_least: float) -> None: + """Bits drawn as an integer alone are nearly all subnormal.""" + assert share_of(_HEX_FILL_VALUES, lambda value: _float_class(value) == kind) >= at_least + + +@pytest.mark.parametrize( + ("definition", "beyond"), + [(FLOAT32_DATA_TYPE, 2**11), (FLOAT64_DATA_TYPE, 2**24)], + ids=_id, +) +def test_integer_fill_values_reach_past_a_narrower_float( + definition: DataTypeDefinition[Any], beyond: int +) -> None: + """Each width draws the integers it holds, not only those the narrowest does.""" + find( + fill_values(definition, {}), + lambda value: isinstance(value, int) and abs(value) > beyond, + settings=_REACH, + ) + + +@st.composite +def _shard_inner_lengths(draw: st.DrawFn) -> tuple[int, ...]: + """The inner chunk lengths of a shard of a rectilinear grid's chunks.""" + shape = draw(st.lists(st.integers(1, 256), min_size=1, max_size=3).map(tuple)) + grid = _read( + RECTILINEAR_CHUNK_GRID, + draw( + declarations( + RECTILINEAR_CHUNK_GRID, chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, shape) + ) + ), + ) + lengths, _ = chunk_grid_lengths(grid, shape) + return draw(_drawn(codec_configurations(SHARDING_INDEXED_CODEC, Chunk(lengths))))["chunk_shape"] + + +def test_shards_of_a_rectilinear_grid_are_often_divided() -> None: + """Edges drawn apart rarely share a factor, which leaves a shard only inner chunks of length 1.""" + assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.4 + + +def _edge_factor(configuration: Mapping[str, Any]) -> int: + """The largest factor every edge of a dimension shares; 0 for a dimension with no edges.""" + dimension = configuration["chunk_shapes"][0] + if isinstance(dimension, int): + return dimension + return math.gcd(*(entry if isinstance(entry, int) else entry[0] for entry in dimension)) + + +_LONGEST_AXES = st.lists( + st.integers(0, 2**63 - 1) | st.just(2**63 - 1), min_size=1, max_size=3 +).map(tuple) + + +def _chunk_lengths_of(configuration: Mapping[str, Any]) -> list[int]: + if "chunk_shape" in configuration: + return list(configuration["chunk_shape"]) + return [ + length + for dimension in configuration["chunk_shapes"] + for length in ( + [dimension] + if isinstance(dimension, int) + else (entry if isinstance(entry, int) else entry[0] for entry in dimension) + ) + ] + + +@pytest.mark.parametrize("definition", _CHUNK_GRIDS, ids=_id) +@given(data=st.data(), shape=_LONGEST_AXES) +def test_chunk_lengths_fit_a_signed_64_bit_size( + definition: ChunkGridDefinition[Any], data: st.DataObject, shape: tuple[int, ...] +) -> None: + """However long an axis is, and however a grid's chunks run past it.""" + configuration = data.draw(chunk_grid_configurations(definition, shape)) + assert max(_chunk_lengths_of(configuration), default=0) <= 2**63 - 1 + declaration = data.draw(declarations(definition, st.just(configuration))) + _, problems = chunk_grid_lengths(_read(definition, declaration), shape) + assert problems == () + + +def test_a_long_dimension_keeps_its_edge_factor() -> None: + """The edge that covers what the runs leave of a dimension shares their factor, or none would.""" + edges = chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, (2**16 + 1,)).filter( + lambda configuration: not isinstance(configuration["chunk_shapes"][0], int) + ) + assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.3 + + +def test_divisors_of_a_very_long_shard_are_found_quickly() -> None: + """A length of 2**64 has its divisors found without counting to its square root.""" + configurations = codec_configurations(SHARDING_INDEXED_CODEC, Chunk((frozenset({2**64}),))) + assert configurations is not None + find( + configurations, + lambda configuration: configuration["chunk_shape"][0] > 2**32, + settings=_REACH, + ) + + +def _field_types(declaration: Any) -> list[str]: # noqa: ANN401 + return [ + field["data_type"] if isinstance(field["data_type"], str) else field["data_type"]["name"] + for field in declaration["configuration"]["fields"] + ] + + +def test_struct_fields_do_not_favor_the_first_data_type() -> None: + """A type drawn after a field's name would be `bool` a fifth of the time.""" + assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.12 diff --git a/packages/zarr-metadata/tests/v3/test_zarr_python_scope.py b/packages/zarr-metadata/tests/v3/test_zarr_python_scope.py index 26159ac323..00d8701960 100644 --- a/packages/zarr-metadata/tests/v3/test_zarr_python_scope.py +++ b/packages/zarr-metadata/tests/v3/test_zarr_python_scope.py @@ -218,6 +218,45 @@ def test_error_a_bytes_codec_without_an_endian_for_fixed_length_utf32() -> None: ] +_NOT_NUMBERS = [ + (_UTF32, ""), + (_NULL_TERMINATED, ""), + ("variable_length_bytes", ""), + (_RAW, ""), +] +"""The data types this scope adds, each beside a fill value it takes; none of them holds numbers.""" + + +@pytest.mark.parametrize("data_type", [data_type for data_type, _ in _NOT_NUMBERS]) +def test_error_a_cast_to_a_data_type_that_holds_no_numbers(data_type: object) -> None: + cast_value = {"name": "cast_value", "configuration": {"data_type": data_type}} + document = _document("int32", 0, [cast_value, _LITTLE]) + reading = read_array_metadata_v3(document, context=ZARR_PYTHON_CONTEXT) + assert [(p.loc, p.kind) for p in reading.problems] == [ + (("codecs", 0, "configuration", "data_type"), "invalid_value") + ] + + +@pytest.mark.parametrize(("data_type", "fill_value"), _NOT_NUMBERS) +def test_error_a_cast_from_a_data_type_that_holds_no_numbers( + data_type: object, fill_value: str +) -> None: + cast_value = {"name": "cast_value", "configuration": {"data_type": "int32"}} + document = _document(data_type, fill_value, [cast_value, _LITTLE]) + reading = read_array_metadata_v3(document, context=ZARR_PYTHON_CONTEXT) + assert [(p.loc, p.kind) for p in reading.problems] == [(("codecs", 0), "invalid_value")] + + +@pytest.mark.parametrize(("data_type", "fill_value"), _NOT_NUMBERS) +def test_error_a_scale_offset_of_a_data_type_that_holds_no_numbers( + data_type: object, fill_value: str +) -> None: + serializer = "vlen-bytes" if data_type == "variable_length_bytes" else _LITTLE + document = _document(data_type, fill_value, ["scale_offset", serializer]) + reading = read_array_metadata_v3(document, context=ZARR_PYTHON_CONTEXT) + assert [(p.loc, p.kind) for p in reading.problems] == [(("codecs", 0), "invalid_value")] + + def test_an_empty_scope_reads_none_of_them() -> None: for name in ( "fixed_length_utf32",