Merge branch 'fix/zero-length-single-invariant' (zarr-developers#4334) into test/rectilinear-declaration-space

Author: d-v-b

changes/4334.bugfix.md

@@ -0,0 +1,3 @@
+Chunk-grid normalization now shares a helper that selects chunk size 1 for a zero-length axis when inferring a full-span chunk size. Chunk edge lengths remain positive while array extents may be zero. `FixedDimension(size=0, ...)` now raises `ValueError`. Rectilinear chunk grids can be created on a zero-length dimension with a non-empty list of positive chunk sizes; these sizes are retained for subsequent growth.
+
+A stored chunk size of 0 on a zero-length axis is read as 1 with a `ZarrUserWarning`, in both Zarr format 2 (`chunks`) and Zarr format 3 (a regular grid's `chunk_shape`, including the JSON `false` written by zarr-python 3.0 for `chunks=False`); a zero chunk size on a positive-length axis is rejected. zarr-python wrote such metadata for empty arrays until 3.4 — Zarr format 2 through 3.3 and by zarr-python 2.x, Zarr format 3 in 3.0 and 3.1 — and appending to one of these Zarr format 2 arrays previously reported success while writing no chunk, so the appended data read back as the fill value. The warning explains how to store a corrected chunk size: open the array writable and call `array.update_attributes({})`.

docs/user-guide/arrays.md

@@ -707,6 +707,12 @@ z.append(np.arange(10, dtype='float64'))
 print(f"After append:  shape={z.shape}, chunk_sizes={z.write_chunk_sizes}")
 ```
 
+A rectilinear array can also be created with a zero-length dimension: because no
+non-empty list of positive chunk sizes can sum to 0, the chunk sizes given for such a
+dimension are stored as-is and describe the chunks the dimension will grow into
+on `append` or `resize` — the same state as resizing an existing rectilinear
+dimension down to 0.
+
 ### Compressors and filters
 
 Rectilinear arrays work with all codecs — compressors, filters, and checksums.

src/zarr/core/chunk_grids.py

@@ -47,22 +47,24 @@
 @dataclass(frozen=True)
 class FixedDimension:
     """Uniform chunk size. Boundary chunks contain less data but are
-    encoded at full size by the codec pipeline."""
+    encoded at full size by the codec pipeline.
 
-    size: int  # chunk edge length (>= 0)
-    extent: int  # array dimension length
+    The chunk edge length is always at least 1, matching the invariant the
+    metadata layer enforces for every stored chunk grid. The extent may be 0:
+    a zero-length axis simply has zero chunks (``ceildiv(0, size) == 0``).
+    """
+
+    size: int  # chunk edge length (>= 1)
+    extent: int  # array dimension length (>= 0)
     nchunks: int = field(init=False, repr=False)
     ngridcells: int = field(init=False, repr=False)
 
     def __post_init__(self) -> None:
-        if self.size < 0:
-            raise ValueError(f"FixedDimension size must be >= 0, got {self.size}")
+        if self.size < 1:
+            raise ValueError(f"FixedDimension size must be >= 1, got {self.size}")
         if self.extent < 0:
             raise ValueError(f"FixedDimension extent must be >= 0, got {self.extent}")
-        if self.size == 0:
-            n = 0
-        else:
-            n = ceildiv(self.extent, self.size)
+        n = ceildiv(self.extent, self.size)
         object.__setattr__(self, "nchunks", n)
         object.__setattr__(self, "ngridcells", n)
 
@@ -71,8 +73,6 @@ def index_to_chunk(self, idx: int) -> int:
             raise IndexError(f"Negative index {idx} is not allowed")
         if idx >= self.extent:
             raise IndexError(f"Index {idx} is out of bounds for extent {self.extent}")
-        if self.size == 0:
-            return 0
         return idx // self.size
 
     def chunk_offset(self, chunk_ix: int) -> int:
@@ -97,8 +97,6 @@ def data_size(self, chunk_ix: int) -> int:
         Does not validate *chunk_ix* — callers must ensure it is in
         ``[0, nchunks)``. Use ``ChunkGrid.__getitem__`` for safe access.
         """
-        if self.size == 0:
-            return 0
         return max(0, min(self.size, self.extent - chunk_ix * self.size))
 
     @property
@@ -112,8 +110,6 @@ def _unique_edge_lengths(self) -> Iterable[int]:
         return (self.size,)
 
     def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]:
-        if self.size == 0:
-            return np.zeros_like(indices)
         return indices // self.size
 
     def with_extent(self, new_extent: int) -> FixedDimension:
@@ -660,6 +656,20 @@ class ChunkLayout(NamedTuple):
     inner: ChunkLayout | None = None
 
 
+def _full_span_chunk_size(span: int) -> int:
+    """The edge length of one chunk covering an entire axis of length *span*.
+
+    This is *the* definition of "one chunk spans the axis" for a possibly
+    zero-length axis. Chunk edge lengths must be at least 1 (the invariant
+    shared by `FixedDimension`, `VaryingDimension` and the stored chunk grid
+    metadata), so a zero-length axis gets chunk size 1 and zero chunks. Every
+    spelling that derives a chunk size from a span — ``chunks=-1``,
+    ``chunks=False``, ``chunks="auto"``, ``shards="auto"`` — must route
+    through this helper rather than clamping on its own.
+    """
+    return max(span, 1)
+
+
 def _guess_regular_chunks(
     shape: tuple[int, ...] | int,
     typesize: int,
@@ -697,11 +707,10 @@ def _guess_regular_chunks(
         shape = (shape,)
 
     if typesize == 0:
-        return shape
+        return tuple(_full_span_chunk_size(s) for s in shape)
 
     ndims = len(shape)
-    # require chunks to have non-zero length for all dimensions
-    chunks = np.maximum(np.array(shape, dtype="=f8"), 1)
+    chunks = np.array([_full_span_chunk_size(s) for s in shape], dtype="=f8")
 
     # Determine the optimal chunk size in bytes using a PyTables expression.
     # This is kept as a float.
@@ -744,13 +753,21 @@ def normalize_chunks_1d(chunks: int | Iterable[object], span: int) -> DimensionG
     the span, and the uniform form is O(1) in the number of chunks — a
     dimension with `2**62` chunks must not materialize one entry per chunk.
 
-    `-1` means "one chunk covering the entire span."
+    `-1` means "one chunk covering the entire span" (see `_full_span_chunk_size`
+    for what that means on a zero-length span).
     Explicit chunk size lists must sum to the span exactly and always produce
     `VaryingDimension`, even when the sizes happen to be uniform: the input
     syntax declares the grid kind, so a per-chunk list is preserved as a
     rectilinear dimension rather than silently collapsed to a regular one,
     which would change how the dimension grows on resize. For scalar sizes
     the last chunk may overhang the span.
+
+    The one exception to the sum rule is a zero-length span: no non-empty list of
+    positive edges can sum to 0, so a non-empty list of positive integers is retained
+    and the edges describe the chunks the axis will grow into on `append` /
+    `resize`. This is the same state a rectilinear axis reaches when it is
+    resized down to 0 — `VaryingDimension` allows trailing edges beyond the
+    extent — so creating at length 0 and shrinking to 0 are indistinguishable.
     """
     # `numbers.Integral` rather than `int` so that numpy integer scalars (which are not
     # `int` subclasses) take the uniform-chunk path instead of being treated as a sequence.
@@ -761,9 +778,7 @@ def normalize_chunks_1d(chunks: int | Iterable[object], span: int) -> DimensionG
         if chunk_size < -1 or chunk_size == 0:
             raise ValueError(f"Chunk size must be positive or -1, got {chunk_size}")
         if chunk_size == -1:
-            # A zero-length span still gets chunk size 1 (chunk sizes must be positive),
-            # matching the auto-chunking clamp in _guess_regular_chunks.
-            return FixedDimension(size=max(span, 1), extent=span)
+            return FixedDimension(size=_full_span_chunk_size(span), extent=span)
         return FixedDimension(size=chunk_size, extent=span)
     else:
         try:
@@ -788,7 +803,9 @@ def normalize_chunks_1d(chunks: int | Iterable[object], span: int) -> DimensionG
         ints: list[int] = [int(c) for c in chunk_list]  # type: ignore[call-overload]
         if any(c <= 0 for c in ints):
             raise ValueError(f"All chunk sizes must be positive, got {ints}")
-        if sum(ints) != span:
+        # A zero-length span cannot be covered by positive edges; the edges are the
+        # chunks the axis will grow into, exactly as after ``resize(0)``.
+        if span > 0 and sum(ints) != span:
             raise ValueError(f"Chunk sizes {ints} do not sum to span {span}")
         return VaryingDimension(ints, extent=span)
 
@@ -829,7 +846,7 @@ def normalize_chunks_nd(
         )
 
     # handle no chunking: one chunk covering every axis. Routed through the -1 sentinel so
-    # the zero-length-axis clamp lives in one place (normalize_chunks_1d).
+    # the zero-length-axis rule lives in one place (_full_span_chunk_size).
     if chunks is False:
         chunks = -1
 
@@ -884,8 +901,10 @@ def _guess_num_chunks_per_axis_shard(
     In other words the shard would be a (2,2,2) grid of (2,2,2) chunks
     i.e., prod(chunk_shape) * (returned_val ** len(chunk_shape)) * item_size = 256 bytes.
 
-    Degenerate chunk shapes — a 0-dimensional shape, or one containing a zero-length
-    axis — return 1, as the search loop's stopping conditions can never be met.
+    Degenerate inputs — a 0-dimensional chunk shape, or a zero-byte chunk (``item_size``
+    of 0; chunk edge lengths themselves are always at least 1) — return 1, as the
+    search loop's stopping conditions can never be met. A zero-length *array* axis
+    needs no special case: the array-bound check fails immediately for it.
 
     Parameters
     ----------
@@ -906,8 +925,8 @@ def _guess_num_chunks_per_axis_shard(
     if max_bytes < bytes_per_chunk:
         return 1
     num_axes = len(chunk_shape)
-    # For a 0-dimensional chunk shape or one with a zero-length axis, both loop
-    # conditions below are constant, so the loop would never terminate.
+    # For a 0-dimensional chunk shape or a zero-byte chunk, both loop conditions
+    # below are constant, so the loop would never terminate.
     if num_axes == 0 or bytes_per_chunk == 0:
         return 1
     chunks_per_shard = 1

src/zarr/core/metadata/common.py

@@ -1,13 +1,69 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+import warnings
+from typing import TYPE_CHECKING, Final
+
+from zarr.errors import ZarrUserWarning
 
 if TYPE_CHECKING:
+    from collections.abc import Sequence
+
     from zarr.core.common import JSON
 
+RESAVE_METADATA_HINT: Final = (
+    "Re-save the array metadata to store the corrected value: open the array "
+    "writable and call `array.update_attributes({})`."
+)
+"""How to persist metadata that was read under a compatibility policy.
+
+`update_attributes` rewrites the whole metadata document from the parsed
+(corrected) metadata, so an empty update is enough.
+"""
+
 
 def parse_attributes(data: dict[str, JSON] | None) -> dict[str, JSON]:
     if data is None:
         return {}
 
     return dict(data)
+
+
+def parse_stored_regular_chunk_shape(
+    chunk_shape: Sequence[int], shape: Sequence[int], *, legacy_writers: str
+) -> tuple[int, ...]:
+    """Validate a stored regular chunk grid's chunk shape against the array shape.
+
+    This is for regular chunk grids only: Zarr format 2 `chunks`, and the
+    `chunk_shape` of a Zarr format 3 `regular` grid. Another chunk grid, such
+    as the rectilinear grid, is free to define its own meaning for a chunk of
+    length 0, so its chunk sizes must not be passed here.
+
+    The chunk shape must have one entry per array axis, and every chunk size
+    must be at least 1. The exception is a chunk size of 0 on a zero-length
+    axis, which `legacy_writers` stored for arrays created empty. An empty axis
+    has no chunk for the size to describe, so it is read as 1 with a
+    `ZarrUserWarning` that says how to re-save valid metadata. A chunk size of
+    0 on a positive-length axis is rejected, because the metadata cannot say
+    how the stored chunks were laid out. JSON `false` counts as 0.
+    """
+    if len(chunk_shape) != len(shape):
+        raise ValueError(
+            f"The chunk shape {tuple(chunk_shape)} and the array shape {tuple(shape)} "
+            "must have the same number of dimensions."
+        )
+    parsed: list[int] = []
+    for dim_idx, (size, extent) in enumerate(zip(chunk_shape, shape, strict=True)):
+        if size < 1:
+            if size < 0 or extent != 0:
+                raise ValueError(
+                    f"Dimension {dim_idx}: chunk edge length must be >= 1, got {size!r}"
+                )
+            warnings.warn(
+                f"Dimension {dim_idx}: chunk edge length {size!r} on a zero-length axis "
+                f"(as written by {legacy_writers}) is treated as 1. {RESAVE_METADATA_HINT}",
+                ZarrUserWarning,
+                stacklevel=3,
+            )
+            size = 1
+        parsed.append(size)
+    return tuple(parsed)

src/zarr/core/metadata/v2.py

@@ -42,7 +42,7 @@
 )
 from zarr.core.config import config, parse_indexing_order
 from zarr.core.json_parse import parse_field
-from zarr.core.metadata.common import parse_attributes
+from zarr.core.metadata.common import parse_attributes, parse_stored_regular_chunk_shape
 
 
 class ArrayV2MetadataDict(TypedDict):
@@ -88,7 +88,11 @@ def __init__(
         Metadata for a Zarr format 2 array.
         """
         shape_parsed = parse_shapelike(shape)
-        chunks_parsed = parse_shapelike(chunks)
+        chunks_parsed = parse_stored_regular_chunk_shape(
+            parse_shapelike(chunks),
+            shape_parsed,
+            legacy_writers="zarr-python 2.x, and by 3.x before 3.4",
+        )
         compressor_parsed = parse_compressor(compressor)
         order_parsed = parse_indexing_order(order)
         dimension_separator_parsed = parse_separator(dimension_separator)
@@ -111,7 +115,6 @@ def __init__(
         object.__setattr__(self, "attributes", attributes_parsed)
 
         # ensure that the metadata document is consistent
-        _ = parse_metadata(self)
 
     @property
     def ndim(self) -> int:
@@ -323,16 +326,6 @@ def parse_compressor(data: object) -> Numcodec | None:
     raise ValueError(msg)
 
 
-def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata:
-    if (l_chunks := len(data.chunks)) != (l_shape := len(data.shape)):
-        msg = (
-            f"The `shape` and `chunks` attributes must have the same length. "
-            f"`chunks` has length {l_chunks}, but `shape` has length {l_shape}."
-        )
-        raise ValueError(msg)
-    return data
-
-
 def get_object_codec_id(maybe_object_codecs: Sequence[JSON]) -> str | None:
     """
     Inspect a sequence of codecs / filters for an "object codec", i.e. a codec

src/zarr/core/metadata/v3.py

@@ -5,6 +5,7 @@
 from dataclasses import dataclass, field, replace
 from typing import TYPE_CHECKING, Any, Final, Literal, NotRequired, TypeGuard, cast
 
+import numpy as np
 from typing_extensions import TypedDict
 
 from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec
@@ -35,7 +36,7 @@
 from zarr.core.dtype import VariableLengthUTF8, ZDType, get_data_type_from_json
 from zarr.core.dtype.common import check_dtype_spec_v3
 from zarr.core.json_parse import parse_field
-from zarr.core.metadata.common import parse_attributes
+from zarr.core.metadata.common import parse_attributes, parse_stored_regular_chunk_shape
 from zarr.errors import MetadataValidationError, NodeTypeValidationError
 from zarr.registry import get_codec_class
 
@@ -476,6 +477,48 @@ class ArrayMetadataJSON_V3(TypedDict, extra_items=AllowedExtraField):  # type: i
 }
 
 
+def _is_regular_chunk_shape(value: object) -> TypeGuard[Sequence[int]]:
+    """Whether a stored `chunk_shape` is a regular chunk shape: a sequence of integers.
+
+    JSON `false` counts, because `bool` is an integer type.
+    """
+    return (
+        isinstance(value, Sequence)
+        and not isinstance(value, str)
+        and all(isinstance(size, int | np.integer) for size in value)
+    )
+
+
+def _parse_stored_regular_chunk_grid(
+    chunk_grid: dict[str, JSON] | ChunkGridMetadata | NamedConfig[str, Any],
+    shape: tuple[int, ...],
+) -> dict[str, JSON] | ChunkGridMetadata | NamedConfig[str, Any]:
+    """Check a stored regular chunk grid's chunk shape against the array shape.
+
+    Only a `regular` grid whose `chunk_shape` is all integers is a regular chunk
+    shape, and only that is handed to `parse_stored_regular_chunk_shape`.
+    Anything else is not a regular chunk shape and is left for the chunk grid
+    parser: other grids define their own chunk semantics. zarr-python 3.0 and
+    3.1 stored `chunk_shape: [0]` (and 3.0 `[false]`) for an array created with
+    a zero-length axis. This runs here rather than in the grid parser because
+    it needs the array shape, which chunk grid metadata does not carry.
+    """
+    if not isinstance(chunk_grid, Mapping) or chunk_grid.get("name") != "regular":
+        return chunk_grid
+    configuration = chunk_grid.get("configuration")
+    if not isinstance(configuration, Mapping):
+        return chunk_grid
+    chunk_shape = configuration.get("chunk_shape")
+    if not _is_regular_chunk_shape(chunk_shape):
+        return chunk_grid
+    parsed = parse_stored_regular_chunk_shape(
+        chunk_shape, shape, legacy_writers="zarr-python 3.0 and 3.1"
+    )
+    corrected: dict[str, Any] = dict(chunk_grid)
+    corrected["configuration"] = {**configuration, "chunk_shape": list(parsed)}
+    return corrected
+
+
 @dataclass(frozen=True, kw_only=True)
 class ArrayV3Metadata(Metadata):
     shape: tuple[int, ...]
@@ -510,7 +553,9 @@ def __init__(
         """
 
         shape_parsed = parse_shapelike(shape)
-        chunk_grid_parsed = parse_chunk_grid(chunk_grid)
+        chunk_grid_parsed = parse_chunk_grid(
+            _parse_stored_regular_chunk_grid(chunk_grid, shape_parsed)
+        )
         chunk_key_encoding_parsed = parse_chunk_key_encoding(chunk_key_encoding)
         dimension_names_parsed = parse_dimension_names(dimension_names)
         # Note: relying on a type method is numpy-specific