Merge commit '0992e948b' into HEAD

Author: d-v-b

changes/4334.bugfix.md

@@ -4,4 +4,4 @@ A chunk edge length is now always at least 1, while an array extent may be 0. Ev
 
 Stored metadata with a regular chunk size of 0 or JSON `false` (Zarr format 2 `chunks`, Zarr format 3 `regular` `chunk_shape`) is now read as chunk size 1 (the inner chunk size for sharded arrays, which previously failed to open), however long the axis is: that is the chunk size `chunks=-1` gives on a zero-length axis, and it does not depend on how far the axis has grown since. zarr-python wrote such sizes for arrays created with a zero-length axis until 3.4, and, from 2.18.7 to 3.2.1, for an explicit chunk size of 0 or `False` on an axis of any length, as in `chunks=(0,)` (those arrays could store no data). On a zero-length axis this opens silently, as before, and appending to it stores chunks of size 1. On an axis of positive length it opens with a `ZarrUserWarning`, which says that the array holds only its fill value, so that recreating it with the wanted chunk shape loses nothing (`zarr.from_array(array.store, name=array.path, data=array, chunks=..., overwrite=True, write_data=False)` keeps its data type, fill value, attributes and codecs), and how to keep it by storing valid metadata instead: `array.update_attributes({})`, then `zarr.consolidate_metadata` if the metadata is consolidated. JSON `true`, as zarr-python 3.0 and 3.2 wrote for a chunk size of `True`, is read as 1, silently, in the chunk grid (regular, and the explicit edges and run-length encoded sizes of a rectilinear one) and in the inner chunk shape of every sharding codec, nested or not. A stored rectilinear chunk grid whose edge lengths are integral JSON floats (`[[4.0, 2]]`), as zarr-python 3.2 wrote for float edges, is read with those edges as integers, silently; a float anywhere no release wrote one (a regular chunk shape, the inner chunk shape of a sharding codec, a run-length repeat count, an edge below 1) is rejected.
 
-Writing data to an array whose stored metadata holds a regular chunk size of 0 or `false` (other than an empty selection) first stores the upgrade of the metadata the store then holds, unless another writer has stored valid metadata since, so that other readers find the chunks written (a `true` chunk size or an integral float edge is read as the value it equals, so a write stores no metadata for it); in a `ZipStore` this adds a second entry for the metadata document, as every metadata update does. If the metadata the store then holds lays out chunks differently (another writer stored a different chunk size since, say), the write raises a `ValueError` asking to reopen the array, and stores nothing. Apart from the array's own metadata writes (`update_attributes`, `resize`), which store the upgrade as before, this is the only operation that stores it: a group's consolidated metadata keeps such an array's metadata as it was stored (`zarr.consolidate_metadata` copies it as the array's own document holds it) until the array stores its upgrade through the group's handle, and changing a group reads and writes no metadata of its members.
+Writing data to an array whose stored metadata holds a regular chunk size of 0 or `false` (other than an empty selection) first stores the repair of the metadata the store then holds, unless another writer has stored valid metadata since, so that other readers find the chunks written (a `true` chunk size or an integral float edge is read as the value it equals, so a write stores no metadata for it); in a `ZipStore` this adds a second entry for the metadata document, as every metadata update does. If the metadata the store then holds lays out chunks differently (another writer stored a different chunk size since, say), the write raises a `ValueError` asking to reopen the array, and stores nothing. Apart from the array's own metadata writes (`update_attributes`, `resize`), which store the repair as before, this is the only operation that stores it: a group's consolidated metadata keeps such an array's metadata as it was stored (`zarr.consolidate_metadata` copies it as the array's own document holds it) until the array stores its repair through the group's handle, and changing a group reads and writes no metadata of its members.

src/zarr/core/array.py

@@ -214,7 +214,7 @@ def parse_array_metadata(data: Any, path: str | None = None) -> ArrayMetadata:
 
     `ArrayV2Metadata` accepts a chunk size of 0, as it always has, though only an
     invalid document holds one: such metadata is read as the documents it would store
-    are (see `zarr.core.metadata.upgrades`), so an array can be built from it. No data
+    are (see `zarr.core.metadata.repair`), so an array can be built from it. No data
     was read or written under that chunk size, so the reading is silent."""
     if isinstance(data, ArrayV2Metadata) and 0 in data.chunks:
         return parse_stored_array(data.to_buffer_dict(default_buffer_prototype()), 2)
@@ -1613,28 +1613,28 @@ async def _save_metadata(self, metadata: ArrayMetadata, ensure_parents: bool = F
 
     def _stored_document_replaced(self) -> None:
         """Record that the store no longer holds a document of this array that needs an
-        upgrade: it holds the upgrade, a valid document, or none. The metadata this handle
+        repair: it holds the repair, a valid document, or none. The metadata this handle
         holds, which a consolidated group handle may share, then stops standing for the
-        document it was read from (see `mark_upgraded`), so no later write through either
+        document it was read from (see `mark_repaired`), so no later write through either
         handle stores that document again."""
         object.__setattr__(self.metadata, "_stored_document", None)
 
-    async def _store_upgraded_document(self) -> None:
-        """Store the upgrade of this array's current stored document, if it needs one,
+    async def _store_repaired_document(self) -> None:
+        """Store the repair of this array's current stored document, if it needs one,
         before chunks are written under this handle's metadata.
 
-        Only for metadata read from a document whose upgrade moves chunks (see
-        `zarr.core.metadata.upgrades`). The document is read again, because the store may
+        Only for metadata read from a document whose repair moves chunks (see
+        `zarr.core.metadata.repair`). The document is read again, because the store may
         hold a newer one than this handle's metadata. If that one lays out chunks
         differently (another writer stored a different chunk size since), this handle
         would write chunks no reader finds, so it raises and stores nothing. If it needs
-        no upgrade (the array was re-saved since, possibly by another implementation), it
-        is left as written; if there is none, there is nothing to upgrade.
+        no repair (the array was re-saved since, possibly by another implementation), it
+        is left as written; if there is none, there is nothing to repair.
 
-        Storing the same upgrade twice is harmless, so handles that write chunks
+        Storing the same repair twice is harmless, so handles that write chunks
         concurrently need no coordination. The read and the store are not one atomic
         step, though: a metadata write by another handle between them (a resize, an
-        attribute update) is replaced by the upgrade of the document read before it, as
+        attribute update) is replaced by the repair of the document read before it, as
         with any two metadata writes that race.
         """
         if self.metadata._stored_document is None:
@@ -1664,9 +1664,9 @@ async def _set_selection(
         fields: Fields | None = None,
     ) -> None:
         if product(indexer.shape) > 0:
-            # Chunks are about to be stored under the upgraded metadata, so store it
+            # Chunks are about to be stored under the repaired metadata, so store it
             # first: every reader of the store then agrees with them.
-            await self._store_upgraded_document()
+            await self._store_repaired_document()
         return await _set_selection(
             self.store_path,
             self.metadata,

src/zarr/core/common.py

@@ -290,7 +290,7 @@ def _subject(name: str, axis: int | None) -> str:
 def _parse_positive_int(value: object, name: str, axis: int | None) -> int:
     """`value` as an `int` of at least 1. A `bool` is read as the `int` it equals; any
     other type, a NumPy integer or a float (even an integral one: stored documents with
-    integral floats are read by `zarr.core.metadata.upgrades`), is rejected."""
+    integral floats are read by `zarr.core.metadata.repair`), is rejected."""
     subject = _subject(name, axis)
     if not isinstance(value, int):
         raise TypeError(f"{subject} must be an int, got {value!r}")

src/zarr/core/group.py

@@ -150,8 +150,8 @@ class ConsolidatedMetadata:
 
     def to_dict(self) -> dict[str, JSON]:
         """The consolidated metadata document. An array read from a stored document that
-        had to be upgraded is written as that document was stored, so every reader of
-        the consolidated metadata reads it as upgraded again (see `mark_upgraded`)."""
+        had to be repaired is written as that document was stored, so every reader of
+        the consolidated metadata reads it as repaired again (see `mark_repaired`)."""
         return {
             "kind": self.kind,
             "must_understand": self.must_understand,

src/zarr/core/metadata/io.py

@@ -10,7 +10,7 @@
 from zarr.core.buffer.core import default_buffer_prototype
 from zarr.core.buffer.cpu import buffer_prototype as cpu_buffer_prototype
 from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON
-from zarr.core.metadata.upgrades import mark_upgraded, upgrade_array_document
+from zarr.core.metadata.repair import mark_repaired, repair_array_document
 from zarr.errors import ArrayNotFoundError, ContainsArrayError
 from zarr.storage._common import StorePath, ensure_no_existing_node
 
@@ -116,9 +116,9 @@ async def upsert_metadata(
 
 def parse_stored_array(documents: Mapping[str, Buffer], zarr_format: ZarrFormat) -> ArrayMetadata:
     """The metadata of an array from its documents (by store key, see `ARRAY_DOCUMENTS`),
-    read with the upgrades but without their warnings (whoever asks has warned, or reads
-    metadata built in code), and marked (see `mark_upgraded`) if they had to be
-    upgraded. Raises `ArrayNotFoundError` if there is no array document among them."""
+    read with the repairs but without their warnings (whoever asks has warned, or reads
+    metadata built in code), and marked (see `mark_repaired`) if they had to be
+    repaired. Raises `ArrayNotFoundError` if there is no array document among them."""
     from zarr.core.array import (
         _array_metadata_dict_v2,
         _array_metadata_dict_v3,
@@ -131,8 +131,8 @@ def parse_stored_array(documents: Mapping[str, Buffer], zarr_format: ZarrFormat)
         stored = _array_metadata_dict_v3(documents[ZARR_JSON])
     else:
         raise ArrayNotFoundError(f"No Zarr format {zarr_format} array metadata document.")
-    upgraded, readings = upgrade_array_document(stored, zarr_format)
-    return mark_upgraded(parse_array_metadata(dict(upgraded)), stored, readings, None, warn=False)
+    repaired, readings = repair_array_document(stored, zarr_format)
+    return mark_repaired(parse_array_metadata(dict(repaired)), stored, readings, None, warn=False)
 
 
 def _build_parents(store_path: StorePath, zarr_format: ZarrFormat) -> dict[str, GroupMetadata]:

src/zarr/core/metadata/upgrades.py src/zarr/core/metadata/repair.pysrc/zarr/core/metadata/upgrades.py renamed to src/zarr/core/metadata/repair.py

@@ -1,21 +1,21 @@
-"""Upgrades that read invalid stored array metadata documents written by older software.
+"""Repairs that read invalid stored array metadata documents written by older software.
 
-This is the only place invalid metadata is read leniently. An upgrade maps a stored
+This is the only place invalid metadata is read leniently. A repair maps a stored
 array metadata document (parsed JSON) to a valid one. `ArrayV2Metadata.from_dict` and
-`ArrayV3Metadata.from_dict` apply the upgrades for their Zarr format, so every path
+`ArrayV3Metadata.from_dict` apply the repairs for their Zarr format, so every path
 that parses a stored document, including consolidated metadata, goes through them.
 
 A reading warns only where the user must act on it; a reading that gives what zarr
-read from the same document before these upgrades existed is silent, so a document
+read from the same document before these repairs existed is silent, so a document
 that opened without a warning still does. The warnings are given once per document,
-after the upgraded document has passed the metadata constructor, so an invalid
+after the repaired document has passed the metadata constructor, so an invalid
 document raises its own error, not a warning about how it was read. Metadata read
-from a document whose upgrade moves chunks (a chunk size read as another size) is
-marked (see `mark_upgraded`), silent or not, so the array stores the upgrade before it
-writes chunks under it. An upgrade that only respells a value zarr already read the
+from a document whose repair moves chunks (a chunk size read as another size) is
+marked (see `mark_repaired`), silent or not, so the array stores the repair before it
+writes chunks under it. A repair that only respells a value zarr already read the
 same way (`true` as 1, `4.0` as 4) moves no chunks, so it is not marked.
 
-To read another kind of invalid document, add an upgrade to `ARRAY_UPGRADES`.
+To read another kind of invalid document, add a repair to `ARRAY_REPAIRS`.
 """
 
 from __future__ import annotations
@@ -36,18 +36,18 @@
 
 
 class Reading(NamedTuple):
-    """How an upgrade read a stored document."""
+    """How a repair read a stored document."""
 
     moves_chunks: bool
-    """Whether chunks written under the upgraded document are not where a reader of the
-    stored document looks for them, so the array must store the upgrade before it
+    """Whether chunks written under the repaired document are not where a reader of the
+    stored document looks for them, so the array must store the repair before it
     writes chunks."""
     warning: str | None
     """What the user must act on, if anything."""
 
 
-type Upgrade = Callable[[ArrayDocument], tuple[ArrayDocument, Reading] | None]
-"""Returns `None` if the document needs no upgrade, else the upgraded document and how
+type Repair = Callable[[ArrayDocument], tuple[ArrayDocument, Reading] | None]
+"""Returns `None` if the document needs no repair, else the repaired document and how
 it was read."""
 
 RECREATE_HINT: Final = (
@@ -64,7 +64,7 @@ class Reading(NamedTuple):
 and a re-save would keep it."""
 
 
-def mark_upgraded[M](
+def mark_repaired[M](
     metadata: M,
     stored: ArrayDocument,
     readings: Sequence[Reading],
@@ -73,9 +73,9 @@ def mark_upgraded[M](
     warn: bool = True,
 ) -> M:
     """Record that `metadata` was read from the document `stored`, which needed the
-    upgrades whose `readings` `upgrade_array_document` returned, if any. If a reading
+    repairs whose `readings` `repair_array_document` returned, if any. If a reading
     moves chunks, keep a copy of `stored` as `_stored_document` on `metadata`, so the
-    array stores the upgrade before it writes chunks under it and consolidated metadata
+    array stores the repair before it writes chunks under it and consolidated metadata
     stores it as it was stored. If `warn`, warn once with the readings' warnings (each
     says what the user must act on and how), naming the array at `path` when the caller
     knows it."""
@@ -142,7 +142,7 @@ def _read_chunk_shape(
     Returns the chunk shape and how it was read, if any entry was read as another value
     (the warning says how the chunk shape was read and, as the array then holds only its
     fill value, recommends recreating it, see `RECREATE_HINT`); `None` if it cannot be
-    read or needs no upgrade.
+    read or needs no repair.
     """
     if not (isinstance(stored, list) and len(stored) == len(spans)):
         return None
@@ -190,22 +190,22 @@ def _read_codec(codec: JSON) -> JSON | None:
     no chunks."""
     match codec:
         case {"name": "sharding_indexed", "configuration": Mapping() as configuration}:
-            upgraded: dict[str, JSON] = {}
+            repaired: dict[str, JSON] = {}
             stored = configuration.get("chunk_shape")
             if isinstance(stored, list) and (
                 read := _read_chunk_shape(stored, [None] * len(stored))
             ):
-                upgraded["chunk_shape"] = read[0]
+                repaired["chunk_shape"] = read[0]
             if isinstance(codecs := configuration.get("codecs"), list) and (
                 inner := _read_codecs(codecs)
             ):
-                upgraded["codecs"] = inner
-            if not upgraded:
+                repaired["codecs"] = inner
+            if not repaired:
                 return None
             # The mapping pattern does not narrow `codec` for mypy.
             return {
                 **cast("Mapping[str, JSON]", codec),
-                "configuration": {**configuration, **upgraded},
+                "configuration": {**configuration, **repaired},
             }
     return None
 
@@ -251,8 +251,8 @@ def _invalid_chunk_sizes_v3(doc: ArrayDocument) -> tuple[ArrayDocument, Reading]
         return None
     match _read_chunk_shape(configuration.get("chunk_shape"), shape, units):
         case chunk_shape, reading:
-            upgraded = {**configuration, "chunk_shape": chunk_shape}
-            return {**doc, "chunk_grid": {**grid, "configuration": upgraded}}, reading
+            repaired = {**configuration, "chunk_shape": chunk_shape}
+            return {**doc, "chunk_grid": {**grid, "configuration": repaired}}, reading
     return None
 
 
@@ -308,33 +308,33 @@ def _invalid_edge_lengths_v3(doc: ArrayDocument) -> tuple[ArrayDocument, Reading
     read = [_read_rectilinear_axis(axis) for axis in stored]
     if not _respelled(read, stored):
         return None
-    upgraded = {**configuration, "chunk_shapes": read}
+    repaired = {**configuration, "chunk_shapes": read}
     # A stored edge is read as the value it equals, so the reading moves no chunks.
-    return {**doc, "chunk_grid": {**grid, "configuration": upgraded}}, Reading(False, None)
+    return {**doc, "chunk_grid": {**grid, "configuration": repaired}}, Reading(False, None)
 
 
-ARRAY_UPGRADES: Final[Mapping[ZarrFormat, tuple[Upgrade, ...]]] = {
+ARRAY_REPAIRS: Final[Mapping[ZarrFormat, tuple[Repair, ...]]] = {
     2: (_invalid_chunk_sizes_v2,),
     # The inner chunk shape is read first: it gives the unit of the outer chunk shape.
-    # The rectilinear edge lengths are read last, after any upgrade that yields a
+    # The rectilinear edge lengths are read last, after any repair that yields a
     # rectilinear chunk grid.
     3: (_invalid_inner_chunk_sizes_v3, _invalid_chunk_sizes_v3, _invalid_edge_lengths_v3),
 }
-"""The upgrades of an array document of each Zarr format, applied in order."""
+"""The repairs of an array document of each Zarr format, applied in order."""
 
 
-def upgrade_array_document(
+def repair_array_document(
     doc: ArrayDocument, zarr_format: ZarrFormat
 ) -> tuple[ArrayDocument, list[Reading]]:
-    """Apply the upgrades for `zarr_format` to a stored array metadata document.
+    """Apply the repairs for `zarr_format` to a stored array metadata document.
 
-    Returns the upgraded document and the reading of each upgrade that changed it, for
-    `mark_upgraded` once the document has been validated.
+    Returns the repaired document and the reading of each repair that changed it, for
+    `mark_repaired` once the document has been validated.
     """
     readings: list[Reading] = []
-    for upgrade in ARRAY_UPGRADES[zarr_format]:
-        upgraded = upgrade(doc)
-        if upgraded is not None:
-            doc, reading = upgraded
+    for repair in ARRAY_REPAIRS[zarr_format]:
+        repaired = repair(doc)
+        if repaired is not None:
+            doc, reading = repaired
             readings.append(reading)
     return doc, readings