Merge branch 'main' into fix/strict-chunk-sizes

Resolves the import conflict in group.py with zarr-developers#4391: keep both import lines.

Assisted-by: ClaudeCode:claude-fable-5-1
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: d-v-b

changes/4391.bugfix.md

@@ -0,0 +1 @@
+Zarr format 3 group metadata now accepts extra fields that are objects with `"must_understand": false`, as the specification requires, and writes them back unchanged; previously opening such a group raised `TypeError`. Extra fields of any other shape raise `MetadataValidationError`, both when reading a metadata document and when constructing `GroupMetadata` or `ArrayV3Metadata` directly. `GroupMetadata` exposes these fields as `extra_fields`, like `ArrayV3Metadata`.

changes/4413.bugfix.md

@@ -0,0 +1 @@
+`Group.require_array` now applies the `config` argument when it returns an existing array. Previously `config` was only used when `require_array` created a new array, so an existing array was returned with the global defaults. An invalid `config` now raises instead of being ignored.

changes/4415.bugfix.md

@@ -0,0 +1 @@
+`zarr.save` now takes `storage_options` as an option instead of treating it as one of the arrays to save: saving a single array with `storage_options` set no longer produces a group holding `arr_0`, and the options now reach the store. `zarr.save_group` no longer fails with `'storage_options' was provided but unused` when the arrays are passed positionally.

changes/4417.misc.md

@@ -0,0 +1 @@
+Fix `test_docstring_consistent_parameters`, which silently passed because it looked up a tuple of parameter names as a single key and concatenated two names into one. The test now checks every parameter documented by the `create_array` variants (`zarr.create_array`, `AsyncGroup.create_array`, `Group.create_array` and `Group.create`), and their docstrings were aligned: `shape`, `dtype`, `data`, `chunks`, `chunk_key_encoding`, `config` and `overwrite` now document the same types and text, `AsyncGroup.create_array` documents `data`, and typos in the chunk key encoding examples, the variable-length data type list and the Zarr format 2 default compressor description are corrected.

changes/4452.bugfix.md

@@ -0,0 +1 @@
+Fixed `FsspecStore.list_dir` and `FsspecStore.list_prefix` returning nothing below the root of a store whose path is `""` or `"/"`, such as a store opened from `"reference://"`. Nested groups in kerchunk-style references now list their members.

src/zarr/api/asynchronous.py

@@ -435,6 +435,7 @@ async def save(
     *args: NDArrayLike,
     zarr_format: ZarrFormat | None = None,
     path: str | None = None,
+    storage_options: dict[str, Any] | None = None,
     **kwargs: Any,  # TODO: type kwargs as valid args to save
 ) -> None:
     """Convenience function to save an array or group of arrays to the local file system.
@@ -451,16 +452,28 @@ async def save(
         The zarr format to use when saving.
     path : str or None, optional
         The path within the group where the arrays will be saved.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
     **kwargs
         NumPy arrays with data to save.
     """
 
     if len(args) == 0 and len(kwargs) == 0:
         raise ValueError("at least one array must be provided")
     if len(args) == 1 and len(kwargs) == 0:
-        await save_array(store, args[0], zarr_format=zarr_format, path=path)
+        await save_array(
+            store, args[0], zarr_format=zarr_format, path=path, storage_options=storage_options
+        )
     else:
-        await save_group(store, *args, zarr_format=zarr_format, path=path, **kwargs)
+        await save_group(
+            store,
+            *args,
+            zarr_format=zarr_format,
+            path=path,
+            storage_options=storage_options,
+            **kwargs,
+        )
 
 
 async def save_array(
@@ -566,16 +579,10 @@ async def save_group(
     if len(args) == 0 and len(kwargs) == 0:
         raise ValueError("at least one array must be provided")
     aws = []
+    # `store_path` already consumed `storage_options`, so passing them on again would
+    # make `make_store_path` reject them as unused.
     for i, arr in enumerate(args):
-        aws.append(
-            save_array(
-                store_path,
-                arr,
-                zarr_format=zarr_format,
-                path=f"arr_{i}",
-                storage_options=storage_options,
-            )
-        )
+        aws.append(save_array(store_path, arr, zarr_format=zarr_format, path=f"arr_{i}"))
     for k, arr in kwargs.items():
         aws.append(save_array(store_path, arr, zarr_format=zarr_format, path=k))
     await asyncio.gather(*aws)
@@ -974,7 +981,7 @@ async def create(
         type of the array and the Zarr format specified. For all data types in Zarr V3, and most
         data types in Zarr V2, the default filters are empty. The only cases where default filters
         are not empty is when the Zarr format is 2, and the data type is a variable-length data type like
-        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthUTF8`][]. In these cases,
+        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases,
         the default filters contains a single element which is a codec specific to that particular data type.
 
         To create an array with no filters, provide an empty iterable or the value `None`.

src/zarr/api/synchronous.py

@@ -259,6 +259,7 @@ def save(
     *args: NDArrayLike,
     zarr_format: ZarrFormat | None = None,
     path: str | None = None,
+    storage_options: dict[str, Any] | None = None,
     **kwargs: Any,  # TODO: type kwargs as valid args to async_api.save
 ) -> None:
     """Save an array or group of arrays to the local file system.
@@ -275,10 +276,22 @@ def save(
         The zarr format to use when saving.
     path : str or None, optional
         The path within the group where the arrays will be saved.
+    storage_options : dict
+        If using an fsspec URL to create the store, these will be passed to
+        the backend implementation. Ignored otherwise.
     **kwargs
         NumPy arrays with data to save.
     """
-    return sync(async_api.save(store, *args, zarr_format=zarr_format, path=path, **kwargs))
+    return sync(
+        async_api.save(
+            store,
+            *args,
+            zarr_format=zarr_format,
+            path=path,
+            storage_options=storage_options,
+            **kwargs,
+        )
+    )
 
 
 def save_array(
@@ -712,7 +725,7 @@ def create(
         type of the array and the Zarr format specified. For all data types in Zarr V3, and most
         data types in Zarr V2, the default filters are empty. The only cases where default filters
         are not empty is when the Zarr format is 2, and the data type is a variable-length data type like
-        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthUTF8`][]. In these cases,
+        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases,
         the default filters contains a single element which is a codec specific to that particular data type.
 
         To create an array with no filters, provide an empty iterable or the value `None`.
@@ -872,7 +885,6 @@ def create_array(
         chunk to bytes.
 
         For Zarr format 3, a "filter" is a codec that takes an array and returns an array,
-
         and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a
         dict representations of [`zarr.abc.codec.ArrayArrayCodec`][].
 
@@ -883,7 +895,7 @@ def create_array(
         type of the array and the Zarr format specified. For all data types in Zarr V3, and most
         data types in Zarr V2, the default filters are empty. The only cases where default filters
         are not empty is when the Zarr format is 2, and the data type is a variable-length data type like
-        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthUTF8`][]. In these cases,
+        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases,
         the default filters contains a single element which is a codec specific to that particular data type.
 
         To create an array with no filters, provide an empty iterable or the value `None`.
@@ -901,6 +913,7 @@ def create_array(
         For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may
         be provided for Zarr format 2.
         If no `compressor` is provided, a default compressor will be used.
+        This default can be changed by modifying the value of `array.v2_default_compressor`
         in [`zarr.config`][zarr.config].
         Use `None` to omit the default compressor.
     serializer : dict[str, JSON] | ArrayBytesCodec, optional
@@ -925,8 +938,8 @@ def create_array(
         Attributes for the array.
     chunk_key_encoding : ChunkKeyEncodingLike, optional
         A specification of how the chunk keys are represented in storage.
-        For Zarr format 3, the default is `{"name": "default", "separator": "/"}}`.
-        For Zarr format 2, the default is `{"name": "v2", "separator": "."}}`.
+        For Zarr format 3, the default is `{"name": "default", "separator": "/"}`.
+        For Zarr format 2, the default is `{"name": "v2", "separator": "."}`.
     dimension_names : Iterable[str], optional
         The names of the dimensions (default is None).
         Zarr format 3 only. Zarr format 2 arrays should not use this parameter.
@@ -935,7 +948,7 @@ def create_array(
         Ignored otherwise.
     overwrite : bool, default False
         Whether to overwrite an array with the same name in the store, if one exists.
-        If `True`, all existing paths in the store will be deleted.
+        If `True`, any existing keys under that path are deleted first.
     config : ArrayConfigLike, optional
         Runtime configuration for the array.
     write_data : bool
@@ -1126,8 +1139,8 @@ def from_array(
         Pass an empty dict to create the array with no attributes.
     chunk_key_encoding : ChunkKeyEncoding, optional
         A specification of how the chunk keys are represented in storage.
-        For Zarr format 3, the default is `{"name": "default", "separator": "/"}}`.
-        For Zarr format 2, the default is `{"name": "v2", "separator": "."}}`.
+        For Zarr format 3, the default is `{"name": "default", "separator": "/"}`.
+        For Zarr format 2, the default is `{"name": "v2", "separator": "."}`.
         If not specified and the data array has the same zarr format as the target array,
         the chunk key encoding of the data array is used.
     dimension_names : Iterable[str | None] | None

src/zarr/core/array.py

@@ -4314,8 +4314,8 @@ async def from_array(
         Pass an empty dict to create the array with no attributes.
     chunk_key_encoding : ChunkKeyEncoding, optional
         A specification of how the chunk keys are represented in storage.
-        For Zarr format 3, the default is `{"name": "default", "separator": "/"}}`.
-        For Zarr format 2, the default is `{"name": "v2", "separator": "."}}`.
+        For Zarr format 3, the default is `{"name": "default", "separator": "/"}`.
+        For Zarr format 2, the default is `{"name": "v2", "separator": "."}`.
         If not specified and the data array has the same zarr format as the target array,
         the chunk key encoding of the data array is used.
     dimension_names : Iterable[str | None] | None
@@ -4512,7 +4512,7 @@ async def init_array(
         type of the array and the Zarr format specified. For all data types in Zarr V3, and most
         data types in Zarr V2, the default filters are empty. The only cases where default filters
         are not empty is when the Zarr format is 2, and the data type is a variable-length data type like
-        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthUTF8`][]. In these cases,
+        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases,
         the default filters contains a single element which is a codec specific to that particular data type.
 
         To create an array with no filters, provide an empty iterable or the value `None`.
@@ -4547,8 +4547,8 @@ async def init_array(
         Attributes for the array.
     chunk_key_encoding : ChunkKeyEncodingLike, optional
         A specification of how the chunk keys are represented in storage.
-        For Zarr format 3, the default is `{"name": "default", "separator": "/"}}`.
-        For Zarr format 2, the default is `{"name": "v2", "separator": "."}}`.
+        For Zarr format 3, the default is `{"name": "default", "separator": "/"}`.
+        For Zarr format 2, the default is `{"name": "v2", "separator": "."}`.
     dimension_names : Iterable[str], optional
         The names of the dimensions (default is None).
         Zarr format 3 only. Zarr format 2 arrays should not use this parameter.
@@ -4754,7 +4754,6 @@ async def create_array(
         chunk to bytes.
 
         For Zarr format 3, a "filter" is a codec that takes an array and returns an array,
-
         and these values must be instances of [`zarr.abc.codec.ArrayArrayCodec`][], or a
         dict representations of [`zarr.abc.codec.ArrayArrayCodec`][].
 
@@ -4765,7 +4764,7 @@ async def create_array(
         type of the array and the Zarr format specified. For all data types in Zarr V3, and most
         data types in Zarr V2, the default filters are empty. The only cases where default filters
         are not empty is when the Zarr format is 2, and the data type is a variable-length data type like
-        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthUTF8`][]. In these cases,
+        [`zarr.dtype.VariableLengthUTF8`][] or [`zarr.dtype.VariableLengthBytes`][]. In these cases,
         the default filters contains a single element which is a codec specific to that particular data type.
 
         To create an array with no filters, provide an empty iterable or the value `None`.
@@ -4783,6 +4782,7 @@ async def create_array(
         For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may
         be provided for Zarr format 2.
         If no `compressor` is provided, a default compressor will be used.
+        This default can be changed by modifying the value of `array.v2_default_compressor`
         in [`zarr.config`][zarr.config].
         Use `None` to omit the default compressor.
     serializer : dict[str, JSON] | ArrayBytesCodec, optional
@@ -4807,8 +4807,8 @@ async def create_array(
         Attributes for the array.
     chunk_key_encoding : ChunkKeyEncodingLike, optional
         A specification of how the chunk keys are represented in storage.
-        For Zarr format 3, the default is `{"name": "default", "separator": "/"}}`.
-        For Zarr format 2, the default is `{"name": "v2", "separator": "."}}`.
+        For Zarr format 3, the default is `{"name": "default", "separator": "/"}`.
+        For Zarr format 2, the default is `{"name": "v2", "separator": "."}`.
     dimension_names : Iterable[str], optional
         The names of the dimensions (default is None).
         Zarr format 3 only. Zarr format 2 arrays should not use this parameter.
@@ -4817,7 +4817,7 @@ async def create_array(
         Ignored otherwise.
     overwrite : bool, default False
         Whether to overwrite an array with the same name in the store, if one exists.
-        If `True`, all existing paths in the store will be deleted.
+        If `True`, any existing keys under that path are deleted first.
     config : ArrayConfigLike, optional
         Runtime configuration for the array.
     write_data : bool