chore: migrate to zarrs 0.24

A dependency bump and the API migration it forces, and nothing else. Kept on its own
because a dependency migration reviewed together with a behaviour change means neither can
be reverted alone.

All of it is one upstream idea -- the #411 refactor -- which is that a codec chain is BOUND
to a data type and fill value once, where it is built, instead of being handed both at every
call. `CodecChain` is now unbound; `decode`, `encode`, `decode_into`, `partial_decoder` and
`recommended_concurrency` live on `CodecChainBound`, reached by `with_context`. So the chain
is bound in the constructor and five call sites stop passing a pair they no longer need.

`zarrs::array::StoragePartialDecoder` is also gone, and nothing named replaced it: the
(storage, key) TUPLE is the store-backed BytesPartialDecoderTraits implementation now.

No behaviour change, with one exception worth naming rather than hiding: a codec chain that
cannot BIND to its data type now fails in the constructor as a TypeError, where before it
surfaced at the first read as a RuntimeError. Binding has to happen somewhere, and the
constructor is the only place it can. The codec metadata is still PARSED where it was, so
an array with both bad codecs and a bad fill value still reports the codecs.

The dependency is a git rev rather than a version, because 0.24 is not released yet, and it
carries a [patch.crates-io] block for zarrs_storage. That patch is load-bearing, not
cosmetic: zarrs_opendal and zarrs_object_store track RELEASED zarrs, so they pull
zarrs_storage from crates.io while zarrs comes from git -- two copies of one crate, two
distinct AsyncReadableStorageTraits, and trait bounds that cannot be satisfied. Patching
that single crate collapses the graph; every other zarrs crate already resolves through the
git checkout, verified from a clean lockfile. Both the pin and the patch are commented with
exactly what to do on release day, and nothing in src/ changes then.

Author: selmanozleyen

Cargo.toml

@@ -10,7 +10,11 @@ crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 pyo3 = { version = "0.27.1", features = ["abi3-py311"] }
-zarrs = { version = "0.23.6", features = ["async", "zlib", "pcodec", "bz2"] }
+# ON THE 0.24 RELEASE: replace with `zarrs = { version = "0.24", features = [...] }` AND
+# delete the [patch.crates-io] block below, in the SAME commit -- see the note there for why
+# doing only one of the two builds silently against the wrong `zarrs_storage`. src/ already
+# compiles against the 0.24 API.
+zarrs = { git = "https://github.com/zarrs/zarrs", rev = "c17fe374b1fa7df8373b6c6f6eb3f1d33c3a3bd7", features = ["async", "zlib", "pcodec", "bz2"] }
 rayon_iter_concurrent_limit = "0.2.0"
 rayon = "1.10.0"
 # fix for https://stackoverflow.com/questions/76593417/package-openssl-was-not-found-in-the-pkg-config-search-path
@@ -29,3 +33,16 @@ zarrs_object_store = "0.5.0" # object_store 0.12
 
 [profile.release]
 lto = true
+
+# ON THE 0.24 RELEASE: delete this, in the same commit that drops the git rev above.
+#
+# `zarrs_opendal` and `zarrs_object_store` do not depend on `zarrs` at all -- only on
+# `zarrs_storage`, from crates.io. While `zarrs` comes from git that is a SECOND copy of
+# `zarrs_storage`, so there are two `AsyncReadableStorageTraits` and the bounds on the async
+# stores cannot be satisfied. Patching the one crate to the same rev collapses the graph.
+#
+# Drop the git rev above WITHOUT deleting this and the build is silently wrong: released
+# `zarrs 0.24` compiled against an unpublished `zarrs_storage`. Nothing warns, because the
+# git copy carries the same version number as the published one with different contents.
+[patch.crates-io]
+zarrs_storage = { git = "https://github.com/zarrs/zarrs", rev = "c17fe374b1fa7df8373b6c6f6eb3f1d33c3a3bd7" }

src/concurrency.rs

@@ -25,7 +25,7 @@ impl ChunkConcurrentLimitAndCodecOptions for Vec<ChunkItem> {
 
         let codec_concurrency = codec_pipeline_impl
             .codec_chain
-            .recommended_concurrency(&item.shape, &codec_pipeline_impl.data_type)
+            .recommended_concurrency(&item.shape)
             .map_codec_err()?;
 
         let min_concurrent_chunks =

src/lib.rs

@@ -18,10 +18,11 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use rayon_iter_concurrent_limit::iter_concurrent_limit;
 use unsafe_cell_slice::UnsafeCellSlice;
 use utils::is_whole_chunk;
+use zarrs::array::codec::api::BytesPartialDecoderTraits;
 use zarrs::array::{
     ArrayBytes, ArrayBytesDecodeIntoTarget, ArrayBytesFixedDisjointView, ArrayMetadata,
-    ArrayPartialDecoderTraits, ArrayToBytesCodecTraits, CodecChain, CodecOptions, DataType,
-    FillValue, StoragePartialDecoder, copy_fill_value_into, update_array_bytes,
+    ArrayPartialDecoderTraits, ArrayToBytesCodecTraits, CodecChain, CodecChainBound, CodecOptions,
+    DataType, FillValue, copy_fill_value_into, update_array_bytes,
 };
 use zarrs::config::global_config;
 use zarrs::convert::array_metadata_v2_to_v3;
@@ -45,7 +46,7 @@ use crate::utils::{PyCodecErrExt, PyErrExt as _};
 #[pyclass]
 pub struct CodecPipelineImpl {
     pub(crate) store: ReadableWritableListableStorage,
-    pub(crate) codec_chain: Arc<CodecChain>,
+    pub(crate) codec_chain: Arc<CodecChainBound>,
     pub(crate) codec_options: CodecOptions,
     pub(crate) chunk_concurrent_minimum: usize,
     pub(crate) chunk_concurrent_maximum: usize,
@@ -58,20 +59,14 @@ impl CodecPipelineImpl {
     fn retrieve_chunk_bytes<'a>(
         &self,
         item: &ChunkItem,
-        codec_chain: &CodecChain,
+        codec_chain: &CodecChainBound,
         codec_options: &CodecOptions,
     ) -> PyResult<ArrayBytes<'a>> {
         let value_encoded = self.store.get(&item.key).map_py_err::<PyRuntimeError>()?;
         let value_decoded = if let Some(value_encoded) = value_encoded {
             let value_encoded: Vec<u8> = value_encoded.into(); // zero-copy in this case
             codec_chain
-                .decode(
-                    value_encoded.into(),
-                    &item.shape,
-                    &self.data_type,
-                    &self.fill_value,
-                    codec_options,
-                )
+                .decode(value_encoded.into(), &item.shape, codec_options)
                 .map_codec_err()?
         } else {
             ArrayBytes::new_fill_value(&self.data_type, item.num_elements, &self.fill_value)
@@ -83,7 +78,7 @@ impl CodecPipelineImpl {
     fn store_chunk_bytes(
         &self,
         item: &ChunkItem,
-        codec_chain: &CodecChain,
+        codec_chain: &CodecChainBound,
         value_decoded: ArrayBytes,
         codec_options: &CodecOptions,
     ) -> PyResult<()> {
@@ -95,13 +90,7 @@ impl CodecPipelineImpl {
             self.store.erase(&item.key).map_py_err::<PyRuntimeError>()
         } else {
             let value_encoded = codec_chain
-                .encode(
-                    value_decoded,
-                    &item.shape,
-                    &self.data_type,
-                    &self.fill_value,
-                    codec_options,
-                )
+                .encode(value_decoded, &item.shape, codec_options)
                 .map(Cow::into_owned)
                 .map_codec_err()?;
 
@@ -115,7 +104,7 @@ impl CodecPipelineImpl {
     fn store_chunk_subset_bytes(
         &self,
         item: &ChunkItem,
-        codec_chain: &CodecChain,
+        codec_chain: &CodecChainBound,
         chunk_subset_bytes: ArrayBytes,
         codec_options: &CodecOptions,
     ) -> PyResult<()> {
@@ -240,8 +229,10 @@ impl CodecPipelineImpl {
             }
             ArrayMetadata::V3(v3) => Cow::Borrowed(v3),
         };
+        // Parsed before binding, so an array with bad codecs and a bad fill value still
+        // reports the codecs.
         let codec_chain =
-            Arc::new(CodecChain::from_metadata(&metadata_v3.codecs).map_py_err::<PyTypeError>()?);
+            CodecChain::from_metadata(&metadata_v3.codecs).map_py_err::<PyTypeError>()?;
         let codec_options = CodecOptions::default().with_validate_checksums(validate_checksums);
 
         let chunk_concurrent_minimum =
@@ -271,6 +262,10 @@ impl CodecPipelineImpl {
             })
             .map_py_err::<PyTypeError>()?;
 
+        let codec_chain = codec_chain
+            .with_context(data_type.clone(), fill_value.clone())
+            .map_py_err::<PyTypeError>()?;
+
         Ok(Self {
             store,
             codec_chain,
@@ -310,18 +305,13 @@ impl CodecPipelineImpl {
         if !partial_chunk_items.is_empty() {
             let key_decoder_pairs =
                 iter_concurrent_limit!(chunk_concurrent_limit, partial_chunk_items, map, |item| {
-                    let storage_handle = Arc::new(StorageHandle::new(self.store.clone()));
-                    let input_handle = StoragePartialDecoder::new(storage_handle, item.key.clone());
+                    // The (storage, key) tuple IS the store-backed `BytesPartialDecoderTraits`.
+                    let input_handle: Arc<dyn BytesPartialDecoderTraits> =
+                        Arc::new((StorageHandle::new(self.store.clone()), item.key.clone()));
                     let partial_decoder = self
                         .codec_chain
                         .clone()
-                        .partial_decoder(
-                            Arc::new(input_handle),
-                            &item.shape,
-                            &self.data_type,
-                            &self.fill_value,
-                            &codec_options,
-                        )
+                        .partial_decoder(input_handle, &item.shape, &codec_options)
                         .map_codec_err()?;
                     Ok((item.key.clone(), partial_decoder))
                 })
@@ -362,8 +352,6 @@ impl CodecPipelineImpl {
                         self.codec_chain.decode_into(
                             Cow::Owned(chunk_encoded),
                             &item.shape,
-                            &self.data_type,
-                            &self.fill_value,
                             target,
                             &codec_options,
                         )