@@ -284,6 +284,19 @@ def dataset_create_kwargs(
284284 kw ["scaleoffset" ] = src .scaleoffset
285285 if src .fillvalue is not None :
286286 kw ["fillvalue" ] = src .fillvalue
287+ if (
288+ target_backend == "hdf5"
289+ and is_zarr_array (src )
290+ and chunks is not None
291+ and tuple (getattr (src , "shape" , ())) != ()
292+ and not _is_string_src (src )
293+ and _zarr_is_compressed (src )
294+ ):
295+ # Zarr codecs have no HDF5 equivalent, but compression itself does.
296+ # Without this a compressed Zarr store came out of every command
297+ # uncompressed, several times its size.
298+ kw ["compression" ] = "gzip"
299+ kw ["compression_opts" ] = GZIP_LEVEL
287300 if target_backend == "zarr" and is_zarr_array (src ):
288301 src_zarr_format = getattr (getattr (src , "metadata" , None ), "zarr_format" , None )
289302 target_format = _target_zarr_format (kw_target )
@@ -424,6 +437,68 @@ def create_dataset(
424437 return parent .create_dataset (name , shape = shape , dtype = dtype , ** kwargs )
425438
426439
440+ #: gzip level for an HDF5 copy of a compressed Zarr array: h5py's default,
441+ #: and what Cell Ranger and most other HDF5 writers use.
442+ GZIP_LEVEL = 4
443+
444+
445+ def _zarr_is_compressed (src : Any ) -> bool :
446+ """Does this Zarr array carry a compressor? (`compressors` covers v2 too.)"""
447+ try :
448+ return bool (getattr (src , "compressors" , None ))
449+ except Exception :
450+ return False
451+
452+
453+ def fit_chunks (kw : dict , shape : Sequence [int ], * , backend : str ) -> dict :
454+ """Make a forwarded chunk shape legal for a dataset of `shape`.
455+
456+ h5py rejects a chunk larger than the dataset, so a chunked source copied
457+ or subset below its own chunk size would otherwise fail outright. A
458+ zero-length dimension cannot be fitted at all; HDF5 is then left to pick
459+ its own chunking, which it does whenever compression asks for one. Zarr
460+ accepts a chunk larger than its array, so there it only has to shrink.
461+
462+ Zarr additionally requires a shard to be a whole number of chunks, so a
463+ clamped chunk invalidates the source's shard geometry. Sharding is a
464+ storage-layout choice rather than data, so it is dropped and left to the
465+ backend rather than recomputed into something arbitrary.
466+ """
467+ chunks = kw .get ("chunks" )
468+ if not isinstance (chunks , (tuple , list )) or not chunks :
469+ return kw
470+
471+ original = tuple (int (c ) for c in chunks )
472+ dims = [int (n ) for n in shape ]
473+ if backend == "hdf5" and any (n == 0 for n in dims [: len (original )]):
474+ kw = dict (kw )
475+ kw .pop ("chunks" )
476+ kw .pop ("shards" , None )
477+ return kw
478+
479+ clamped = tuple (
480+ min (c , dims [i ]) if i < len (dims ) and dims [i ] > 0 else c
481+ for i , c in enumerate (original )
482+ )
483+ if clamped == original :
484+ return kw
485+
486+ kw = dict (kw )
487+ kw ["chunks" ] = clamped
488+ kw .pop ("shards" , None )
489+ return kw
490+
491+
492+ def _is_null (src : Any ) -> bool :
493+ """Is `src` an explicit null -- `h5py.Empty` on HDF5, or tagged `null`?"""
494+ if is_hdf5_dataset (src ) and src .shape is None :
495+ return True
496+ value = src .attrs .get ("encoding-type" , None )
497+ if isinstance (value , bytes ):
498+ value = value .decode ("utf-8" )
499+ return value == "null"
500+
501+
427502def _target_zarr_format (zarr_format : Optional [int ]) -> Optional [int ]:
428503 """The destination's Zarr version, or None when the caller did not say.
429504
@@ -570,17 +645,27 @@ def copy_dataset(src: Any, dst_group: Any, name: str) -> Any:
570645 """
571646 from adata .elements .strings import target_dtype
572647
573- shape = tuple (src .shape ) if getattr (src , "shape" , None ) is not None else ()
574648 target_backend = "zarr" if is_zarr_group (dst_group ) else "hdf5"
649+ if _is_null (src ):
650+ # anndata writes an absent element, `raw` most often, as a null: an
651+ # empty dataspace on HDF5 and a 0-d bool on Zarr. Neither can be read
652+ # and written back as a value -- `h5py.Empty` has none -- so it is
653+ # written afresh in the target's own spelling.
654+ from adata .elements .write import write_null
655+
656+ ds = write_null (dst_group , name )
657+ copy_attrs (src .attrs , ds .attrs , target_backend = target_backend )
658+ return ds
659+
660+ shape = tuple (src .shape )
575661 zformat = zarr_format_of (dst_group )
662+ kw = dataset_create_kwargs (src , target_backend = target_backend , zarr_format = zformat )
576663 ds = create_dataset (
577664 dst_group ,
578665 name ,
579666 shape = shape ,
580667 dtype = target_dtype (src .dtype , target_backend , zformat ),
581- ** dataset_create_kwargs (
582- src , target_backend = target_backend , zarr_format = zformat
583- ),
668+ ** fit_chunks (kw , shape , backend = target_backend ),
584669 )
585670 copy_attrs (src .attrs , ds .attrs , target_backend = target_backend )
586671
0 commit comments