11from __future__ import annotations
22
3- import bisect
43import itertools
54import math
65import operator
2423
2524import zarr
2625from zarr .core .common import (
26+ RunLengthEdges ,
2727 ShapeLike ,
2828 ceildiv_int ,
2929 parse_shapelike ,
@@ -130,63 +130,75 @@ def _size_repr(self) -> str:
130130 return str (self .size )
131131
132132
133- @dataclass (frozen = True )
133+ _MAX_EDGES_IN_REPR = 100
134+ """The most chunk edges of one dimension written out one by one in a repr or an error
135+ message; a longer dimension is shown run-length encoded."""
136+
137+
138+ def _edges_repr (edges : RunLengthEdges ) -> str :
139+ """The edges as a tuple, or run-length encoded if there are too many to write out."""
140+ if edges .num_edges <= _MAX_EDGES_IN_REPR :
141+ return repr (tuple (edges ))
142+ return repr (edges .to_rle ())
143+
144+
145+ @dataclass (frozen = True , init = False )
134146class VaryingDimension :
135147 """Explicit per-chunk sizes. The last chunk may extend past the array
136148 extent (``extent < sum(edges)``), in which case ``data_size`` clips to
137149 the valid region while ``chunk_size`` returns the full edge length for
138150 codec processing. This underflow is allowed to match how regular grids
139151 handle boundary chunks, and to support shrinking an array without
140- rewriting chunk edges (the spec allows trailing edges beyond the extent)."""
152+ rewriting chunk edges (the spec allows trailing edges beyond the extent).
153+
154+ The edges are held run-length encoded (see `RunLengthEdges`), so building
155+ the dimension, resizing it and every lookup cost time and memory in the
156+ number of runs of equal edges, not in the number of chunks: a dimension of
157+ ``2**40`` equal chunks is one run."""
141158
142- edges : tuple [int , ...] # per-chunk edge lengths (all > 0)
143- cumulative : tuple [int , ...] # prefix sums for O(log n) lookup
159+ edges : RunLengthEdges # per-chunk edge lengths (all > 0), run-length encoded
144160 extent : int # array dimension length (may be < sum(edges) after resize)
145161 nchunks : int = field (init = False , repr = False ) # cached at construction
146162 ngridcells : int = field (init = False , repr = False ) # cached at construction
147163
148- # TODO(perf): for long dimensions (O(million chunks)):
149- # - with_extent/resize recompute cumulative sums and nchunks from scratch;
150- # add a fast path that reuses the existing cumulative tuple.
151- # - Consider storing cumulative as ndarray so bisect calls can use
152- # np.searchsorted. Scalar lookups (chunk_offset, index_to_chunk)
153- # would need benchmarking to confirm no regression.
154164 def __init__ (self , edges : Sequence [int ], extent : int ) -> None :
155- edges_tuple = tuple (edges )
156- if not edges_tuple :
165+ rle = RunLengthEdges . from_edges (edges )
166+ if rle . num_edges == 0 :
157167 raise ValueError ("VaryingDimension edges must not be empty" )
158- if any (e <= 0 for e in edges_tuple ):
159- raise ValueError (f"All edge lengths must be > 0, got { edges_tuple } " )
160- cumulative = tuple (itertools .accumulate (edges_tuple ))
168+ if any (size <= 0 for size in rle .sizes ):
169+ raise ValueError (f"All edge lengths must be > 0, got { _edges_repr (rle )} " )
161170 if extent < 0 :
162171 raise ValueError (f"VaryingDimension extent must be >= 0, got { extent } " )
163- if extent > cumulative [- 1 ]:
164- raise ValueError (
165- f"VaryingDimension extent { extent } exceeds sum of edges { cumulative [- 1 ]} "
166- )
167- object .__setattr__ (self , "edges" , edges_tuple )
168- object .__setattr__ (self , "cumulative" , cumulative )
172+ if extent > rle .total :
173+ raise ValueError (f"VaryingDimension extent { extent } exceeds sum of edges { rle .total } " )
174+ object .__setattr__ (self , "edges" , rle )
169175 object .__setattr__ (self , "extent" , extent )
170176 # Cache nchunks: number of chunks that overlap [0, extent)
171- if extent == 0 :
172- n = 0
173- else :
174- n = bisect .bisect_left (cumulative , extent ) + 1
177+ n = 0 if extent == 0 else rle .index_at (extent - 1 ) + 1
175178 object .__setattr__ (self , "nchunks" , n )
176- object .__setattr__ (self , "ngridcells" , len (edges_tuple ))
179+ object .__setattr__ (self , "ngridcells" , rle .num_edges )
180+
181+ @property
182+ def cumulative (self ) -> tuple [int , ...]:
183+ """Prefix sums of the edges, one entry per chunk.
184+
185+ Lookups do not use this: it is computed on each access, in time and
186+ memory proportional to the number of chunks.
187+ """
188+ return tuple (itertools .accumulate (self .edges ))
177189
178190 def index_to_chunk (self , idx : int ) -> int :
179191 if idx < 0 or idx >= self .extent :
180192 raise IndexError (f"Index { idx } out of bounds for dimension with extent { self .extent } " )
181- return bisect . bisect_right ( self .cumulative , idx )
193+ return self .edges . index_at ( idx )
182194
183195 def chunk_offset (self , chunk_ix : int ) -> int :
184196 """Start position of chunk *chunk_ix* in array coordinates.
185197
186198 Does not validate *chunk_ix* — callers must ensure it is in
187199 ``[0, ngridcells)``. Use ``ChunkGrid.__getitem__`` for safe access.
188200 """
189- return self .cumulative [ chunk_ix - 1 ] if chunk_ix > 0 else 0
201+ return self .edges . offset_of ( chunk_ix ) if chunk_ix > 0 else 0
190202
191203 def chunk_size (self , chunk_ix : int ) -> int :
192204 """Buffer size for codec processing.
@@ -202,25 +214,28 @@ def data_size(self, chunk_ix: int) -> int:
202214 Does not validate *chunk_ix* — callers must ensure it is in
203215 ``[0, ngridcells)``. Use ``ChunkGrid.__getitem__`` for safe access.
204216 """
205- offset = self .cumulative [chunk_ix - 1 ] if chunk_ix > 0 else 0
206- return max (0 , min (self .edges [chunk_ix ], self .extent - offset ))
217+ return max (0 , min (self .edges [chunk_ix ], self .extent - self .chunk_offset (chunk_ix )))
218+
219+ @property
220+ def _data_sizes (self ) -> tuple [int , ...]:
221+ """`data_size` of every chunk that holds data, in order."""
222+ if self .nchunks == 0 :
223+ return ()
224+ last = self .nchunks - 1
225+ return (* itertools .islice (self .edges , last ), self .data_size (last ))
207226
208227 @property
209228 def _unique_edge_lengths (self ) -> Iterable [int ]:
210- """Distinct chunk edge lengths for this dimension (lazily deduplicated) .
229+ """Distinct chunk edge lengths for this dimension, in order of first use .
211230
212231 Used by shard validation to check that every unique edge length
213- is divisible by the inner chunk size. Lazy deduplication avoids
214- materializing all edges for dimensions with many repeated sizes .
232+ is divisible by the inner chunk size. Reads one size per run of
233+ equal edges, not one per chunk .
215234 """
216- seen : set [int ] = set ()
217- for e in self .edges :
218- if e not in seen :
219- seen .add (e )
220- yield e
235+ return tuple (dict .fromkeys (self .edges .sizes ))
221236
222237 def indices_to_chunks (self , indices : npt .NDArray [np .intp ]) -> npt .NDArray [np .intp ]:
223- return np . searchsorted ( self .cumulative , indices , side = "right" )
238+ return self .edges . indices_at ( indices )
224239
225240 def with_extent (self , new_extent : int ) -> VaryingDimension :
226241 """Re-bind to *new_extent* without modifying edges.
@@ -229,7 +244,7 @@ def with_extent(self, new_extent: int) -> VaryingDimension:
229244 are already correct. Raises if the
230245 existing edges don't cover *new_extent*.
231246 """
232- edge_sum = self .cumulative [ - 1 ]
247+ edge_sum = self .edges . total
233248 if edge_sum < new_extent :
234249 raise ValueError (
235250 f"VaryingDimension edge sum { edge_sum } is less than new extent { new_extent } "
@@ -246,15 +261,15 @@ def resize(self, new_extent: int) -> VaryingDimension:
246261 """
247262 if new_extent == self .extent :
248263 return self
249- elif new_extent > self .cumulative [ - 1 ] :
250- expanded_edges = list ( self .edges ) + [ new_extent - self .cumulative [ - 1 ]]
251- return VaryingDimension (expanded_edges , extent = new_extent )
264+ elif new_extent > self .edges . total :
265+ grown = self .edges . with_edge ( new_extent - self .edges . total )
266+ return VaryingDimension (grown , extent = new_extent )
252267 else :
253268 return VaryingDimension (self .edges , extent = new_extent )
254269
255270 @property
256271 def _size_repr (self ) -> str :
257- return repr ( tuple ( self .edges ) )
272+ return _edges_repr ( self .edges )
258273
259274
260275@runtime_checkable
@@ -391,7 +406,7 @@ class ChunkGrid:
391406 ``grid[coords]`` to return a ``ChunkSpec`` without external parameters.
392407
393408 Internally represents each dimension as either FixedDimension (uniform chunks)
394- or VaryingDimension (per-chunk edge lengths with prefix sums ).
409+ or VaryingDimension (per-chunk edge lengths, run-length encoded ).
395410 """
396411
397412 _dimensions : tuple [DimensionGrid , ...]
@@ -506,7 +521,12 @@ def chunk_sizes(self) -> tuple[tuple[int, ...], ...]:
506521 One inner tuple per dimension, each containing the data size
507522 of every chunk along that dimension.
508523 """
509- return tuple (tuple (d .data_size (i ) for i in range (d .nchunks )) for d in self ._dimensions )
524+ return tuple (
525+ d ._data_sizes
526+ if isinstance (d , VaryingDimension )
527+ else tuple (d .data_size (i ) for i in range (d .nchunks ))
528+ for d in self ._dimensions
529+ )
510530
511531 # -- Collection interface --
512532
0 commit comments