Skip to content

Commit 851c682

Browse files
committed
perf(chunk-grids): hold rectilinear chunk edges as run-length encoded runs
Opening an array with a rectilinear chunk grid cost time and memory proportional to the repeat counts in its stored metadata: `from_dict` expanded every run into one int per chunk, validation and `VaryingDimension` walked and re-accumulated them, and `to_dict` re-compressed them. A ~300 byte document declaring `[[1, 10**12]]` got the process OOM-killed. Add `RunLengthEdges`, an immutable `Sequence[int]` stored as merged `(size, count)` runs with bisect-over-runs lookups, and use it for the explicit edges of `RectilinearChunkGridMetadata.chunk_shapes` and `VaryingDimension.edges`. Metadata parsing, validation, serialization, `update_shape`, the dimension lookups, resize and the sharding codec's divisibility check now cost O(number of runs). Assisted-by: ClaudeCode:claude-fable-5.1
1 parent 25d390b commit 851c682

8 files changed

Lines changed: 580 additions & 108 deletions

File tree

‎changes/+rle-runs.bugfix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Opening an array with a rectilinear chunk grid, and indexing it, now costs time and memory proportional to the size of the grid's run-length encoded metadata rather than to its number of chunks: a stored grid such as `[[1, 1000000000000]]`, a few hundred bytes of JSON, previously took seconds per ten million chunks to open and could exhaust memory. The explicit edges in `RectilinearChunkGridMetadata.chunk_shapes` and `VaryingDimension.edges` are now held as `zarr.core.common.RunLengthEdges`, an immutable sequence that reads like the tuple of edges and compares equal to it, but is not a `tuple` instance. `VaryingDimension.cumulative` is now computed on access. The repr of a chunk grid shows a dimension of more than 100 chunks in its run-length encoded form.

‎src/zarr/codecs/sharding.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
merge_and_encode_chunk,
4646
)
4747
from zarr.core.common import (
48+
RunLengthEdges,
4849
ShapeLike,
4950
parse_named_configuration,
5051
parse_shapelike,
@@ -620,8 +621,10 @@ def validate(
620621
if isinstance(chunk_grid, RegularChunkGridMetadata):
621622
edges_per_dim: tuple[tuple[int, ...], ...] = tuple((s,) for s in chunk_grid.chunk_shape)
622623
elif isinstance(chunk_grid, RectilinearChunkGridMetadata):
624+
# One size per run of equal edges, not one per chunk.
623625
edges_per_dim = tuple(
624-
(s,) if isinstance(s, int) else s for s in chunk_grid.chunk_shapes
626+
(s,) if isinstance(s, int) else RunLengthEdges.from_edges(s).sizes
627+
for s in chunk_grid.chunk_shapes
625628
)
626629
else:
627630
raise TypeError(

‎src/zarr/core/chunk_grids.py‎

Lines changed: 67 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
from __future__ import annotations
22

3-
import bisect
43
import itertools
54
import math
65
import operator
@@ -24,6 +23,7 @@
2423

2524
import zarr
2625
from zarr.core.common import (
26+
RunLengthEdges,
2727
ShapeLike,
2828
ceildiv_int,
2929
parse_shapelike,
@@ -130,63 +130,75 @@ def _size_repr(self) -> str:
130130
return str(self.size)
131131

132132

133-
@dataclass(frozen=True)
133+
_MAX_EDGES_IN_REPR = 100
134+
"""The most chunk edges of one dimension written out one by one in a repr or an error
135+
message; a longer dimension is shown run-length encoded."""
136+
137+
138+
def _edges_repr(edges: RunLengthEdges) -> str:
139+
"""The edges as a tuple, or run-length encoded if there are too many to write out."""
140+
if edges.num_edges <= _MAX_EDGES_IN_REPR:
141+
return repr(tuple(edges))
142+
return repr(edges.to_rle())
143+
144+
145+
@dataclass(frozen=True, init=False)
134146
class VaryingDimension:
135147
"""Explicit per-chunk sizes. The last chunk may extend past the array
136148
extent (``extent < sum(edges)``), in which case ``data_size`` clips to
137149
the valid region while ``chunk_size`` returns the full edge length for
138150
codec processing. This underflow is allowed to match how regular grids
139151
handle boundary chunks, and to support shrinking an array without
140-
rewriting chunk edges (the spec allows trailing edges beyond the extent)."""
152+
rewriting chunk edges (the spec allows trailing edges beyond the extent).
153+
154+
The edges are held run-length encoded (see `RunLengthEdges`), so building
155+
the dimension, resizing it and every lookup cost time and memory in the
156+
number of runs of equal edges, not in the number of chunks: a dimension of
157+
``2**40`` equal chunks is one run."""
141158

142-
edges: tuple[int, ...] # per-chunk edge lengths (all > 0)
143-
cumulative: tuple[int, ...] # prefix sums for O(log n) lookup
159+
edges: RunLengthEdges # per-chunk edge lengths (all > 0), run-length encoded
144160
extent: int # array dimension length (may be < sum(edges) after resize)
145161
nchunks: int = field(init=False, repr=False) # cached at construction
146162
ngridcells: int = field(init=False, repr=False) # cached at construction
147163

148-
# TODO(perf): for long dimensions (O(million chunks)):
149-
# - with_extent/resize recompute cumulative sums and nchunks from scratch;
150-
# add a fast path that reuses the existing cumulative tuple.
151-
# - Consider storing cumulative as ndarray so bisect calls can use
152-
# np.searchsorted. Scalar lookups (chunk_offset, index_to_chunk)
153-
# would need benchmarking to confirm no regression.
154164
def __init__(self, edges: Sequence[int], extent: int) -> None:
155-
edges_tuple = tuple(edges)
156-
if not edges_tuple:
165+
rle = RunLengthEdges.from_edges(edges)
166+
if rle.num_edges == 0:
157167
raise ValueError("VaryingDimension edges must not be empty")
158-
if any(e <= 0 for e in edges_tuple):
159-
raise ValueError(f"All edge lengths must be > 0, got {edges_tuple}")
160-
cumulative = tuple(itertools.accumulate(edges_tuple))
168+
if any(size <= 0 for size in rle.sizes):
169+
raise ValueError(f"All edge lengths must be > 0, got {_edges_repr(rle)}")
161170
if extent < 0:
162171
raise ValueError(f"VaryingDimension extent must be >= 0, got {extent}")
163-
if extent > cumulative[-1]:
164-
raise ValueError(
165-
f"VaryingDimension extent {extent} exceeds sum of edges {cumulative[-1]}"
166-
)
167-
object.__setattr__(self, "edges", edges_tuple)
168-
object.__setattr__(self, "cumulative", cumulative)
172+
if extent > rle.total:
173+
raise ValueError(f"VaryingDimension extent {extent} exceeds sum of edges {rle.total}")
174+
object.__setattr__(self, "edges", rle)
169175
object.__setattr__(self, "extent", extent)
170176
# Cache nchunks: number of chunks that overlap [0, extent)
171-
if extent == 0:
172-
n = 0
173-
else:
174-
n = bisect.bisect_left(cumulative, extent) + 1
177+
n = 0 if extent == 0 else rle.index_at(extent - 1) + 1
175178
object.__setattr__(self, "nchunks", n)
176-
object.__setattr__(self, "ngridcells", len(edges_tuple))
179+
object.__setattr__(self, "ngridcells", rle.num_edges)
180+
181+
@property
182+
def cumulative(self) -> tuple[int, ...]:
183+
"""Prefix sums of the edges, one entry per chunk.
184+
185+
Lookups do not use this: it is computed on each access, in time and
186+
memory proportional to the number of chunks.
187+
"""
188+
return tuple(itertools.accumulate(self.edges))
177189

178190
def index_to_chunk(self, idx: int) -> int:
179191
if idx < 0 or idx >= self.extent:
180192
raise IndexError(f"Index {idx} out of bounds for dimension with extent {self.extent}")
181-
return bisect.bisect_right(self.cumulative, idx)
193+
return self.edges.index_at(idx)
182194

183195
def chunk_offset(self, chunk_ix: int) -> int:
184196
"""Start position of chunk *chunk_ix* in array coordinates.
185197
186198
Does not validate *chunk_ix* — callers must ensure it is in
187199
``[0, ngridcells)``. Use ``ChunkGrid.__getitem__`` for safe access.
188200
"""
189-
return self.cumulative[chunk_ix - 1] if chunk_ix > 0 else 0
201+
return self.edges.offset_of(chunk_ix) if chunk_ix > 0 else 0
190202

191203
def chunk_size(self, chunk_ix: int) -> int:
192204
"""Buffer size for codec processing.
@@ -202,25 +214,28 @@ def data_size(self, chunk_ix: int) -> int:
202214
Does not validate *chunk_ix* — callers must ensure it is in
203215
``[0, ngridcells)``. Use ``ChunkGrid.__getitem__`` for safe access.
204216
"""
205-
offset = self.cumulative[chunk_ix - 1] if chunk_ix > 0 else 0
206-
return max(0, min(self.edges[chunk_ix], self.extent - offset))
217+
return max(0, min(self.edges[chunk_ix], self.extent - self.chunk_offset(chunk_ix)))
218+
219+
@property
220+
def _data_sizes(self) -> tuple[int, ...]:
221+
"""`data_size` of every chunk that holds data, in order."""
222+
if self.nchunks == 0:
223+
return ()
224+
last = self.nchunks - 1
225+
return (*itertools.islice(self.edges, last), self.data_size(last))
207226

208227
@property
209228
def _unique_edge_lengths(self) -> Iterable[int]:
210-
"""Distinct chunk edge lengths for this dimension (lazily deduplicated).
229+
"""Distinct chunk edge lengths for this dimension, in order of first use.
211230
212231
Used by shard validation to check that every unique edge length
213-
is divisible by the inner chunk size. Lazy deduplication avoids
214-
materializing all edges for dimensions with many repeated sizes.
232+
is divisible by the inner chunk size. Reads one size per run of
233+
equal edges, not one per chunk.
215234
"""
216-
seen: set[int] = set()
217-
for e in self.edges:
218-
if e not in seen:
219-
seen.add(e)
220-
yield e
235+
return tuple(dict.fromkeys(self.edges.sizes))
221236

222237
def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]:
223-
return np.searchsorted(self.cumulative, indices, side="right")
238+
return self.edges.indices_at(indices)
224239

225240
def with_extent(self, new_extent: int) -> VaryingDimension:
226241
"""Re-bind to *new_extent* without modifying edges.
@@ -229,7 +244,7 @@ def with_extent(self, new_extent: int) -> VaryingDimension:
229244
are already correct. Raises if the
230245
existing edges don't cover *new_extent*.
231246
"""
232-
edge_sum = self.cumulative[-1]
247+
edge_sum = self.edges.total
233248
if edge_sum < new_extent:
234249
raise ValueError(
235250
f"VaryingDimension edge sum {edge_sum} is less than new extent {new_extent}"
@@ -246,15 +261,15 @@ def resize(self, new_extent: int) -> VaryingDimension:
246261
"""
247262
if new_extent == self.extent:
248263
return self
249-
elif new_extent > self.cumulative[-1]:
250-
expanded_edges = list(self.edges) + [new_extent - self.cumulative[-1]]
251-
return VaryingDimension(expanded_edges, extent=new_extent)
264+
elif new_extent > self.edges.total:
265+
grown = self.edges.with_edge(new_extent - self.edges.total)
266+
return VaryingDimension(grown, extent=new_extent)
252267
else:
253268
return VaryingDimension(self.edges, extent=new_extent)
254269

255270
@property
256271
def _size_repr(self) -> str:
257-
return repr(tuple(self.edges))
272+
return _edges_repr(self.edges)
258273

259274

260275
@runtime_checkable
@@ -391,7 +406,7 @@ class ChunkGrid:
391406
``grid[coords]`` to return a ``ChunkSpec`` without external parameters.
392407
393408
Internally represents each dimension as either FixedDimension (uniform chunks)
394-
or VaryingDimension (per-chunk edge lengths with prefix sums).
409+
or VaryingDimension (per-chunk edge lengths, run-length encoded).
395410
"""
396411

397412
_dimensions: tuple[DimensionGrid, ...]
@@ -506,7 +521,12 @@ def chunk_sizes(self) -> tuple[tuple[int, ...], ...]:
506521
One inner tuple per dimension, each containing the data size
507522
of every chunk along that dimension.
508523
"""
509-
return tuple(tuple(d.data_size(i) for i in range(d.nchunks)) for d in self._dimensions)
524+
return tuple(
525+
d._data_sizes
526+
if isinstance(d, VaryingDimension)
527+
else tuple(d.data_size(i) for i in range(d.nchunks))
528+
for d in self._dimensions
529+
)
510530

511531
# -- Collection interface --
512532

0 commit comments

Comments
 (0)