From 76e9cae800f021bb42c29a65cb2faa1ecb09579b Mon Sep 17 00:00:00 2001 From: Nash Aschenbath Date: Sat, 7 Nov 2026 13:13:49 +0800 Subject: [PATCH] Add optimized key-range deletion to SortedDict --- docs/sorteddict.rst | 1 + src/sortedcontainers/sorteddict.py | 45 +++++ src/sortedcontainers/sortedlist.py | 130 +++++++++++++++ tests/test_sorteddict_range_del.py | 258 +++++++++++++++++++++++++++++ 4 files changed, 434 insertions(+) create mode 100644 tests/test_sorteddict_range_del.py diff --git a/docs/sorteddict.rst b/docs/sorteddict.rst index 1790c980..d6f076cc 100644 --- a/docs/sorteddict.rst +++ b/docs/sorteddict.rst @@ -12,6 +12,7 @@ SortedDict .. automethod:: __getitem__ .. automethod:: __setitem__ .. automethod:: __delitem__ + .. automethod:: range_del .. automethod:: __iter__ .. automethod:: __len__ .. automethod:: setdefault diff --git a/src/sortedcontainers/sorteddict.py b/src/sortedcontainers/sorteddict.py index 273c9ce3..a4c40702 100644 --- a/src/sortedcontainers/sorteddict.py +++ b/src/sortedcontainers/sorteddict.py @@ -52,6 +52,7 @@ class SortedDict(dict): * :func:`SortedDict.clear` * :func:`SortedDict.pop` * :func:`SortedDict.popitem` + * :func:`SortedDict.range_del` Methods for looking up items: @@ -230,6 +231,50 @@ def __delitem__(self, key): dict.__delitem__(self, key) self._list_remove(key) + def range_del(self, minimum=None, maximum=None, inclusive=(True, True)): + """Remove keys between `minimum` and `maximum`. + + ``inclusive`` is a pair of booleans indicating whether the lower and + upper bounds are included. ``None`` for either bound means the + corresponding end of the sorted dictionary. The operation removes the + range in one block-level update and returns ``None``. + + Bounds use the same ordering as :meth:`irange`. With a key function, + bounds are dictionary keys, and all keys with an equal comparison key + are included or excluded together. Reversed or empty ranges do nothing. + + Finding the bounds takes approximately ``O(log(n))`` comparisons. + Deletion takes ``O(k + m + n/m)`` time in the worst case, where ``k`` + is the number of removed keys and ``m`` is the sorted-list load factor. + This includes copying boundary blocks and moving references in the + flat list of blocks. It uses ``O(k + m)`` temporary space, avoids + building the positional index, and does not search for each key again. + + >>> sd = SortedDict({1: 'a', 2: 'b', 3: 'c', 4: 'd'}) + >>> sd.range_del(2, 3) + >>> sd + SortedDict({1: 'a', 4: 'd'}) + >>> sd.range_del(4) + >>> sd + SortedDict({1: 'a'}) + + :param minimum: minimum key to remove + :param maximum: maximum key to remove + :param inclusive: pair of booleans + :return: ``None`` + + """ + keys = self._list._delete_range_by_bounds(minimum, maximum, inclusive) + if not keys: + return + + # Delete from the mapping directly after all range comparisons and + # coordinate calculations have succeeded. Keys are required to remain + # hashable and order-stable by SortedDict's existing contract. + dict_delitem = dict.__delitem__ + for key in keys: + dict_delitem(self, key) + def __iter__(self): """Return an iterator over the keys of the sorted dict. diff --git a/src/sortedcontainers/sortedlist.py b/src/sortedcontainers/sortedlist.py index aecf7871..933c20dd 100644 --- a/src/sortedcontainers/sortedlist.py +++ b/src/sortedcontainers/sortedlist.py @@ -1072,6 +1072,136 @@ def irange(self, minimum=None, maximum=None, inclusive=(True, True), reverse=Fal return self._islice(min_pos, min_idx, max_pos, max_idx, reverse) + def _range_bounds(self, minimum=None, maximum=None, inclusive=(True, True)): + """Return block coordinates for a value range. + + This is the coordinate part of :meth:`irange`. It deliberately works + with block-local bisect operations and does not build the positional + index, which is useful for operations that already have the values in + sorted order. + """ + if self.key is not None: + minimum = self._key(minimum) if minimum is not None else None + maximum = self._key(maximum) if maximum is not None else None + _lists = self._keys + else: + _lists = self._lists + + _maxes = self._maxes + if not _maxes: + return None + + if minimum is None: + min_pos = 0 + min_idx = 0 + elif inclusive[0]: + min_pos = bisect_left(_maxes, minimum) + if min_pos == len(_maxes): + return None + min_idx = bisect_left(_lists[min_pos], minimum) + else: + min_pos = bisect_right(_maxes, minimum) + if min_pos == len(_maxes): + return None + min_idx = bisect_right(_lists[min_pos], minimum) + + if maximum is None: + max_pos = len(_maxes) - 1 + max_idx = len(_lists[max_pos]) + elif inclusive[1]: + max_pos = bisect_right(_maxes, maximum) + if max_pos == len(_maxes): + max_pos -= 1 + max_idx = len(_lists[max_pos]) + else: + max_idx = bisect_right(_lists[max_pos], maximum) + else: + max_pos = bisect_left(_maxes, maximum) + if max_pos == len(_maxes): + max_pos -= 1 + max_idx = len(_lists[max_pos]) + else: + max_idx = bisect_left(_lists[max_pos], maximum) + + if min_pos > max_pos or (min_pos == max_pos and min_idx >= max_idx): + return None + + return min_pos, min_idx, max_pos, max_idx + + def _delete_range(self, min_pos, min_idx, max_pos, max_idx): + """Delete a contiguous range using block coordinates. + + The operation avoids positional-index construction and per-item + deletion. Only the boundary blocks are copied; complete interior + blocks are discarded in one slice. The small amount of local + rebalancing keeps short fragments out of the middle of the structure. + """ + arrays = [self._lists] + if hasattr(self, '_keys'): + arrays.append(self._keys) + + lists = self._lists + removed = sum(len(block) for block in lists[min_pos : max_pos + 1]) + removed -= min_idx + len(lists[max_pos]) - max_idx + fragments = [ + blocks[min_pos][:min_idx] + blocks[max_pos][max_idx:] for blocks in arrays + ] + stop = max_pos + 1 + + # A short surviving fragment needs one neighbor to restore the load + # invariant. Keep cached keys alongside values without calling _key. + size = len(fragments[0]) + if 0 < size < (self._load >> 1): + if min_pos: + min_pos -= 1 + fragments = [ + blocks[min_pos] + fragments[index] + for index, blocks in enumerate(arrays) + ] + elif stop < len(lists): + fragments = [ + fragments[index] + blocks[stop] + for index, blocks in enumerate(arrays) + ] + stop += 1 + + size = len(fragments[0]) + replacements = [[] for _ in arrays] + if size: + # Even splitting avoids a short last fragment in the middle of + # the list. At most two old boundary blocks and one neighbor are + # involved; whole interior blocks need no element-wise rebuilding. + count = (size + (self._load << 1) - 1) // (self._load << 1) + width, extra = divmod(size, count) + offset = 0 + for pos in range(count): + end = offset + width + (pos < extra) + for index, fragment in enumerate(fragments): + replacements[index].append(fragment[offset:end]) + offset = end + + maxes = [block[-1] for block in replacements[-1]] + for index, blocks in enumerate(arrays): + blocks[min_pos:stop] = replacements[index] + self._maxes[min_pos:stop] = maxes + self._len -= removed + # Any successful structural deletion invalidates the positional index; + # lazy rebuilding preserves the normal cache contract. + self._index[:] = [] + self._offset = 0 + + def _delete_range_by_bounds( + self, minimum=None, maximum=None, inclusive=(True, True) + ): + """Delete and return values in a sorted value range.""" + bounds = self._range_bounds(minimum, maximum, inclusive) + if bounds is None: + return [] + min_pos, min_idx, max_pos, max_idx = bounds + values = list(self._islice(min_pos, min_idx, max_pos, max_idx, False)) + self._delete_range(min_pos, min_idx, max_pos, max_idx) + return values + def __len__(self): """Return the size of the sorted list. diff --git a/tests/test_sorteddict_range_del.py b/tests/test_sorteddict_range_del.py new file mode 100644 index 00000000..398fb1fa --- /dev/null +++ b/tests/test_sorteddict_range_del.py @@ -0,0 +1,258 @@ +"""Range deletion preserves mapping, ordering, and block invariants.""" + +from itertools import product +from operator import neg +from random import Random + +import pytest + +from sortedcontainers import SortedDict + + +def decade(value): + return value // 10 + + +def selected(value, minimum, maximum, inclusive, key): + key = key or (lambda item: item) + value = key(value) + if minimum is not None: + lower = key(minimum) + if value < lower or (value == lower and not inclusive[0]): + return False + if maximum is not None: + upper = key(maximum) + if value > upper or (value == upper and not inclusive[1]): + return False + return True + + +@pytest.mark.parametrize('key', [None, neg, decade]) +@pytest.mark.parametrize('inclusive', list(product((False, True), repeat=2))) +def test_range_del_bounds_and_views(key, inclusive): + for minimum, maximum in product((None, -1, 0, 10, 19, 20, 40), repeat=2): + mapping = SortedDict(key, ((value, object()) for value in range(30))) + mapping._reset(4) + before = list(mapping.items()) + expected = [ + (value, item) + for value, item in before + if not selected(value, minimum, maximum, inclusive, key) + ] + keys, items, values = mapping.keys(), mapping.items(), mapping.values() + + assert mapping.range_del(minimum, maximum, inclusive) is None + + assert list(items) == expected + assert list(keys) == [value for value, item in expected] + assert list(values) == [item for value, item in expected] + assert dict(mapping) == dict(expected) + mapping._check() + + +@pytest.mark.parametrize('key', [None, neg, decade]) +def test_range_del_empty(key): + mapping = SortedDict(key) + assert mapping.range_del() is None + assert mapping.range_del(1, 9) is None + assert not mapping + mapping._check() + + +@pytest.mark.parametrize('key', [None, neg, decade]) +@pytest.mark.parametrize('load', [4, 5, 8, 25]) +def test_range_del_mutations(key, load): + rng = Random(180) + mapping = SortedDict(key, ((value, value) for value in range(100))) + mapping._reset(load) + model = dict(mapping) + for step in range(300): + if step % 3: + for _ in range(rng.randrange(1, 30)): + value = rng.randrange(-100, 100) + mapping[value] = step + model[value] = step + else: + minimum = rng.choice([None, rng.randrange(-120, 120)]) + maximum = rng.choice([None, rng.randrange(-120, 120)]) + inclusive = (rng.choice([True, False]), rng.choice([True, False])) + model = { + value: item + for value, item in model.items() + if not selected(value, minimum, maximum, inclusive, key) + } + mapping.range_del(minimum, maximum, inclusive) + + assert dict(mapping) == model + expected = sorted(model, key=key) + assert list(mapping) == expected + mapping._check() + if mapping: + # Exercise rebuilding and later invalidation of the index cache. + assert mapping.peekitem(len(mapping) // 2)[0] == expected[len(mapping) // 2] + + +@pytest.mark.parametrize('key', [None, neg]) +def test_range_del_preserves_empty_range_index(key): + mapping = SortedDict(key, ((value, value) for value in range(100))) + mapping._reset(4) + mapping.peekitem(51) + index = mapping._list._index + original = index[:] + assert original + + mapping.range_del(40, 40, inclusive=(False, False)) + assert mapping._list._index is index + assert index == original + mapping._check() + + lower, upper = (60, 40) if key is neg else (40, 60) + mapping.range_del(lower, upper) + assert not index + assert mapping.peekitem(20)[0] == list(mapping)[20] + mapping._check() + + +@pytest.mark.parametrize('key', [None, neg, decade]) +def test_range_del_empty_gaps_preserve_block_layout_and_index(key): + mapping = SortedDict(key, ((value * 10, value) for value in range(100))) + mapping._reset(4) + mapping.peekitem(51) + index = mapping._list._index[:] + blocks = mapping._list._lists[:] + before = list(mapping.items()) + for position in range(len(blocks) - 1): + lower, upper = blocks[position][-1], blocks[position + 1][0] + mapping.range_del(lower, upper, (False, False)) + assert list(mapping.items()) == before + assert mapping._list._index == index + assert all( + current is previous + for current, previous in zip(mapping._list._lists, blocks) + ) + mapping._check() + + +@pytest.mark.parametrize('key', [None, neg, decade]) +def test_range_del_avoids_per_item_search_and_index(key, monkeypatch): + mapping = SortedDict(key, ((value, value) for value in range(1000))) + mapping._reset(8) + + def unexpected(*args, **kwargs): + pytest.fail('range deletion used positional or per-item deletion') + + for name in ('_build_index', '_loc', '_pos', '_delete', 'remove'): + monkeypatch.setattr(mapping._list, name, unexpected) + monkeypatch.setattr(mapping, '_list_remove', unexpected) + lower, upper = (700, 200) if key is neg else (200, 700) + mapping.range_del(lower, upper) + assert len(mapping) == sum( + not selected(value, lower, upper, (True, True), key) for value in range(1000) + ) + mapping._check() + + +def test_range_del_uses_cached_sort_keys_and_retains_unaffected_blocks(): + calls = [] + + def key(value): + calls.append(value) + return value // 10 + + mapping = SortedDict(key, ((value, object()) for value in range(1000))) + mapping._reset(8) + first_values = mapping._list._lists[0] + first_keys = mapping._list._keys[0] + last_values = mapping._list._lists[-1] + last_keys = mapping._list._keys[-1] + survivors = {value: mapping[value] for value in (0, 100, 800, 999)} + calls.clear() + + mapping.range_del(210, 610) + + assert calls == [210, 610] + assert mapping._list._lists[0] is first_values + assert mapping._list._keys[0] is first_keys + assert mapping._list._lists[-1] is last_values + assert mapping._list._keys[-1] is last_keys + assert all(mapping[value] is item for value, item in survivors.items()) + mapping._check() + + +def test_range_del_comparison_failure_does_not_mutate(): + mapping = SortedDict((value, value) for value in range(100)) + mapping._reset(4) + mapping.peekitem(51) + before = list(mapping.items()) + blocks = [block[:] for block in mapping._list._lists] + index = mapping._list._index[:] + + with pytest.raises(TypeError): + mapping.range_del(20, 'incomparable') + + assert list(mapping.items()) == before + assert mapping._list._lists == blocks + assert mapping._list._index == index + mapping._check() + + +def test_range_del_key_failure_does_not_mutate(): + def key(value): + if value == 'failure': + raise ValueError('invalid bound') + return -value + + mapping = SortedDict(key, ((value, value) for value in range(100))) + mapping._reset(4) + before = list(mapping.items()) + blocks = [block[:] for block in mapping._list._keys] + + with pytest.raises(ValueError, match='invalid bound'): + mapping.range_del(80, 'failure') + + assert list(mapping.items()) == before + assert mapping._list._keys == blocks + mapping._check() + + +def test_range_del_unordered_keys_and_cached_methods(): + class Key: + def __init__(self, rank): + self.rank = rank + + keys = [Key(rank) for rank in range(40)] + mapping = SortedDict(lambda value: value.rank // 5, ((key, key) for key in keys)) + mapping._reset(4) + irange = mapping.irange + bisect_left = mapping.bisect_left + minimum, maximum = Key(10), Key(20) + mapping.range_del(minimum, maximum) + assert list(mapping) == keys[:10] + keys[25:] + assert list(irange()) == list(mapping) + assert bisect_left(Key(30)) == 15 + + new = Key(11) + mapping[new] = new + assert list(irange(Key(10), Key(14))) == [new] + assert mapping.popitem(0) == (keys[0], keys[0]) + assert mapping.popitem() == (keys[-1], keys[-1]) + mapping._check() + + +@pytest.mark.parametrize('key', [None, neg, decade]) +def test_range_del_block_boundaries(key): + for size in (1, 5, 17, 31, 48): + for start in range(size): + for stop in range(start + 1, size + 1): + mapping = SortedDict(key, ((value, value) for value in range(size))) + mapping._reset(4) + ordered = list(mapping) + lower, upper = ordered[start], ordered[stop - 1] + expected = [ + value + for value in ordered + if not selected(value, lower, upper, (True, True), key) + ] + mapping.range_del(lower, upper) + assert list(mapping) == expected + mapping._check()