Skip to content

Commit a0689d7

Browse files
anidotnetclaude
andcommitted
Optimize in filter index scan with direct lookups #1258
InFilter.applyOnIndex iterated over every entry in the index map and checked set membership, making each `in` query O(index size). On large collections this degraded to a full index walk while `eq` (a direct lookup) stayed fast. Look up each value directly via IndexMap.get instead, turning the scan into O(values * log size). Values are iterated through a sorted set (reversed when the scan order is reversed) so `in` combined with an order-by on the indexed field still returns results in index order. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 7e610bc commit a0689d7

3 files changed

Lines changed: 37 additions & 7 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
## Release 4.3.4 - Unreleased
2+
3+
### Performance Improvements
4+
5+
- Made `in` filter index scans look up each value directly instead of scanning every index entry, so `in` queries on large indexed collections are now as fast as `eq` instead of degrading to a full index walk #1258
6+
17
## Release 4.3.3 - Jun 26, 2026
28

39
### New Changes

‎nitrite/src/main/java/org/dizitart/no2/filters/InFilter.java‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import org.dizitart.no2.index.IndexMap;
2626

2727
import java.util.*;
28-
import java.util.stream.Collectors;
2928

3029
/**
3130
* @author Anindya Chatterjee
@@ -53,16 +52,24 @@ public boolean apply(Pair<NitriteId, Document> element) {
5352
}
5453

5554
public List<?> applyOnIndex(IndexMap indexMap) {
56-
// convert comparable set to DBValue set
57-
Set<DBValue> dbValueSet = comparableSet.stream().map(value -> value == null ? DBNull.getInstance()
58-
: new DBValue(value)).collect(Collectors.toSet());
55+
// collect the values to look up in a sorted set, so the scan follows
56+
// the natural (or reverse) order of the index
57+
NavigableSet<DBValue> dbValueSet = new TreeSet<>();
58+
for (Comparable<?> value : comparableSet) {
59+
dbValueSet.add(value == null ? DBNull.getInstance() : new DBValue(value));
60+
}
5961

6062
List<NavigableMap<DBValue, Object>> subMap = new ArrayList<>();
6163
List<NitriteId> nitriteIds = new ArrayList<>();
6264

63-
for (Pair<DBValue, ?> entry : indexMap.entries()) {
64-
if (dbValueSet.contains(entry.getFirst())) {
65-
processIndexValue(entry.getSecond(), subMap, nitriteIds);
65+
// look up each value directly in the index instead of scanning every
66+
// entry, turning the scan from O(index size) into O(values * log size)
67+
Iterable<DBValue> scanOrder = indexMap.isReverseScan()
68+
? dbValueSet.descendingSet() : dbValueSet;
69+
for (DBValue dbValue : scanOrder) {
70+
Object value = indexMap.get(dbValue);
71+
if (value != null) {
72+
processIndexValue(value, subMap, nitriteIds);
6673
}
6774
}
6875

‎nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindBySingleFieldIndexTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,23 @@ public void testInFilterUsesIndex() {
152152
assertNotNull("notIn filter should use index scan", plan.getIndexScanFilter());
153153
assertNotNull(plan.getIndexDescriptor());
154154
assertNull("notIn filter should not fall back to collection scan", plan.getCollectionScanFilter());
155+
156+
// direct-lookup index scan must still honor the requested sort order
157+
cursor = collection.find(where("firstName").in("fn3", "fn1", "fn2"),
158+
orderBy("firstName", SortOrder.Ascending));
159+
List<Object> ascending = new ArrayList<>();
160+
for (Document doc : cursor) {
161+
ascending.add(doc.get("firstName"));
162+
}
163+
assertEquals(Arrays.asList("fn1", "fn2", "fn3"), ascending);
164+
165+
cursor = collection.find(where("firstName").in("fn3", "fn1", "fn2"),
166+
orderBy("firstName", SortOrder.Descending));
167+
List<Object> descending = new ArrayList<>();
168+
for (Document doc : cursor) {
169+
descending.add(doc.get("firstName"));
170+
}
171+
assertEquals(Arrays.asList("fn3", "fn2", "fn1"), descending);
155172
}
156173

157174
@Test

0 commit comments

Comments
 (0)