Skip to content

Commit 30de6ea

Browse files
committed
Optimize query performance by implementing covered count for index scans and enhancing size calculations in DocumentStream and TransactionalMap. Add tests to verify covered count behavior.
1 parent 0e6ca19 commit 30de6ea

5 files changed

Lines changed: 106 additions & 4 deletions

File tree

‎nitrite/src/main/java/org/dizitart/no2/collection/operation/FindOptimizer.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,12 @@ private void planForIndexScanningFilters(FindPlan findPlan, Set<ComparableFilter
275275
private void planForCollectionScanningFilters(FindPlan findPlan, Set<ComparableFilter> indexScanFilters,
276276
Set<Filter> columnScanFilters, List<Filter> filters) {
277277
for (Filter filter : filters) {
278+
// Filter.ALL matches every document, so it never needs to run as a post-filter.
279+
// Skipping it keeps a plain find() free of a collection-scan filter, which lets the
280+
// cursor answer size() from the map size instead of iterating every document.
281+
if (filter == Filter.ALL) {
282+
continue;
283+
}
278284
// ignore the elected filters for index scan and
279285
// insert rest of the filters for column scan
280286
// NOTE: for byId filter, index scan filters will always be empty

‎nitrite/src/main/java/org/dizitart/no2/collection/operation/ReadOperations.java‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,20 +107,46 @@ private void prepareLogicalFilter(LogicalFilter logicalFilter) {
107107
}
108108

109109
private DocumentCursor createCursor(FindPlan findPlan) {
110-
RecordStream<Pair<NitriteId, Document>> recordStream = findSuitableStream(findPlan);
110+
// -1 means "not an index scan"; the index branch records the exact id-set size here.
111+
long[] indexedIdCount = { -1 };
112+
RecordStream<Pair<NitriteId, Document>> recordStream = findSuitableStream(findPlan, indexedIdCount);
111113
DocumentStream cursor = new DocumentStream(recordStream, processorChain);
112114
cursor.setFindPlan(findPlan);
115+
cursor.setCoveredCount(computeCoveredCount(findPlan, indexedIdCount[0]));
113116
return cursor;
114117
}
115118

116-
private RecordStream<Pair<NitriteId, Document>> findSuitableStream(FindPlan findPlan) {
119+
/**
120+
* Returns the exact match count when the query is fully answered without fetching documents,
121+
* or {@code null} when the cursor must be drained to count. The count is exact only when
122+
* nothing downstream drops or changes cardinality (a post-filter, skip, or limit); sort does
123+
* not change the count, and an OR-union needs de-duplication so its count cannot be derived.
124+
*/
125+
private Long computeCoveredCount(FindPlan findPlan, long indexedIdCount) {
126+
if (!findPlan.getSubPlans().isEmpty()
127+
|| findPlan.getCollectionScanFilter() != null
128+
|| findPlan.getSkip() != null
129+
|| findPlan.getLimit() != null
130+
|| findPlan.getByIdFilter() != null) {
131+
return null;
132+
}
133+
if (findPlan.getIndexDescriptor() != null) {
134+
// the index supplied the exact matching id set
135+
return indexedIdCount >= 0 ? indexedIdCount : null;
136+
}
137+
// pure full scan over the whole collection
138+
return nitriteMap.size();
139+
}
140+
141+
private RecordStream<Pair<NitriteId, Document>> findSuitableStream(FindPlan findPlan, long[] indexedIdCount) {
117142
RecordStream<Pair<NitriteId, Document>> rawStream;
118143

119144
if (!findPlan.getSubPlans().isEmpty()) {
120145
// or filters get all sub stream by finding suitable stream of all sub plans
121146
List<RecordStream<Pair<NitriteId, Document>>> subStreams = new ArrayList<>();
122147
for (FindPlan subPlan : findPlan.getSubPlans()) {
123-
RecordStream<Pair<NitriteId, Document>> suitableStream = findSuitableStream(subPlan);
148+
// a sub-plan's own id count cannot answer the union's count (dedup), so discard it
149+
RecordStream<Pair<NitriteId, Document>> suitableStream = findSuitableStream(subPlan, new long[]{ -1 });
124150
subStreams.add(suitableStream);
125151
}
126152

@@ -148,6 +174,10 @@ private RecordStream<Pair<NitriteId, Document>> findSuitableStream(FindPlan find
148174
NitriteIndexer indexer = nitriteConfig.findIndexer(indexDescriptor.getIndexType());
149175
LinkedHashSet<NitriteId> nitriteIds = indexer.findByFilter(findPlan, nitriteConfig);
150176

177+
// the index supplied the exact matching id set; record its size so a size()
178+
// with no row-dropping step downstream can answer from it without fetching
179+
indexedIdCount[0] = nitriteIds.size();
180+
151181
// create indexed stream from optimized filter
152182
rawStream = new IndexedStream(nitriteIds, nitriteMap);
153183
} else {

‎nitrite/src/main/java/org/dizitart/no2/common/streams/DocumentStream.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.dizitart.no2.exceptions.InvalidOperationException;
3030
import org.dizitart.no2.exceptions.ValidationException;
3131
import org.dizitart.no2.common.processors.ProcessorChain;
32+
import org.dizitart.no2.common.util.Iterables;
3233

3334
import java.util.Collections;
3435
import java.util.Iterator;
@@ -44,12 +45,28 @@ public class DocumentStream implements DocumentCursor {
4445
@Getter @Setter
4546
private FindPlan findPlan;
4647

48+
/**
49+
* The exact match count when the query is fully answered by an index scan or a plain
50+
* full scan with no post-filter, skip, limit, or OR-union. When set, {@link #size()}
51+
* returns it directly instead of fetching and deserializing every matching document.
52+
*/
53+
@Setter
54+
private Long coveredCount;
55+
4756
public DocumentStream(RecordStream<Pair<NitriteId, Document>> recordStream,
4857
ProcessorChain processorChain) {
4958
this.recordStream = recordStream;
5059
this.processorChain = processorChain;
5160
}
5261

62+
@Override
63+
public long size() {
64+
if (coveredCount != null) {
65+
return coveredCount;
66+
}
67+
return Iterables.size(this);
68+
}
69+
5370
@Override
5471
public RecordStream<Document> project(Document projection) {
5572
validateProjection(projection);

‎nitrite/src/main/java/org/dizitart/no2/transaction/TransactionalMap.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,16 @@ public long size() {
150150
if (cleared) {
151151
return 0;
152152
}
153-
return backingMap.size();
153+
// Mirror entries(): the staged (backingMap) rows plus the committed primary rows that
154+
// are not tombstoned. Returning backingMap.size() alone ignores the primary rows the
155+
// cursor actually iterates, which makes a transactional find().size() under-count.
156+
long count = backingMap.size();
157+
for (K key : primary.keys()) {
158+
if (!tombstones.contains(key)) {
159+
count++;
160+
}
161+
}
162+
return count;
154163
}
155164

156165
@Override

‎nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionFindBySingleFieldIndexTest.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,46 @@ public void testCompoundIndexTerminalRangeMatchesFullScan() {
237237
and(where("folder").eq(7), where("date").gte(200), where("date").lte(400))).size());
238238
}
239239

240+
@Test
241+
public void testCoveredCountSizeMatchesIteration() {
242+
// size() may short-circuit to the index id-set size (or the map size) when the query is
243+
// fully index-covered with no post-filter, skip, or limit. That fast count must always
244+
// agree with actually draining the cursor - on both covered and non-covered paths.
245+
NitriteCollection coll = db.getCollection("covered_count");
246+
for (int age = 0; age < 100; age++) {
247+
coll.insert(Document.createDocument("age", age).put("even", age % 2 == 0));
248+
}
249+
coll.createIndex(IndexOptions.indexOptions(IndexType.NON_UNIQUE), "age");
250+
251+
// Plain find(): covered by the map size.
252+
assertEquals(countByIteration(coll.find()), coll.find().size());
253+
assertEquals(100, coll.find().size());
254+
255+
// Indexed equality and range: covered by the index id-set size.
256+
assertEquals(countByIteration(coll.find(where("age").eq(42))),
257+
coll.find(where("age").eq(42)).size());
258+
DocumentCursor range = coll.find(where("age").gte(30).and(where("age").lte(50)));
259+
assertEquals(countByIteration(range), range.size());
260+
261+
// Non-covered: a post-filter on an unindexed field must still count correctly (no
262+
// short-circuit), as must skip/limit.
263+
DocumentCursor postFiltered = coll.find(where("even").eq(true));
264+
assertEquals(countByIteration(postFiltered), postFiltered.size());
265+
assertEquals(50, postFiltered.size());
266+
DocumentCursor limited = coll.find(where("age").gte(0),
267+
org.dizitart.no2.collection.FindOptions.skipBy(10).limit(5));
268+
assertEquals(countByIteration(limited), limited.size());
269+
assertEquals(5, limited.size());
270+
}
271+
272+
private static long countByIteration(DocumentCursor cursor) {
273+
long count = 0;
274+
for (Document ignored : cursor) {
275+
count++;
276+
}
277+
return count;
278+
}
279+
240280
@Test
241281
public void testFindByNonUniqueIndex() throws ParseException {
242282
insert();

0 commit comments

Comments
 (0)