Fix spatial index access after reopening MVStore files (#1334)

- Read persisted R-tree cursor keys through H2's Spatial interface.
- Compare MVSpatialKey bounds with H2's deserialized spatial keys without unsafe casts.
- Verify intersecting and contained queries, updates, removals and a second reopen on a real file.
- Record the fix under Unreleased while preserving the on-disk format and public signatures.
- Validate all 5455 MVStore adapter tests with zero failures or errors and one skip.

Co-authored-by: GPT-6.1-Sol high <noreply@openai.com>

Author: sfkamath

CHANGELOG.md

@@ -1,3 +1,12 @@
+## Unreleased
+
+### Issue Fixes
+
+- Fix MVStore spatial-index queries, updates and removals after reopening a file. H2 reconstructs
+  persisted R-tree keys as its own `Spatial` implementation. Cursors and bounds comparisons now
+  accept that implementation instead of casting every key to `MVSpatialKey`. The on-disk format
+  and public method signatures are unchanged.
+
 ## Release 5.4.2 - Oct 19, 2026
 
 ### Security Fixes

nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVSpatialKey.java

@@ -40,6 +40,19 @@ public Spatial clone(long id) {
 
     @Override
     public boolean equalsIgnoringId(Spatial o) {
-        return Arrays.equals(minMax, ((MVSpatialKey)o).minMax);
+        if (o instanceof MVSpatialKey) {
+            return Arrays.equals(minMax, ((MVSpatialKey) o).minMax);
+        }
+        // Persisted keys are reconstructed by H2 as its own Spatial implementation.
+        if (isNull() || o.isNull()) {
+            return isNull() && o.isNull();
+        }
+        for (int dimension = 0; dimension < minMax.length / 2; dimension++) {
+            if (Float.compare(min(dimension), o.min(dimension)) != 0
+                || Float.compare(max(dimension), o.max(dimension)) != 0) {
+                return false;
+            }
+        }
+        return true;
     }
 }

nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVRTreeMap.java

@@ -31,6 +31,7 @@
 import org.dizitart.no2.store.NitriteStore;
 import org.h2.mvstore.MVStore;
 import org.h2.mvstore.rtree.MVRTreeMap;
+import org.h2.mvstore.rtree.Spatial;
 
 /**
  * @author Anindya Chatterjee
@@ -53,7 +54,7 @@ class NitriteMVRTreeMap<Key extends BoundingBox, Value> implements NitriteRTree<
     @Override
     public void add(final Key key, final NitriteId nitriteId) {
         if (nitriteId != null) {
-            final MVSpatialKey spatialKey = getKey(key, nitriteId.getIdValue());
+            final Spatial spatialKey = getKey(key, nitriteId.getIdValue());
             final MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
             try {
                 mvMap.add(spatialKey, key);
@@ -66,7 +67,7 @@ public void add(final Key key, final NitriteId nitriteId) {
     @Override
     public void remove(final Key key, final NitriteId nitriteId) {
         if (nitriteId != null) {
-            final MVSpatialKey spatialKey = getKey(key, nitriteId.getIdValue());
+            final Spatial spatialKey = getKey(key, nitriteId.getIdValue());
             final MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
             try {
                 mvMap.remove(spatialKey);
@@ -78,13 +79,13 @@ public void remove(final Key key, final NitriteId nitriteId) {
 
     @Override
     public RecordStream<NitriteId> findIntersectingKeys(final Key key) {
-        final MVSpatialKey spatialKey = getKey(key, 0L);
+        final Spatial spatialKey = getKey(key, 0L);
         return getRecordStream(() -> mvMap.findIntersectingKeys(spatialKey));
     }
 
     @Override
     public RecordStream<NitriteId> findContainedKeys(final Key key) {
-        final MVSpatialKey spatialKey = getKey(key, 0L);
+        final Spatial spatialKey = getKey(key, 0L);
         return getRecordStream(() -> mvMap.findContainedKeys(spatialKey));
     }
 
@@ -179,7 +180,7 @@ public NitriteId next() {
             }
             ensureOpen();
             try {
-                final MVSpatialKey next = (MVSpatialKey) treeCursor.next();
+                final Spatial next = treeCursor.next();
                 if (next == null) {
                     exhausted = true;
                     cleanable.clean();

nitrite-mvstore-adapter/src/test/java/org/dizitart/no2/mvstore/ReopenedRTreeTest.java

@@ -0,0 +1,71 @@
+package org.dizitart.no2.mvstore;
+
+import org.dizitart.no2.Nitrite;
+import org.dizitart.no2.collection.NitriteId;
+import org.dizitart.no2.common.RecordStream;
+import org.dizitart.no2.index.BoundingBox;
+import org.dizitart.no2.store.NitriteRTree;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+
+/** H2 reads persisted R-tree keys as DefaultSpatial rather than Nitrite's MVSpatialKey. */
+public class ReopenedRTreeTest {
+    @Rule
+    public TemporaryFolder directory = new TemporaryFolder();
+
+    @Test
+    public void testPersistedKeysCanBeQueriedUpdatedAndRemoved() {
+        File path = new File(directory.getRoot(), "spatial.db");
+        BoundingBox first = new BoundingBox(1, 2, 1, 2);
+        BoundingBox second = new BoundingBox(10, 11, 10, 11);
+        BoundingBox moved = new BoundingBox(20, 21, 20, 21);
+        BoundingBox all = new BoundingBox(0, 30, 0, 30);
+        NitriteId firstId = NitriteId.createId("1");
+        NitriteId secondId = NitriteId.createId("2");
+
+        try (Nitrite db = open(path)) {
+            NitriteRTree<BoundingBox, BoundingBox> tree = tree(db);
+            tree.add(first, firstId);
+            tree.add(second, secondId);
+            db.commit();
+        }
+        try (Nitrite db = open(path)) {
+            NitriteRTree<BoundingBox, BoundingBox> tree = tree(db);
+            assertIds(tree.findIntersectingKeys(all), "1", "2");
+            assertIds(tree.findContainedKeys(all), "1", "2");
+            tree.remove(first, firstId);
+            tree.add(moved, firstId);
+            assertIds(tree.findIntersectingKeys(first));
+            assertIds(tree.findIntersectingKeys(moved), "1");
+            tree.remove(second, secondId);
+            assertEquals(1, tree.size());
+            db.commit();
+        }
+        try (Nitrite db = open(path)) {
+            assertIds(tree(db).findContainedKeys(all), "1");
+        }
+    }
+
+    private static Nitrite open(File path) {
+        return Nitrite.builder().loadModule(MVStoreModule.withConfig().filePath(path)
+            .autoCompact(false).build()).openOrCreate();
+    }
+
+    private static NitriteRTree<BoundingBox, BoundingBox> tree(Nitrite db) {
+        return db.getStore().openRTree("bounds", BoundingBox.class, BoundingBox.class);
+    }
+
+    private static void assertIds(RecordStream<NitriteId> stream, String... ids) {
+        Set<NitriteId> expected = Arrays.stream(ids).map(NitriteId::createId).collect(Collectors.toSet());
+        assertEquals(expected, new HashSet<>(stream.toList()));
+    }
+}