Skip to content

Commit fa90809

Browse files
DarkAtraDarkAtra
committed
fix(gh-1284): MVStore backend can grow uncontrollably
1 parent d445e07 commit fa90809

11 files changed

Lines changed: 830 additions & 135 deletions

File tree

‎nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreModuleBuilder.java‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,26 @@
1616

1717
package org.dizitart.no2.mvstore;
1818

19+
import java.io.File;
20+
import java.util.HashSet;
21+
import java.util.Set;
22+
23+
import org.dizitart.no2.store.events.StoreEventListener;
24+
import org.h2.mvstore.FileStore;
25+
1926
import lombok.AccessLevel;
2027
import lombok.Getter;
2128
import lombok.Setter;
2229
import lombok.experimental.Accessors;
23-
import org.dizitart.no2.store.events.StoreEventListener;
24-
import org.h2.mvstore.FileStore;
25-
26-
import java.io.File;
27-
import java.util.HashSet;
28-
import java.util.Set;
2930

3031
/**
3132
* The MVStoreModuleBuilder class is responsible for building an instance of
3233
* {@link MVStoreModule}. It provides methods to set various configuration
3334
* options for the MVStore database.
34-
*
35-
* @since 4.0
36-
* @see MVStoreModule
35+
*
3736
* @author Anindya Chatterjee
37+
* @see MVStoreModule
38+
* @since 4.0
3839
*/
3940
@Getter
4041
@Setter
@@ -81,6 +82,13 @@ public class MVStoreModuleBuilder {
8182
*/
8283
private boolean autoCommit = true;
8384

85+
/**
86+
* Flag to enable/disable auto-compact mode. If set to true, fragmented
87+
* chunks or chunks that are sufficiently below the target fill rate of 90%
88+
* are reclaimed. This will typically shrink the file.
89+
*/
90+
private boolean autoCompact = true;
91+
8492
/**
8593
* Indicates whether the MVStore should be opened in recovery mode or not.
8694
*/
@@ -174,6 +182,7 @@ public MVStoreModule build() {
174182
dbConfig.compress(compress());
175183
dbConfig.compressHigh(compressHigh());
176184
dbConfig.autoCommit(autoCommit());
185+
dbConfig.autoCompact(autoCompact());
177186
dbConfig.recoveryMode(recoveryMode());
178187
dbConfig.cacheSize(cacheSize());
179188
dbConfig.cacheConcurrency(cacheConcurrency());

‎nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/MVStoreUtils.java‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,18 @@
1616

1717
package org.dizitart.no2.mvstore;
1818

19-
import lombok.extern.slf4j.Slf4j;
19+
import static org.dizitart.no2.common.util.StringUtils.isNullOrEmpty;
20+
21+
import java.io.File;
22+
import java.util.Map;
23+
2024
import org.dizitart.no2.exceptions.InvalidOperationException;
2125
import org.dizitart.no2.exceptions.NitriteIOException;
2226
import org.dizitart.no2.mvstore.compat.v1.UpgradeUtil;
2327
import org.h2.mvstore.MVStore;
2428
import org.h2.mvstore.MVStoreException;
2529

26-
import java.io.File;
27-
import java.util.Map;
28-
29-
import static org.dizitart.no2.common.util.StringUtils.isNullOrEmpty;
30+
import lombok.extern.slf4j.Slf4j;
3031

3132
/**
3233
* @author Anindya Chatterjee.
@@ -55,7 +56,7 @@ static MVStore openOrCreate(MVStoreConfig storeConfig) {
5556
}
5657
} catch (MVStoreException me) {
5758
if (me.getMessage().contains("file is locked")) {
58-
throw new NitriteIOException("Database is already opened in other process");
59+
throw new NitriteIOException("Database is already opened in other process", me);
5960
}
6061

6162
if (dbFile != null) {
@@ -128,8 +129,10 @@ private static MVStore.Builder createBuilder(MVStoreConfig mvStoreConfig) {
128129
builder = builder.autoCommitBufferSize(mvStoreConfig.autoCommitBufferSize());
129130
}
130131

131-
// auto compact disabled github issue #41
132-
builder.autoCompactFillRate(0);
132+
if (!mvStoreConfig.autoCompact()) {
133+
// disables background compaction
134+
builder.autoCompactFillRate(0);
135+
}
133136

134137
if (mvStoreConfig.encryptionKey() != null) {
135138
builder = builder.encryptionKey(mvStoreConfig.encryptionKey());

‎nitrite-mvstore-adapter/src/main/java/org/dizitart/no2/mvstore/NitriteMVMap.java‎

Lines changed: 120 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -16,45 +16,56 @@
1616

1717
package org.dizitart.no2.mvstore;
1818

19+
import static org.dizitart.no2.common.util.ValidationUtils.notNull;
20+
21+
import java.lang.ref.Cleaner;
22+
import java.util.Iterator;
23+
import java.util.Map;
24+
import java.util.NoSuchElementException;
25+
import java.util.Set;
26+
import java.util.concurrent.ConcurrentHashMap;
27+
import java.util.concurrent.atomic.AtomicBoolean;
28+
import java.util.function.Supplier;
29+
1930
import org.dizitart.no2.common.RecordStream;
2031
import org.dizitart.no2.common.tuples.Pair;
32+
import org.dizitart.no2.exceptions.NitriteIOException;
2133
import org.dizitart.no2.store.NitriteMap;
2234
import org.dizitart.no2.store.NitriteStore;
2335
import org.h2.mvstore.MVMap;
2436
import org.h2.mvstore.MVStore;
2537

26-
import java.util.Iterator;
27-
import java.util.Map;
28-
import java.util.concurrent.atomic.AtomicBoolean;
29-
30-
import static org.dizitart.no2.common.util.ValidationUtils.notNull;
31-
3238
/**
33-
* @since 1.0
3439
* @author Anindya Chatterjee
40+
* @since 1.0
3541
*/
3642
class NitriteMVMap<Key, Value> implements NitriteMap<Key, Value> {
43+
44+
private static final Cleaner CLEANER = Cleaner.create();
45+
3746
private final MVMap<Key, Value> mvMap;
3847
private final NitriteStore<?> nitriteStore;
3948
private final MVStore mvStore;
4049
private final AtomicBoolean droppedFlag;
4150
private final AtomicBoolean closedFlag;
51+
private final Set<VersionUsage> versionUsages;
4252

43-
NitriteMVMap(MVMap<Key, Value> mvMap, NitriteStore<?> nitriteStore) {
53+
NitriteMVMap(final MVMap<Key, Value> mvMap, final NitriteStore<?> nitriteStore) {
4454
this.mvMap = mvMap;
4555
this.nitriteStore = nitriteStore;
4656
this.mvStore = mvMap.getStore();
4757
this.closedFlag = new AtomicBoolean(false);
4858
this.droppedFlag = new AtomicBoolean(false);
59+
this.versionUsages = ConcurrentHashMap.newKeySet();
4960
}
5061

5162
@Override
52-
public boolean containsKey(Key key) {
63+
public boolean containsKey(final Key key) {
5364
return mvMap.containsKey(key);
5465
}
5566

5667
@Override
57-
public Value get(Key key) {
68+
public Value get(final Key key) {
5869
return mvMap.get(key);
5970
}
6071

@@ -65,7 +76,7 @@ public NitriteStore<?> getStore() {
6576

6677
@Override
6778
public void clear() {
68-
MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
79+
final MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
6980
try {
7081
mvMap.clear();
7182
updateLastModifiedTime();
@@ -81,14 +92,14 @@ public String getName() {
8192

8293
@Override
8394
public RecordStream<Value> values() {
84-
return RecordStream.fromIterable(mvMap.values());
95+
return () -> versionedIterator(() -> mvMap.values().iterator());
8596
}
8697

8798
@Override
88-
public Value remove(Key key) {
89-
MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
99+
public Value remove(final Key key) {
100+
final MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
90101
try {
91-
Value value = mvMap.remove(key);
102+
final Value value = mvMap.remove(key);
92103
updateLastModifiedTime();
93104
return value;
94105
} finally {
@@ -98,13 +109,13 @@ public Value remove(Key key) {
98109

99110
@Override
100111
public RecordStream<Key> keys() {
101-
return RecordStream.fromIterable(mvMap.keySet());
112+
return () -> versionedIterator(() -> mvMap.keySet().iterator());
102113
}
103114

104115
@Override
105-
public void put(Key key, Value value) {
116+
public void put(final Key key, final Value value) {
106117
notNull(value, "value cannot be null");
107-
MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
118+
final MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
108119
try {
109120
mvMap.put(key, value);
110121
updateLastModifiedTime();
@@ -119,11 +130,11 @@ public long size() {
119130
}
120131

121132
@Override
122-
public Value putIfAbsent(Key key, Value value) {
133+
public Value putIfAbsent(final Key key, final Value value) {
123134
notNull(value, "value cannot be null");
124-
MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
135+
final MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
125136
try {
126-
Value v = mvMap.putIfAbsent(key, value);
137+
final Value v = mvMap.putIfAbsent(key, value);
127138
updateLastModifiedTime();
128139
return v;
129140
} finally {
@@ -133,8 +144,8 @@ public Value putIfAbsent(Key key, Value value) {
133144

134145
@Override
135146
public RecordStream<Pair<Key, Value>> entries() {
136-
return () -> new Iterator<>() {
137-
final Iterator<Map.Entry<Key, Value>> entryIterator = mvMap.entrySet().iterator();
147+
return () -> versionedIterator(() -> new Iterator<>() {
148+
private final Iterator<Map.Entry<Key, Value>> entryIterator = mvMap.entrySet().iterator();
138149

139150
@Override
140151
public boolean hasNext() {
@@ -143,15 +154,19 @@ public boolean hasNext() {
143154

144155
@Override
145156
public Pair<Key, Value> next() {
146-
Map.Entry<Key, Value> entry = entryIterator.next();
157+
final Map.Entry<Key, Value> entry = entryIterator.next();
147158
return new Pair<>(entry.getKey(), entry.getValue());
148159
}
149-
};
160+
});
150161
}
151162

152163
@Override
153164
public RecordStream<Pair<Key, Value>> reversedEntries() {
154-
return () -> new ReverseIterator<>(mvMap);
165+
return () -> versionedIterator(() -> new ReverseIterator<>(mvMap));
166+
}
167+
168+
private <Element> Iterator<Element> versionedIterator(final Supplier<Iterator<Element>> iteratorSupplier) {
169+
return new VersionedIterator<>(mvStore, iteratorSupplier, versionUsages);
155170
}
156171

157172
@Override
@@ -165,22 +180,22 @@ public Key lastKey() {
165180
}
166181

167182
@Override
168-
public Key higherKey(Key key) {
183+
public Key higherKey(final Key key) {
169184
return mvMap.higherKey(key);
170185
}
171186

172187
@Override
173-
public Key ceilingKey(Key key) {
188+
public Key ceilingKey(final Key key) {
174189
return mvMap.ceilingKey(key);
175190
}
176191

177192
@Override
178-
public Key lowerKey(Key key) {
193+
public Key lowerKey(final Key key) {
179194
return mvMap.lowerKey(key);
180195
}
181196

182197
@Override
183-
public Key floorKey(Key key) {
198+
public Key floorKey(final Key key) {
184199
return mvMap.floorKey(key);
185200
}
186201

@@ -194,11 +209,12 @@ public void drop() {
194209
if (!droppedFlag.get()) {
195210
droppedFlag.compareAndSet(false, true);
196211
closedFlag.compareAndSet(false, true);
212+
releaseVersionUsages();
197213

198-
MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
214+
final MVStore.TxCounter txCounter = mvStore.registerVersionUsage();
199215
try {
200-
nitriteStore.closeMap(getName());
201-
nitriteStore.removeMap(getName());
216+
nitriteStore.closeMap(mvMap.getName());
217+
nitriteStore.removeMap(mvMap.getName());
202218
} finally {
203219
mvStore.deregisterVersionUsage(txCounter);
204220
}
@@ -214,12 +230,82 @@ public boolean isDropped() {
214230
public void close() {
215231
if (!closedFlag.get() && !droppedFlag.get()) {
216232
closedFlag.compareAndSet(false, true);
217-
nitriteStore.closeMap(getName());
233+
releaseVersionUsages();
234+
nitriteStore.closeMap(mvMap.getName());
218235
}
219236
}
220237

221238
@Override
222239
public boolean isClosed() {
223240
return closedFlag.get();
224241
}
242+
243+
private void releaseVersionUsages() {
244+
for (final VersionUsage versionUsage : versionUsages) {
245+
versionUsage.release();
246+
}
247+
}
248+
249+
private static class VersionedIterator<Element> implements Iterator<Element> {
250+
251+
private final Iterator<Element> iterator;
252+
private final Cleaner.Cleanable cleanable;
253+
private final VersionUsage versionUsage;
254+
private boolean exhausted;
255+
256+
private VersionedIterator(final MVStore mvStore,
257+
final Supplier<Iterator<Element>> iteratorSupplier,
258+
final Set<VersionUsage> versionUsages) {
259+
260+
versionUsage = new VersionUsage(mvStore, mvStore.registerVersionUsage(), versionUsages);
261+
versionUsages.add(versionUsage);
262+
263+
try {
264+
this.iterator = iteratorSupplier.get();
265+
this.cleanable = CLEANER.register(this, versionUsage::release);
266+
} catch (final RuntimeException | Error e) {
267+
versionUsage.release();
268+
throw e;
269+
}
270+
}
271+
272+
@Override
273+
public boolean hasNext() {
274+
if (exhausted) {
275+
return false;
276+
}
277+
ensureOpen();
278+
try {
279+
final boolean hasNext = iterator.hasNext();
280+
if (!hasNext) {
281+
exhausted = true;
282+
cleanable.clean();
283+
}
284+
return hasNext;
285+
} catch (final RuntimeException | Error e) {
286+
cleanable.clean();
287+
throw e;
288+
}
289+
}
290+
291+
@Override
292+
public Element next() {
293+
if (exhausted) {
294+
throw new NoSuchElementException();
295+
}
296+
ensureOpen();
297+
try {
298+
return iterator.next();
299+
} catch (final RuntimeException | Error e) {
300+
cleanable.clean();
301+
throw e;
302+
}
303+
}
304+
305+
private void ensureOpen() {
306+
if (versionUsage.isReleased()) {
307+
throw new NitriteIOException("MVStore is closed");
308+
}
309+
}
310+
}
225311
}

0 commit comments

Comments
 (0)