From cbb3cf89b8b7fb0a93e966568e652cc7bfc0aeac Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 11:51:05 +0800 Subject: [PATCH 01/33] partial --- .../cache/schema/DataNodeSchemaCache.java | 486 +++++++++------- .../schema/DataNodeSchemaCacheMetrics.java | 60 +- .../analyze/cache/schema/DeviceLastCache.java | 270 +++++++++ .../cache/schema/DeviceSchemaCache.java | 531 ++++++++++++++++++ .../analyze/cache/schema/IDeviceSchema.java | 24 + .../cache/schema/SchemaCacheEntry.java | 116 +--- .../cache/schema/TableDeviceCacheEntry.java | 224 ++++++++ .../cache/schema/TreeDeviceNormalSchema.java | 107 ++++ .../schema/TreeDeviceTemplateSchema.java | 49 ++ .../schema/dualkeycache/IDualKeyCache.java | 88 +-- .../dualkeycache/IDualKeyCacheStats.java | 6 + .../impl/CacheEntryGroupImpl.java | 59 +- .../schema/dualkeycache/impl/CacheStats.java | 37 +- .../dualkeycache/impl/DualKeyCacheImpl.java | 509 ++++++++--------- .../impl/FIFOCacheEntryManager.java | 77 ++- .../dualkeycache/impl/ICacheEntryGroup.java | 14 +- .../dualkeycache/impl/ICacheEntryManager.java | 13 +- .../impl/LRUCacheEntryManager.java | 57 +- .../commons/service/metric/enums/Metric.java | 1 + 19 files changed, 2040 insertions(+), 688 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/IDeviceSchema.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceNormalSchema.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceTemplateSchema.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 1c8ac8cb0a09e..d4ca47349876c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -22,30 +22,33 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.service.metric.MetricService; -import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.schema.view.LogicalViewSchema; +import org.apache.iotdb.db.exception.metadata.view.InsertNonWritableViewException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; -import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; +import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.IntFunction; -import java.util.function.IntPredicate; - -import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; +import java.util.stream.Collectors; +import java.util.stream.IntStream; /** * This class takes the responsibility of metadata cache management of all DataRegions under @@ -53,40 +56,23 @@ */ public class DataNodeSchemaCache { - private static final Logger logger = LoggerFactory.getLogger(DataNodeSchemaCache.class); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private final ITemplateManager templateManager = ClusterTemplateManager.getInstance(); - private final DeviceUsingTemplateSchemaCache deviceUsingTemplateSchemaCache; - - private final TimeSeriesSchemaCache timeSeriesSchemaCache; + private final DeviceSchemaCache deviceSchemaCache = new DeviceSchemaCache(); // cache update or clean have higher priority than cache read private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); private DataNodeSchemaCache() { - deviceUsingTemplateSchemaCache = new DeviceUsingTemplateSchemaCache(templateManager); - timeSeriesSchemaCache = new TimeSeriesSchemaCache(); - - MetricService.getInstance().addMetricSet(new DataNodeSchemaCacheMetrics(this)); - } - - public long getHitCount() { - return deviceUsingTemplateSchemaCache.getHitCount() + timeSeriesSchemaCache.getHitCount(); - } - - public long getRequestCount() { - return deviceUsingTemplateSchemaCache.getRequestCount() - + timeSeriesSchemaCache.getRequestCount(); + // private constructor } public static DataNodeSchemaCache getInstance() { - return DataNodeSchemaCacheHolder.INSTANCE; + return TreeDeviceSchemaCacheManagerHolder.INSTANCE; } /** singleton pattern. */ - private static class DataNodeSchemaCacheHolder { + private static class TreeDeviceSchemaCacheManagerHolder { private static final DataNodeSchemaCache INSTANCE = new DataNodeSchemaCache(); } @@ -113,18 +99,27 @@ public void releaseWriteLock() { * @param measurements * @return timeseries partialPath and its SchemaEntity */ - public ClusterSchemaTree get(PartialPath devicePath, String[] measurements) { - return timeSeriesSchemaCache.get(devicePath, measurements); - } - - public ClusterSchemaTree get(PartialPath fullPath) { - ClusterSchemaTree clusterSchemaTree = - deviceUsingTemplateSchemaCache.get(fullPath.getDevicePath()); - if (clusterSchemaTree == null || clusterSchemaTree.isEmpty()) { - return timeSeriesSchemaCache.get(fullPath); - } else { - return clusterSchemaTree; + public ClusterSchemaTree get(final PartialPath devicePath, final String[] measurements) { + final ClusterSchemaTree tree = new ClusterSchemaTree(); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getNodes()); + if (!(schema instanceof TreeDeviceNormalSchema)) { + return tree; } + final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; + for (final String measurement : measurements) { + final SchemaCacheEntry entry = treeSchema.getSchemaCacheEntry(measurement); + if (Objects.nonNull(entry)) { + tree.appendSingleMeasurement( + devicePath.concatNode(measurement), + entry.getSchema(), + entry.getTagMap(), + null, + null, + treeSchema.isAligned()); + } + } + tree.setDatabases(Collections.singleton(treeSchema.getDatabase())); + return tree; } /** @@ -133,8 +128,17 @@ public ClusterSchemaTree get(PartialPath fullPath) { * @param devicePath full path of the device * @return empty if cache miss or the device path is not a template activated path */ - public ClusterSchemaTree getMatchedSchemaWithTemplate(PartialPath devicePath) { - return deviceUsingTemplateSchemaCache.getMatchedSchemaWithTemplate(devicePath); + public ClusterSchemaTree getMatchedSchemaWithTemplate(final PartialPath devicePath) { + final ClusterSchemaTree tree = new ClusterSchemaTree(); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getNodes()); + if (!(schema instanceof TreeDeviceTemplateSchema)) { + return tree; + } + final TreeDeviceTemplateSchema treeSchema = (TreeDeviceTemplateSchema) schema; + Template template = templateManager.getTemplate(treeSchema.getTemplateId()); + tree.appendTemplateDevice(devicePath, template.isDirectAligned(), template.getId(), template); + tree.setDatabases(Collections.singleton(treeSchema.getDatabase())); + return tree; } /** @@ -143,14 +147,49 @@ public ClusterSchemaTree getMatchedSchemaWithTemplate(PartialPath devicePath) { * @param fullPath full path * @return empty if cache miss */ - public ClusterSchemaTree getMatchedSchemaWithoutTemplate(PartialPath fullPath) { - return timeSeriesSchemaCache.get(fullPath); + public ClusterSchemaTree getMatchedSchemaWithoutTemplate(final PartialPath fullPath) { + final ClusterSchemaTree tree = new ClusterSchemaTree(); + final IDeviceSchema schema = + deviceSchemaCache.getDeviceSchema( + Arrays.copyOf(fullPath.getNodes(), fullPath.getNodeLength() - 1)); + if (!(schema instanceof TreeDeviceNormalSchema)) { + return tree; + } + final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; + final SchemaCacheEntry entry = treeSchema.getSchemaCacheEntry(fullPath.getMeasurement()); + if (Objects.isNull(entry)) { + return tree; + } + tree.appendSingleMeasurement( + fullPath, entry.getSchema(), entry.getTagMap(), null, null, treeSchema.isAligned()); + tree.setDatabases(Collections.singleton(treeSchema.getDatabase())); + return tree; } - public List computeWithoutTemplate(ISchemaComputation schemaComputation) { - List result = timeSeriesSchemaCache.computeAndRecordLogicalView(schemaComputation); + public List computeWithoutTemplate(final ISchemaComputation schemaComputation) { + final List indexOfMissingMeasurements = new ArrayList<>(); + final String[] measurements = schemaComputation.getMeasurements(); + + final IDeviceSchema schema = + deviceSchemaCache.getDeviceSchema(schemaComputation.getDevicePath().getNodes()); + if (!(schema instanceof TreeDeviceNormalSchema)) { + return IntStream.range(0, schemaComputation.getMeasurements().length) + .boxed() + .collect(Collectors.toList()); + } + final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; + + for (int i = 0; i < schemaComputation.getMeasurements().length; i++) { + final SchemaCacheEntry value = treeSchema.getSchemaCacheEntry(measurements[i]); + if (value == null) { + indexOfMissingMeasurements.add(i); + } else { + schemaComputation.computeMeasurement(i, value); + } + } + schemaComputation.computeDevice(treeSchema.isAligned()); schemaComputation.recordRangeOfLogicalViewSchemaListNow(); - return result; + return indexOfMissingMeasurements; } /** @@ -163,177 +202,248 @@ public List computeWithoutTemplate(ISchemaComputation schemaComputation * @return The indexes of missed views and full paths of their source paths will be returned. */ public Pair, List> computeSourceOfLogicalView( - ISchemaComputation schemaComputation) { + final ISchemaComputation schemaComputation) { if (!schemaComputation.hasLogicalViewNeedProcess()) { return new Pair<>(new ArrayList<>(), new ArrayList<>()); } - return timeSeriesSchemaCache.computeSourceOfLogicalView(schemaComputation); - } - public List computeWithTemplate(ISchemaComputation schemaComputation) { - return deviceUsingTemplateSchemaCache.compute(schemaComputation); + final List indexOfMissingMeasurements = new ArrayList<>(); + final Pair beginToEnd = + schemaComputation.getRangeOfLogicalViewSchemaListRecorded(); + final List logicalViewSchemaList = + schemaComputation.getLogicalViewSchemaList(); + final List indexListOfLogicalViewPaths = + schemaComputation.getIndexListOfLogicalViewPaths(); + + for (int i = beginToEnd.left; i < beginToEnd.right; i++) { + final LogicalViewSchema logicalViewSchema = logicalViewSchemaList.get(i); + final int realIndex = indexListOfLogicalViewPaths.get(i); + if (!logicalViewSchema.isWritable()) { + throw new RuntimeException( + new InsertNonWritableViewException( + schemaComputation + .getDevicePath() + .concatAsMeasurementPath(schemaComputation.getMeasurements()[realIndex]) + .getFullPath())); + } + + final PartialPath fullPath = logicalViewSchema.getSourcePathIfWritable(); + final IDeviceSchema schema = + deviceSchemaCache.getDeviceSchema(fullPath.getDevicePath().getNodes()); + if (!(schema instanceof TreeDeviceNormalSchema)) { + indexOfMissingMeasurements.add(i); + continue; + } + + final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; + final SchemaCacheEntry value = treeSchema.getSchemaCacheEntry(fullPath.getMeasurement()); + if (Objects.isNull(value)) { + indexOfMissingMeasurements.add(i); + continue; + } + + if (value.isLogicalView()) { + throw new RuntimeException( + new UnsupportedOperationException( + String.format( + "The source of view [%s] is also a view! Nested view is unsupported! " + + "Please check it.", + logicalViewSchema.getSourcePathIfWritable()))); + } + + schemaComputation.computeMeasurementOfView(realIndex, value, treeSchema.isAligned()); + } + + return new Pair<>( + indexOfMissingMeasurements, + indexOfMissingMeasurements.stream() + .map(index -> logicalViewSchemaList.get(index).getSourcePathStringIfWritable()) + .collect(Collectors.toList())); } - /** - * Store the fetched schema in either the schemaCache or templateSchemaCache, depending on its - * associated device. - */ - public void put(ClusterSchemaTree tree) { - PartialPath devicePath; - for (DeviceSchemaInfo deviceSchemaInfo : tree.getAllDevices()) { - devicePath = deviceSchemaInfo.getDevicePath(); - if (deviceSchemaInfo.getTemplateId() != NON_TEMPLATE) { - deviceUsingTemplateSchemaCache.put( - devicePath, tree.getBelongedDatabase(devicePath), deviceSchemaInfo.getTemplateId()); - } else { - for (MeasurementPath measurementPath : deviceSchemaInfo.getMeasurementSchemaPathList()) { - timeSeriesSchemaCache.putSingleMeasurementPath( - tree.getBelongedDatabase(devicePath), measurementPath); - } + public List computeWithTemplate(final ISchemaComputation computation) { + final List indexOfMissingMeasurements = new ArrayList<>(); + final String[] measurements = computation.getMeasurements(); + final IDeviceSchema deviceSchema = + deviceSchemaCache.getDeviceSchema(computation.getDevicePath().getNodes()); + + if (!(deviceSchema instanceof TreeDeviceTemplateSchema)) { + return IntStream.range(0, measurements.length).boxed().collect(Collectors.toList()); + } + + final TreeDeviceTemplateSchema deviceTemplateSchema = (TreeDeviceTemplateSchema) deviceSchema; + + computation.computeDevice( + templateManager.getTemplate(deviceTemplateSchema.getTemplateId()).isDirectAligned()); + final Map templateSchema = + templateManager.getTemplate(deviceTemplateSchema.getTemplateId()).getSchemaMap(); + for (int i = 0; i < measurements.length; i++) { + if (!templateSchema.containsKey(measurements[i])) { + indexOfMissingMeasurements.add(i); + continue; } + final IMeasurementSchema schema = templateSchema.get(measurements[i]); + computation.computeMeasurement( + i, + new IMeasurementSchemaInfo() { + @Override + public String getName() { + return schema.getMeasurementName(); + } + + @Override + public IMeasurementSchema getSchema() { + if (isLogicalView()) { + return new LogicalViewSchema( + schema.getMeasurementName(), ((LogicalViewSchema) schema).getExpression()); + } else { + return this.getSchemaAsMeasurementSchema(); + } + } + + @Override + public MeasurementSchema getSchemaAsMeasurementSchema() { + return new MeasurementSchema( + schema.getMeasurementName(), + schema.getType(), + schema.getEncodingType(), + schema.getCompressor()); + } + + @Override + public LogicalViewSchema getSchemaAsLogicalViewSchema() { + throw new RuntimeException( + new UnsupportedOperationException( + "Function getSchemaAsLogicalViewSchema is not supported in DeviceUsingTemplateSchemaCache.")); + } + + @Override + public Map getTagMap() { + return null; + } + + @Override + public Map getAttributeMap() { + return null; + } + + @Override + public String getAlias() { + return null; + } + + @Override + public boolean isLogicalView() { + return schema.isLogicalView(); + } + }); } + return indexOfMissingMeasurements; } - public TimeValuePair getLastCache(PartialPath seriesPath) { - return timeSeriesSchemaCache.getLastCache(seriesPath); + /** + * Store the fetched schema in either the {@link TreeDeviceNormalSchema} or {@link + * TreeDeviceTemplateSchema}, depending on its associated device. + */ + public void put(final ClusterSchemaTree tree) { + tree.getAllDevices() + .forEach( + deviceSchemaInfo -> + deviceSchemaCache.putDeviceSchema( + tree.getBelongedDatabase(deviceSchemaInfo.getDevicePath()), deviceSchemaInfo)); } - public boolean getLastCache(final Map> inputMap) { - return timeSeriesSchemaCache.getLastCache(inputMap); + public TimeValuePair getLastCache(final MeasurementPath seriesPath) { + return deviceSchemaCache.getLastEntry( + null, seriesPath.getIDeviceID(), seriesPath.getMeasurement()); } - public void invalidateLastCache(PartialPath path) { + public void invalidateLastCache(final MeasurementPath path) { if (!CommonDescriptor.getInstance().getConfig().isLastCacheEnable()) { return; } - takeReadLock(); - try { - timeSeriesSchemaCache.invalidateLastCache(path); - } finally { - releaseReadLock(); - } + deviceSchemaCache.invalidateLastCache(path.getDevicePath(), path.getMeasurement()); } - public void invalidateLastCacheInDataRegion(String database) { + public void invalidateDatabaseLastCache(final String database) { if (!CommonDescriptor.getInstance().getConfig().isLastCacheEnable()) { return; } - takeReadLock(); - try { - timeSeriesSchemaCache.invalidateDataRegionLastCache(database); - } finally { - releaseReadLock(); - } + deviceSchemaCache.invalidateLastCache(database); } - /** get SchemaCacheEntry and update last cache */ - @TestOnly - public void updateLastCache( - PartialPath devicePath, - String measurement, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - timeSeriesSchemaCache.updateLastCache( - devicePath, measurement, timeValuePair, highPriorityUpdate, latestFlushedTime); - } - - public void updateLastCache( - String database, - PartialPath devicePath, - String[] measurements, - MeasurementSchema[] measurementSchemas, - boolean isAligned, - IntFunction timeValuePairProvider, - IntPredicate shouldUpdateProvider, - boolean highPriorityUpdate, - Long latestFlushedTime) { - takeReadLock(); - try { - timeSeriesSchemaCache.updateLastCache( - database, - devicePath, - measurements, - measurementSchemas, - isAligned, - timeValuePairProvider, - shouldUpdateProvider, - highPriorityUpdate, - latestFlushedTime); - } finally { - releaseReadLock(); - } - } - - public void updateLastCacheWithoutLock( - String database, - PartialPath devicePath, - String[] measurements, - MeasurementSchema[] measurementSchemas, - boolean isAligned, - IntFunction timeValuePairProvider, - IntPredicate shouldUpdateProvider, - boolean highPriorityUpdate, - Long latestFlushedTime) { - timeSeriesSchemaCache.updateLastCache( - database, - devicePath, - measurements, - measurementSchemas, - isAligned, - timeValuePairProvider, - shouldUpdateProvider, - highPriorityUpdate, - latestFlushedTime); + /** + * Update the {@link DeviceLastCache} in writing for tree model. If a measurement is with all + * {@code null}s or is an id/attribute column, its {@link TimeValuePair[]} shall be {@code null}. + * For correctness, this will put the {@link TableDeviceCacheEntry} lazily and only update the + * existing {@link DeviceLastCache}s of measurements. + * + * @param database the device's database, WITH "root" + * @param deviceID {@link IDeviceID} + * @param measurements the fetched measurements + * @param timeValuePairs the {@link TimeValuePair}s with indexes corresponding to the measurements + */ + public void updateLastCacheIfExists( + final String database, + final IDeviceID deviceID, + final String[] measurements, + final @Nonnull TimeValuePair[] timeValuePairs, + final boolean isAligned, + final IMeasurementSchema[] measurementSchemas) { + deviceSchemaCache.updateLastCache( + database, deviceID, measurements, timeValuePairs, isAligned, measurementSchemas, false); } /** - * get or create SchemaCacheEntry and update last cache, only support non-aligned sensor or - * aligned sensor without only one sub sensor + * Update the {@link DeviceLastCache} on query in tree model. + * + *

Note: The query shall put the {@link DeviceLastCache} twice: + * + *

- First time set the "isCommit" to {@code false} before the query accesses data. It is just + * to allow the writing to update the cache, then avoid that the query put a stale value to cache + * and break the consistency. WARNING: The writing may temporarily put a stale value in cache if a + * stale value is written, but it won't affect the eventual consistency. + * + *

- Second time put the calculated {@link TimeValuePair}, and use {@link + * #updateLastCacheIfExists(String, IDeviceID, String[], TimeValuePair[], boolean, + * IMeasurementSchema[])}. The input {@link TimeValuePair} shall never be or contain {@code null}, + * if the measurement is with all {@code null}s, its {@link TimeValuePair} shall be {@link + * DeviceLastCache#EMPTY_TIME_VALUE_PAIR}. This method is not supposed to update time column. + * + *

If the query has ended abnormally, it shall call this to invalidate the entry it has pushed + * in the first time, to avoid the stale writing damaging the eventual consistency. In this case + * and the "isInvalidate" shall be {@code true}. + * + * @param database the device's database, WITH "root" + * @param measurementPath the fetched {@link MeasurementPath} + * @param isInvalidate {@code true} if invalidate the first pushed cache, or {@code null} for the + * first fetch. */ public void updateLastCache( - String storageGroup, - MeasurementPath measurementPath, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - takeReadLock(); - try { - timeSeriesSchemaCache.updateLastCache( - storageGroup, measurementPath, timeValuePair, highPriorityUpdate, latestFlushedTime); - } finally { - releaseReadLock(); - } - } - - public void invalidate(String database) { - deviceUsingTemplateSchemaCache.invalidateCache(database); - timeSeriesSchemaCache.invalidate(database); - } - - public void invalidate(List partialPathList) { - boolean doPrecise = true; - for (PartialPath partialPath : partialPathList) { - if (partialPath.getDevicePath().hasWildcard()) { - doPrecise = false; - break; - } - } - if (doPrecise) { - deviceUsingTemplateSchemaCache.invalidateCache(partialPathList); - timeSeriesSchemaCache.invalidate(partialPathList); - } else { - invalidateAll(); - } + final String database, final MeasurementPath measurementPath, final boolean isInvalidate) { + deviceSchemaCache.updateLastCache( + database, + measurementPath.getIDeviceID(), + new String[] {measurementPath.getMeasurement()}, + isInvalidate ? new TimeValuePair[] {null} : null, + measurementPath.isUnderAlignedEntity(), + new IMeasurementSchema[] {measurementPath.getMeasurementSchema()}, + true); } - public void invalidateAll() { - deviceUsingTemplateSchemaCache.invalidateCache(); - timeSeriesSchemaCache.invalidateAll(); + public void invalidate(final List partialPathList) { + // Currently invalidate by device + partialPathList.forEach( + measurementPath -> { + final boolean isMultiLevelWildcardMeasurement = + PathPatternUtil.isMultiLevelMatchWildcard(measurementPath.getMeasurement()); + deviceSchemaCache.invalidateCache( + isMultiLevelWildcardMeasurement ? measurementPath : measurementPath.getDevicePath(), + isMultiLevelWildcardMeasurement); + }); } public void cleanUp() { - deviceUsingTemplateSchemaCache.invalidateCache(); - timeSeriesSchemaCache.invalidateAll(); + deviceSchemaCache.invalidateAll(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCacheMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCacheMetrics.java index 696d28c5118a9..bb567143408e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCacheMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCacheMetrics.java @@ -29,19 +29,23 @@ import java.util.Objects; public class DataNodeSchemaCacheMetrics implements IMetricSet { - private DataNodeSchemaCache dataNodeSchemaCache; - public DataNodeSchemaCacheMetrics(DataNodeSchemaCache dataNodeSchemaCache) { - this.dataNodeSchemaCache = dataNodeSchemaCache; + private static final String SCHEMA_CACHE_TOTAL_USAGE = "schema_cache_total_usage"; + private static final String SCHEMA_CACHE_MEM_CAPACITY = "schema_cache_mem_capacity"; + + private final DeviceSchemaCache deviceSchemaCache; + + public DataNodeSchemaCacheMetrics(final DeviceSchemaCache dataNodeSchemaCache) { + this.deviceSchemaCache = dataNodeSchemaCache; } @Override - public void bindTo(AbstractMetricService metricService) { + public void bindTo(final AbstractMetricService metricService) { metricService.createAutoGauge( Metric.CACHE.toString(), MetricLevel.IMPORTANT, - dataNodeSchemaCache, - DataNodeSchemaCache::getHitCount, + deviceSchemaCache, + DeviceSchemaCache::getHitCount, Tag.NAME.toString(), "SchemaCache", Tag.TYPE.toString(), @@ -49,16 +53,35 @@ public void bindTo(AbstractMetricService metricService) { metricService.createAutoGauge( Metric.CACHE.toString(), MetricLevel.IMPORTANT, - dataNodeSchemaCache, - DataNodeSchemaCache::getRequestCount, + deviceSchemaCache, + DeviceSchemaCache::getRequestCount, Tag.NAME.toString(), "SchemaCache", Tag.TYPE.toString(), "all"); + metricService.createAutoGauge( + Metric.MEM.toString(), + MetricLevel.IMPORTANT, + deviceSchemaCache, + DeviceSchemaCache::getMemoryUsage, + Tag.NAME.toString(), + SCHEMA_CACHE_TOTAL_USAGE); + metricService.createAutoGauge( + Metric.MEM.toString(), + MetricLevel.IMPORTANT, + deviceSchemaCache, + DeviceSchemaCache::capacity, + Tag.NAME.toString(), + SCHEMA_CACHE_MEM_CAPACITY); + metricService.createAutoGauge( + Metric.CACHE_ENTRIES_NUM.toString(), + MetricLevel.IMPORTANT, + deviceSchemaCache, + DeviceSchemaCache::entriesCount); } @Override - public void unbindFrom(AbstractMetricService metricService) { + public void unbindFrom(final AbstractMetricService metricService) { metricService.remove( MetricType.AUTO_GAUGE, Metric.CACHE.toString(), @@ -73,22 +96,33 @@ public void unbindFrom(AbstractMetricService metricService) { "SchemaCache", Tag.TYPE.toString(), "all"); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.MEM.toString(), + Tag.NAME.toString(), + SCHEMA_CACHE_TOTAL_USAGE); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.MEM.toString(), + Tag.NAME.toString(), + SCHEMA_CACHE_MEM_CAPACITY); + metricService.remove(MetricType.AUTO_GAUGE, Metric.CACHE_ENTRIES_NUM.toString()); } @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - DataNodeSchemaCacheMetrics that = (DataNodeSchemaCacheMetrics) o; - return Objects.equals(dataNodeSchemaCache, that.dataNodeSchemaCache); + final DataNodeSchemaCacheMetrics that = (DataNodeSchemaCacheMetrics) o; + return Objects.equals(deviceSchemaCache, that.deviceSchemaCache); } @Override public int hashCode() { - return Objects.hash(dataNodeSchemaCache); + return Objects.hash(deviceSchemaCache); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java new file mode 100644 index 0000000000000..4cc6ab9369a13 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +@ThreadSafe +public class DeviceLastCache { + static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(DeviceLastCache.class) + + (int) RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); + + public static final TsPrimitiveType EMPTY_PRIMITIVE_TYPE = + new TsPrimitiveType() { + @Override + public void setObject(Object o) { + // Do nothing + } + + @Override + public void reset() { + // Do nothing + } + + @Override + public int getSize() { + return 0; + } + + @Override + public Object getValue() { + return null; + } + + @Override + public String getStringValue() { + return null; + } + + @Override + public TSDataType getDataType() { + return null; + } + }; + + private static final Optional> HIT_AND_ALL_NULL = + Optional.of(new Pair<>(OptionalLong.empty(), null)); + public static final TimeValuePair EMPTY_TIME_VALUE_PAIR = + new TimeValuePair(Long.MIN_VALUE, EMPTY_PRIMITIVE_TYPE); + private static final TimeValuePair PLACEHOLDER_TIME_VALUE_PAIR = + new TimeValuePair(Long.MIN_VALUE, EMPTY_PRIMITIVE_TYPE); + + // Time is seen as "" as a measurement + private final Map measurement2CachedLastMap = new ConcurrentHashMap<>(); + + int initOrInvalidate( + final String tableName, final String[] measurements, final boolean isInvalidate) { + final AtomicInteger diff = new AtomicInteger(0); + + for (final String measurement : measurements) { + // Removing table measurement, do not put cache + if (Objects.isNull(measurement)) { + continue; + } + final TimeValuePair newPair = isInvalidate ? null : PLACEHOLDER_TIME_VALUE_PAIR; + + measurement2CachedLastMap.compute( + measurement, + (measurementKey, tvPair) -> { + if (Objects.isNull(newPair)) { + diff.addAndGet( + -((false ? 0 : (int) RamUsageEstimator.sizeOf(measurement)) + + getTVPairEntrySize(tvPair))); + return null; + } + if (Objects.isNull(tvPair)) { + diff.addAndGet( + (false ? 0 : (int) RamUsageEstimator.sizeOf(measurement)) + + (int) RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY); + return newPair; + } + return tvPair; + }); + } + return diff.get(); + } + + int tryUpdate( + final @Nonnull String[] measurements, final @Nonnull TimeValuePair[] timeValuePairs) { + return tryUpdate(measurements, timeValuePairs, false); + } + + int tryUpdate( + final @Nonnull String[] measurements, + final @Nonnull TimeValuePair[] timeValuePairs, + final boolean invalidateNull) { + final AtomicInteger diff = new AtomicInteger(0); + long lastTime = Long.MIN_VALUE; + + for (int i = 0; i < measurements.length; ++i) { + if (Objects.isNull(timeValuePairs[i])) { + if (invalidateNull) { + measurement2CachedLastMap.remove(measurements[i]); + } + continue; + } + + final int finalI = i; + if (lastTime < timeValuePairs[i].getTimestamp()) { + lastTime = timeValuePairs[i].getTimestamp(); + } + measurement2CachedLastMap.computeIfPresent( + measurements[i], + (measurement, tvPair) -> { + if (tvPair.getTimestamp() <= timeValuePairs[finalI].getTimestamp()) { + diff.addAndGet(getDiffSize(tvPair, timeValuePairs[finalI])); + return timeValuePairs[finalI]; + } + return tvPair; + }); + } + final long finalLastTime = lastTime; + measurement2CachedLastMap.computeIfPresent( + "", + (time, tvPair) -> + tvPair.getTimestamp() < finalLastTime + ? new TimeValuePair(finalLastTime, EMPTY_PRIMITIVE_TYPE) + : tvPair); + return diff.get(); + } + + @GuardedBy("DataRegionInsertLock#writeLock") + int invalidate(final String measurement, final boolean isTableModel) { + final AtomicInteger diff = new AtomicInteger(); + final AtomicLong time = new AtomicLong(); + measurement2CachedLastMap.computeIfPresent( + measurement, + (s, timeValuePair) -> { + diff.set( + (isTableModel ? 0 : (int) RamUsageEstimator.sizeOf(s)) + + getTVPairEntrySize(timeValuePair)); + time.set(timeValuePair.getTimestamp()); + return null; + }); + if (diff.get() == 0) { + return 0; + } + measurement2CachedLastMap.computeIfPresent( + "", + (s, timeValuePair) -> { + if (timeValuePair.getTimestamp() <= time.get()) { + diff.addAndGet(getTVPairEntrySize(timeValuePair)); + return null; + } + return timeValuePair; + }); + + return diff.get(); + } + + private int getTVPairEntrySize(final TimeValuePair tvPair) { + return (int) RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + ((Objects.isNull(tvPair) + || tvPair == PLACEHOLDER_TIME_VALUE_PAIR + || tvPair == EMPTY_TIME_VALUE_PAIR) + ? 0 + : tvPair.getSize()); + } + + @Nullable + TimeValuePair getTimeValuePair(final @Nonnull String measurement) { + final TimeValuePair result = measurement2CachedLastMap.get(measurement); + return result != PLACEHOLDER_TIME_VALUE_PAIR ? result : null; + } + + // Shall pass in "" if last by time + Optional> getLastRow( + final @Nonnull String sourceMeasurement, final List targetMeasurements) { + final TimeValuePair pair = measurement2CachedLastMap.get(sourceMeasurement); + if (Objects.isNull(pair) || pair == PLACEHOLDER_TIME_VALUE_PAIR) { + return Optional.empty(); + } + + if (pair == EMPTY_TIME_VALUE_PAIR) { + return HIT_AND_ALL_NULL; + } + final long alignTime = pair.getTimestamp(); + + return Optional.of( + new Pair<>( + OptionalLong.of(alignTime), + targetMeasurements.stream() + .map( + targetMeasurement -> { + if (!targetMeasurement.isEmpty()) { + final TimeValuePair tvPair = + measurement2CachedLastMap.get(targetMeasurement); + if (Objects.isNull(tvPair)) { + return null; + } + return tvPair.getTimestamp() == alignTime + ? tvPair.getValue() + : EMPTY_PRIMITIVE_TYPE; + } else { + return new TsPrimitiveType.TsLong(alignTime); + } + }) + .toArray(TsPrimitiveType[]::new))); + } + + int estimateSize() { + return INSTANCE_SIZE + + (int) RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY * measurement2CachedLastMap.size() + + measurement2CachedLastMap.values().stream() + .mapToInt(TimeValuePair::getSize) + .reduce(0, Integer::sum); + } + + private static int getDiffSize( + final TimeValuePair oldTimeValuePair, final TimeValuePair newTimeValuePair) { + if (oldTimeValuePair == EMPTY_TIME_VALUE_PAIR + || oldTimeValuePair == PLACEHOLDER_TIME_VALUE_PAIR) { + return newTimeValuePair.getSize(); + } + final TsPrimitiveType oldValue = oldTimeValuePair.getValue(); + final TsPrimitiveType newValue = newTimeValuePair.getValue(); + if (oldValue == null) { + return newValue == null ? 0 : newValue.getSize(); + } else if (oldValue instanceof TsPrimitiveType.TsBinary) { + return (newValue == null ? 0 : newValue.getSize()) - oldValue.getSize(); + } else { + return newValue == null ? -oldValue.getSize() : 0; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java new file mode 100644 index 0000000000000..827b2e2219ebd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -0,0 +1,531 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.ExtendedPartialPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.db.conf.DataNodeMemoryConfig; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegion; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.ToIntFunction; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; + +/** + * The {@link DeviceSchemaCache} caches some of the devices and their: attributes of tables / + * measurement info / template info. The last value of one device is also cached here. Here are the + * semantics of attributes, other semantics are omitted: + * + *

1. If a deviceId misses cache, it does not necessarily mean that the device does not exist, + * Since the cache records only part of the devices. + * + *

2. If a device is in cache, the attributes will be finally identical to the {@link + * SchemaRegion}'s version. In reading this may temporarily return false result, and in writing when + * the new attribute differs from the original ones, we send the new attributes to the {@link + * SchemaRegion} anyway. This may not update the attributes in {@link SchemaRegion} since the + * attributes in cache may be stale, but it's okay and {@link SchemaRegion} will just do nothing. + * + *

3. When the attributeMap does not contain an attributeKey, then the value is {@code null}. + * Note that we do not tell whether an attributeKey exists here, and it shall be judged from table + * schema. + */ +@ThreadSafe +public class DeviceSchemaCache { + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final DataNodeMemoryConfig memoryConfig = + IoTDBDescriptor.getInstance().getMemoryConfig(); + private static final Logger logger = LoggerFactory.getLogger(DeviceSchemaCache.class); + + /** + * In table model: {@literal <}{@link QualifiedObjectName}, {@link IDeviceID}, lastCache / + * attributes{@literal >} + * + *

In tree model: {@literal <}Pair{@literal <}{@code null}, tableName(translated){@literal >}, + * {@link IDeviceID}(translated), Map{@literal <}Measurement, Schema{@literal + * >}/templateInfo{@literal >} + */ + private final IDualKeyCache dualKeyCache; + + private final Map treeModelDatabasePool = new ConcurrentHashMap<>(); + + private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); + + DeviceSchemaCache() { + dualKeyCache = + new DualKeyCacheBuilder() + .cacheEvictionPolicy( + DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) + .memoryCapacity(config.getAllocateMemoryForSchemaCache()) + .firstKeySizeComputer(TableId::estimateSize) + .secondKeySizeComputer(deviceID -> (int) deviceID.ramBytesUsed()) + .valueSizeComputer(TableDeviceCacheEntry::estimateSize) + .build(); + MetricService.getInstance().addMetricSet(new DataNodeSchemaCacheMetrics(this)); + } + + /////////////////////////////// Last Cache /////////////////////////////// + + /** + * Update the last cache in writing or the second push of last cache query. If a measurement is + * with all {@code null}s or is a tag/attribute column, its {@link TimeValuePair}[] shall be + * {@code null}. For correctness, this will put the cache lazily and only update the existing last + * caches of measurements. + * + * @param database the device's database, without "root" + * @param deviceId {@link IDeviceID} + * @param measurements the fetched measurements + * @param timeValuePairs the {@link TimeValuePair}s with indexes corresponding to the measurements + * @param invalidateNull when true invalidate cache entries where timeValuePairs[i] == null; when + * false ignore cache entries where timeValuePairs[i] == null + */ + public void updateLastCacheIfExists( + final String database, + final IDeviceID deviceId, + final String[] measurements, + final TimeValuePair[] timeValuePairs, + boolean invalidateNull) { + dualKeyCache.update( + new TableId(database, deviceId.getTableName()), + deviceId, + null, + entry -> entry.tryUpdateLastCache(measurements, timeValuePairs, invalidateNull), + false); + } + + /** + * Update the last cache in writing or the second push of last cache query. If a measurement is + * with all {@code null}s or is a tag/attribute column, its {@link TimeValuePair}[] shall be + * {@code null}. For correctness, this will put the cache lazily and only update the existing last + * caches of measurements. + * + * @param database the device's database, without "root" + * @param deviceId {@link IDeviceID} + * @param measurements the fetched measurements + * @param timeValuePairs the {@link TimeValuePair}s with indexes corresponding to the measurements + */ + public void updateLastCacheIfExists( + final String database, + final IDeviceID deviceId, + final String[] measurements, + final TimeValuePair[] timeValuePairs) { + updateLastCacheIfExists(database, deviceId, measurements, timeValuePairs, false); + } + + /** + * Get the last {@link TimeValuePair} of a measurement, the measurement shall never be "time". + * + * @param database the device's database, without "root", {@code null} for tree model + * @param deviceId {@link IDeviceID} + * @param measurement the measurement to get + * @return {@code null} iff cache miss, {@link DeviceLastCache#EMPTY_TIME_VALUE_PAIR} iff cache + * hit but result is {@code null}, and the result value otherwise. + */ + public TimeValuePair getLastEntry( + final @Nullable String database, final IDeviceID deviceId, final String measurement) { + final TableDeviceCacheEntry entry = + dualKeyCache.get(new TableId(database, deviceId.getTableName()), deviceId); + return Objects.nonNull(entry) ? entry.getTimeValuePair(measurement) : null; + } + + /** + * Get the last {@link TimeValuePair}s of given measurements, the measurements shall never be + * "time". If you want to get the last of "time", use "" to represent. + * + * @param database the device's database, without "root", {@code null} for tree model + * @param deviceId {@link IDeviceID} + * @param measurements the measurements to get + * @return {@code null} iff cache miss, {@link DeviceLastCache#EMPTY_TIME_VALUE_PAIR} iff cache + * hit but result is {@code null}, and the result value otherwise. + */ + public TimeValuePair[] getLastEntries( + final @Nullable String database, final IDeviceID deviceId, final String[] measurements) { + final TableDeviceCacheEntry entry = + dualKeyCache.get(new TableId(database, deviceId.getTableName()), deviceId); + return Objects.nonNull(entry) + ? Arrays.stream(measurements).map(entry::getTimeValuePair).toArray(TimeValuePair[]::new) + : null; + } + + /** + * Get the last value of measurements last by a target measurement. If the caller wants to last by + * time or get the time last by another source measurement, the measurement shall be "" to + * indicate the time column. + * + * @param database the device's database, without "root" + * @param deviceId {@link IDeviceID} + * @param sourceMeasurement the measurement to get + * @return {@code Optional.empty()} iff the last cache is miss at all; Or the optional of a pair, + * the {@link Pair#left} will be the source measurement's last time, (OptionalLong.empty() iff + * the source measurement is all {@code null}); {@link Pair#right} will be an {@link + * TsPrimitiveType} array, whose element will be {@code null} if cache miss, {@link + * DeviceLastCache#EMPTY_PRIMITIVE_TYPE} iff cache hit and the measurement is without any + * values when last by the source measurement's time, and the result value otherwise. + */ + public Optional> getLastRow( + final String database, + final IDeviceID deviceId, + final String sourceMeasurement, + final List targetMeasurements) { + final TableDeviceCacheEntry entry = + dualKeyCache.get(new TableId(database, deviceId.getTableName()), deviceId); + return Objects.nonNull(entry) + ? entry.getLastRow(sourceMeasurement, targetMeasurements) + : Optional.empty(); + } + + /** + * Invalidate the last cache of one table. + * + * @param database the device's database, without "root" + * @param table tableName + */ + public void invalidateLastCache(final String database, final String table) { + dualKeyCache.update( + new TableId(database, table), deviceId -> true, entry -> -entry.invalidateLastCache()); + } + + /** + * Invalidate the last cache of one device. + * + * @param database the device's database, without "root" + * @param deviceId IDeviceID + */ + public void invalidateLastCache(final String database, final IDeviceID deviceId) { + dualKeyCache.update( + new TableId(database, deviceId.getTableName()), + deviceId, + null, + entry -> -entry.invalidateLastCache(), + false); + } + + /////////////////////////////// Tree model /////////////////////////////// + + public void putDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { + final PartialPath devicePath = deviceSchemaInfo.getDevicePath(); + final IDeviceID deviceID = devicePath.getIDeviceID(); + final String previousDatabase = treeModelDatabasePool.putIfAbsent(database, database); + + dualKeyCache.update( + new TableId(null, deviceID.getTableName()), + deviceID, + new TableDeviceCacheEntry(), + entry -> + entry.setDeviceSchema( + Objects.nonNull(previousDatabase) ? previousDatabase : database, deviceSchemaInfo), + true); + } + + public IDeviceSchema getDeviceSchema(final String[] devicePath) { + return getDeviceSchema( + IDeviceID.Factory.DEFAULT_FACTORY.create( + StringArrayDeviceID.splitDeviceIdString(devicePath))); + } + + public IDeviceSchema getDeviceSchema(final IDeviceID deviceID) { + final TableDeviceCacheEntry entry = + dualKeyCache.get(new TableId(null, deviceID.getTableName()), deviceID); + return Objects.nonNull(entry) ? entry.getDeviceSchema() : null; + } + + void updateLastCache( + final String database, + final IDeviceID deviceID, + final String[] measurements, + final @Nullable TimeValuePair[] timeValuePairs, + final boolean isAligned, + final IMeasurementSchema[] measurementSchemas, + final boolean initOrInvalidate) { + final String previousDatabase = treeModelDatabasePool.putIfAbsent(database, database); + final String database2Use = Objects.nonNull(previousDatabase) ? previousDatabase : database; + + dualKeyCache.update( + new TableId(null, deviceID.getTableName()), + deviceID, + new TableDeviceCacheEntry(), + initOrInvalidate + ? entry -> + entry.setMeasurementSchema( + database2Use, isAligned, measurements, measurementSchemas) + + entry.initOrInvalidateLastCache( + deviceID.getTableName(), measurements, Objects.nonNull(timeValuePairs)) + : entry -> + entry.setMeasurementSchema( + database2Use, isAligned, measurements, measurementSchemas) + + entry.tryUpdateLastCache(measurements, timeValuePairs), + Objects.isNull(timeValuePairs)); + } + + public boolean getLastCache( + final Map>>> inputMap) { + return dualKeyCache.batchApply(inputMap, TableDeviceCacheEntry::updateInputMap); + } + + // WARNING: This is not guaranteed to affect table model's cache + void invalidateLastCache(final PartialPath devicePath, final String measurement) { + final ToIntFunction updateFunction = + PathPatternUtil.hasWildcard(measurement) + ? entry -> -entry.invalidateLastCache() + : entry -> -entry.invalidateLastCache(measurement, false); + + if (!devicePath.hasWildcard()) { + final IDeviceID deviceID = devicePath.getIDeviceID(); + dualKeyCache.update( + new TableId(null, deviceID.getTableName()), deviceID, null, updateFunction, false); + } else { + // This may take quite a long time to perform, yet we assume that the "invalidateLastCache" is + // only called by deletions, which has a low frequency; and it has avoided that + // the un-related paths being cleared, like "root.*.b.c.**" affects + // "root.*.d.c.**", thereby lower the query performance. + dualKeyCache.update( + tableId -> { + try { + return devicePath.matchPrefixPath(new PartialPath(tableId.getTableName())); + } catch (final IllegalPathException e) { + logger.warn( + "Illegal tableID {} found in cache when invalidating by path {}, invalidate it anyway", + tableId.getTableName(), + devicePath); + return true; + } + }, + cachedDeviceID -> { + try { + return new PartialPath(cachedDeviceID).matchFullPath(devicePath); + } catch (final IllegalPathException e) { + logger.warn( + "Illegal deviceID {} found in cache when invalidating by path {}, invalidate it anyway", + cachedDeviceID, + devicePath); + return true; + } + }, + updateFunction); + } + } + + // WARNING: This is not guaranteed to affect table model's cache + void invalidateCache( + final @Nonnull PartialPath devicePath, final boolean isMultiLevelWildcardMeasurement) { + if (!devicePath.hasWildcard()) { + final IDeviceID deviceID = devicePath.getIDeviceID(); + dualKeyCache.invalidate(new TableId(null, deviceID.getTableName()), deviceID); + } else { + // This may take quite a long time to perform, yet we assume that the "invalidateLastCache" is + // only called by deletions, which has a low frequency; and it has avoided that + // the un-related paths being cleared, like "root.*.b.c.**" affects + // "root.*.d.c.**", thereby lower the query performance. + dualKeyCache.invalidate( + tableId -> { + try { + return devicePath.matchPrefixPath(new PartialPath(tableId.getTableName())); + } catch (final IllegalPathException e) { + logger.warn( + "Illegal tableID {} found in cache when invalidating by path {}, invalidate it anyway", + tableId.getTableName(), + devicePath); + return true; + } + }, + cachedDeviceID -> { + try { + return isMultiLevelWildcardMeasurement + ? devicePath.matchPrefixPath(new PartialPath(cachedDeviceID)) + : devicePath.matchFullPath(new PartialPath(cachedDeviceID)); + } catch (final IllegalPathException e) { + logger.warn( + "Illegal deviceID {} found in cache when invalidating by path {}, invalidate it anyway", + cachedDeviceID, + devicePath); + return true; + } + }); + } + } + + /////////////////////////////// Management /////////////////////////////// + + long getHitCount() { + return dualKeyCache.stats().hitCount(); + } + + long getRequestCount() { + return dualKeyCache.stats().requestCount(); + } + + long getMemoryUsage() { + return dualKeyCache.stats().memoryUsage(); + } + + long capacity() { + return dualKeyCache.stats().capacity(); + } + + long entriesCount() { + return dualKeyCache.stats().entriesCount(); + } + + void invalidateLastCache(final @Nonnull String database) { + readWriteLock.writeLock().lock(); + + try { + dualKeyCache.update( + tableId -> + Objects.isNull(tableId.getDatabase()) && tableId.getTableName().startsWith(database), + deviceID -> true, + entry -> -entry.invalidateLastCache()); + dualKeyCache.update( + tableId -> + Objects.isNull(tableId.getDatabase()) && database.startsWith(tableId.getTableName()), + deviceID -> deviceID.matchDatabaseName(database), + entry -> -entry.invalidateLastCache()); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + public void invalidate(final @Nonnull String database) { + readWriteLock.writeLock().lock(); + try { + dualKeyCache.invalidate( + tableId -> + Objects.isNull(tableId.getDatabase()) && tableId.getTableName().startsWith(database), + deviceID -> true); + dualKeyCache.invalidate( + tableId -> + Objects.isNull(tableId.getDatabase()) && database.startsWith(tableId.getTableName()), + deviceID -> deviceID.matchDatabaseName(database)); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + // The fuzzy filters are not considered because: + // 1. We can actually invalidate more cache entries than we need. + // 2. Constructing the filterOperators may require some time and complication + // 3. The fuzzy filters may contain attributes, which may not exist or be stale + public void invalidate( + final String database, final String tableName, final List patterns) { + readWriteLock.writeLock().lock(); + try { + final TableId firstKey = new TableId(database, tableName); + if (patterns.isEmpty()) { + dualKeyCache.invalidate(firstKey); + } else { + final List multiMatchList = + patterns.stream() + .filter( + idFilter -> { + if (!idFilter.hasWildcard()) { + final IDeviceID deviceId = + IDeviceID.Factory.DEFAULT_FACTORY.create( + Arrays.copyOfRange( + idFilter.getNodes(), 2, idFilter.getNodeLength())); + dualKeyCache.invalidate(firstKey, deviceId); + return false; + } + return true; + }) + .collect(Collectors.toList()); + + dualKeyCache.invalidate( + firstKey, + deviceId -> { + final String[] segments = (String[]) deviceId.getSegments(); + for (int i = 1; i < segments.length; ++i) { + for (final PartialPath path : multiMatchList) { + final int pathIndex = i + 2; + if (path.getNodes()[pathIndex].equals(segments[i]) + || path.getNodes()[pathIndex].equals(ONE_LEVEL_PATH_WILDCARD) + && ((ExtendedPartialPath) path).match(pathIndex, segments[i])) { + return true; + } + } + } + return false; + }); + } + } finally { + readWriteLock.writeLock().unlock(); + } + } + + public void invalidateLastCache() { + readWriteLock.writeLock().lock(); + try { + dualKeyCache.update(tableId -> true, deviceID -> true, entry -> -entry.invalidateLastCache()); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + public void invalidateTreeSchema() { + readWriteLock.writeLock().lock(); + try { + dualKeyCache.update( + tableId -> true, deviceID -> true, entry -> -entry.invalidateTreeSchema()); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + public void invalidateAll() { + readWriteLock.writeLock().lock(); + try { + dualKeyCache.invalidateAll(); + } finally { + readWriteLock.writeLock().unlock(); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/IDeviceSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/IDeviceSchema.java new file mode 100644 index 0000000000000..473429ac0a660 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/IDeviceSchema.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +public interface IDeviceSchema { + int estimateSize(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java index d97d5ac30f2c7..1b46e2c0faf65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java @@ -21,50 +21,47 @@ import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.ILastCacheContainer; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.LastCacheContainer; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import java.util.Map; public class SchemaCacheEntry implements IMeasurementSchemaInfo { - private final String storageGroup; + /** + * MeasurementSchema takes 75B + * + *

    + *
  • MeasurementSchema + *
      + *
    • Object header, 8B + *
    • String measurementId basic, 8 + 8 + 4 + 8 + 4 = 32B + *
    • type, encoding, compressor, 3 B + *
    • encodingConverter, 8 + 8 + 8 = 24B + *
    • props, 8B + *
    + *
+ */ + private static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(SchemaCacheEntry.class) + 75; private final IMeasurementSchema iMeasurementSchema; - private final Map tagMap; - private final boolean isAligned; - @SuppressWarnings("java:S3077") - private volatile ILastCacheContainer lastCacheContainer = null; + private final Map tagMap; public SchemaCacheEntry( - String storageGroup, - IMeasurementSchema iMeasurementSchema, - Map tagMap, - boolean isAligned) { - this.storageGroup = storageGroup.intern(); + final @Nonnull IMeasurementSchema iMeasurementSchema, + final @Nullable Map tagMap) { this.iMeasurementSchema = iMeasurementSchema; - this.isAligned = isAligned; this.tagMap = tagMap; } - public String getSchemaEntryId() { - return iMeasurementSchema.getMeasurementId(); - } - - public String getStorageGroup() { - return storageGroup; - } - - public IMeasurementSchema getIMeasurementSchema() { - return iMeasurementSchema; - } - @Override public Map getTagMap() { return tagMap; @@ -79,64 +76,15 @@ public TSDataType getTsDataType() { return iMeasurementSchema.getType(); } - public boolean isAligned() { - return isAligned; - } - - public ILastCacheContainer getLastCacheContainer() { - return lastCacheContainer; - } - - public int updateLastCache( - TimeValuePair timeValuePair, boolean highPriorityUpdate, Long latestFlushedTime) { - - if (lastCacheContainer == null) { - synchronized (this) { - if (lastCacheContainer == null) { - ILastCacheContainer tmp = new LastCacheContainer(); - int changeSize = tmp.estimateSize(); - changeSize += tmp.updateCachedLast(timeValuePair, highPriorityUpdate, latestFlushedTime); - lastCacheContainer = tmp; - return changeSize; - } - } - } - return lastCacheContainer.updateCachedLast( - timeValuePair, highPriorityUpdate, latestFlushedTime); - } - - /** - * Total basic 100B - * - *
    - *
  • SchemaCacheEntry Object header, 8B - *
  • isAligned, 1B - *
  • LastCacheContainer reference, 8B - *
  • MeasurementSchema - *
      - *
    • Reference, 8B - *
    • Object header, 8B - *
    • String measurementId basic, 8 + 8 + 4 + 8 + 4 = 32B - *
    • type, encoding, compressor, 3 B - *
    • encodingConverter, 8 + 8 + 8 = 24B - *
    • props, 8B - *
    - *
- */ - public static int estimateSize(SchemaCacheEntry schemaCacheEntry) { - // each char takes 2B in Java - int lastCacheContainerSize = - schemaCacheEntry.getLastCacheContainer() == null - ? 0 - : schemaCacheEntry.getLastCacheContainer().estimateSize(); - return 100 - + 2 * schemaCacheEntry.getIMeasurementSchema().getMeasurementId().length() - + lastCacheContainerSize; + public static int estimateSize(final SchemaCacheEntry schemaCacheEntry) { + return INSTANCE_SIZE + + 2 * schemaCacheEntry.getSchema().getMeasurementName().length() + + (int) RamUsageEstimator.sizeOfMap(schemaCacheEntry.getTagMap()); } @Override public String getName() { - return iMeasurementSchema.getMeasurementId(); + return iMeasurementSchema.getMeasurementName(); } @Override @@ -169,12 +117,4 @@ public String getAlias() { public boolean isLogicalView() { return this.iMeasurementSchema.isLogicalView(); } - - public int invalidateLastCache() { - if (this.lastCacheContainer == null || this.lastCacheContainer.getCachedLast() == null) { - return 0; - } - - return this.lastCacheContainer.invalidateLastCache(); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java new file mode 100644 index 0000000000000..a5f375d37bddd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +import javax.annotation.Nonnull; +import javax.annotation.concurrent.ThreadSafe; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; + +@ThreadSafe +public class TableDeviceCacheEntry { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TableDeviceCacheEntry.class) + + 2 * RamUsageEstimator.shallowSizeOfInstance(AtomicReference.class); + + // the cached attributeMap may not be the latest, but there won't be any correctness problems + // because when missing getting the key-value from this attributeMap, caller will try to get or + // create from remote + // there may exist key is not null, but value is null in this map, which means that the key's + // corresponding value is null, doesn't mean that the key doesn't exist + private final AtomicReference deviceSchema = new AtomicReference<>(); + private final AtomicReference lastCache = new AtomicReference<>(); + + /////////////////////////////// Attribute /////////////////////////////// + + int invalidateAttribute() { + final AtomicInteger size = new AtomicInteger(0); + deviceSchema.updateAndGet( + schema -> { + if (schema instanceof TableAttributeSchema) { + size.set(schema.estimateSize()); + return null; + } + return schema; + }); + return size.get(); + } + + int invalidateAttributeColumn(final String attribute) { + final AtomicInteger size = new AtomicInteger(0); + deviceSchema.updateAndGet( + schema -> { + if (schema instanceof TableAttributeSchema) { + size.set(((TableAttributeSchema) schema).removeAttribute(attribute)); + return schema; + } + return schema; + }); + return size.get(); + } + + /////////////////////////////// Tree model /////////////////////////////// + + int setDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { + // Safe here because tree schema is invalidated by the whole entry + if (deviceSchemaInfo.getTemplateId() == NON_TEMPLATE) { + final int result = + (deviceSchema.compareAndSet( + null, new TreeDeviceNormalSchema(database, deviceSchemaInfo.isAligned())) + ? TreeDeviceNormalSchema.INSTANCE_SIZE + : 0); + return deviceSchema.get() instanceof TreeDeviceNormalSchema + ? result + + ((TreeDeviceNormalSchema) deviceSchema.get()) + .update(deviceSchemaInfo.getMeasurementSchemaInfoList()) + : 0; + } else { + return deviceSchema.compareAndSet( + null, new TreeDeviceTemplateSchema(database, deviceSchemaInfo.getTemplateId())) + ? TreeDeviceTemplateSchema.INSTANCE_SIZE + : 0; + } + } + + int setMeasurementSchema( + final String database, + final boolean isAligned, + final String[] measurements, + final IMeasurementSchema[] schemas) { + if (schemas == null) { + return 0; + } + // Safe here because tree schema is invalidated by the whole entry + final int result = + (deviceSchema.compareAndSet(null, new TreeDeviceNormalSchema(database, isAligned)) + ? TreeDeviceNormalSchema.INSTANCE_SIZE + : 0); + return deviceSchema.get() instanceof TreeDeviceNormalSchema + ? result + ((TreeDeviceNormalSchema) deviceSchema.get()).update(measurements, schemas) + : 0; + } + + IDeviceSchema getDeviceSchema() { + return deviceSchema.get(); + } + + int invalidateTreeSchema() { + final AtomicInteger size = new AtomicInteger(0); + deviceSchema.updateAndGet( + schema -> { + if (schema instanceof TreeDeviceNormalSchema + || schema instanceof TreeDeviceTemplateSchema) { + size.set(schema.estimateSize()); + return null; + } + return schema; + }); + return size.get(); + } + + /////////////////////////////// Last Cache /////////////////////////////// + + int initOrInvalidateLastCache( + final String tableName, final String[] measurements, final boolean isInvalidate) { + int result = + lastCache.compareAndSet(null, new DeviceLastCache()) ? DeviceLastCache.INSTANCE_SIZE : 0; + final DeviceLastCache cache = lastCache.get(); + result += + Objects.nonNull(cache) ? cache.initOrInvalidate(tableName, measurements, isInvalidate) : 0; + return Objects.nonNull(lastCache.get()) ? result : 0; + } + + int tryUpdateLastCache( + final String[] measurements, final TimeValuePair[] timeValuePairs, boolean invalidateNull) { + final DeviceLastCache cache = lastCache.get(); + final int result = + Objects.nonNull(cache) ? cache.tryUpdate(measurements, timeValuePairs, invalidateNull) : 0; + return Objects.nonNull(lastCache.get()) ? result : 0; + } + + int tryUpdateLastCache(final String[] measurements, final TimeValuePair[] timeValuePairs) { + return tryUpdateLastCache(measurements, timeValuePairs, false); + } + + int invalidateLastCache(final String measurement, final boolean isTableModel) { + final DeviceLastCache cache = lastCache.get(); + final int result = Objects.nonNull(cache) ? cache.invalidate(measurement, isTableModel) : 0; + return Objects.nonNull(lastCache.get()) ? result : 0; + } + + TimeValuePair getTimeValuePair(final String measurement) { + final DeviceLastCache cache = lastCache.get(); + return Objects.nonNull(cache) ? cache.getTimeValuePair(measurement) : null; + } + + boolean updateInputMap(final @Nonnull Map> updateMap) { + // Shall only call this for original table device + for (final String measurement : updateMap.keySet()) { + final TimeValuePair result = getTimeValuePair(measurement); + if (result == null) { + return false; + } + updateMap.get(measurement).setRight(result); + } + return true; + } + + // Shall pass in "" if last by time + Optional> getLastRow( + final String sourceMeasurement, final List targetMeasurements) { + final DeviceLastCache cache = lastCache.get(); + return Objects.nonNull(cache) + ? cache.getLastRow(sourceMeasurement, targetMeasurements) + : Optional.empty(); + } + + int invalidateLastCache() { + final AtomicInteger size = new AtomicInteger(0); + lastCache.updateAndGet( + cacheEntry -> { + if (Objects.nonNull(cacheEntry)) { + size.set(cacheEntry.estimateSize()); + } + return null; + }); + return size.get(); + } + + /////////////////////////////// Management /////////////////////////////// + + int estimateSize() { + final IDeviceSchema schema = deviceSchema.get(); + final DeviceLastCache cache = lastCache.get(); + return (int) + (INSTANCE_SIZE + + (Objects.nonNull(schema) ? schema.estimateSize() : 0) + + (Objects.nonNull(cache) ? cache.estimateSize() : 0)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceNormalSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceNormalSchema.java new file mode 100644 index 0000000000000..d944f5f483933 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceNormalSchema.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; + +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class TreeDeviceNormalSchema implements IDeviceSchema { + + static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(TreeDeviceTemplateSchema.class) + + (int) RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); + private final String database; + private final boolean isAligned; + + private final ConcurrentMap measurementMap = new ConcurrentHashMap<>(); + + public TreeDeviceNormalSchema(final String database, final boolean isAligned) { + this.database = database; + this.isAligned = isAligned; + } + + public String getDatabase() { + return database; + } + + public boolean isAligned() { + return isAligned; + } + + public SchemaCacheEntry getSchemaCacheEntry(final String measurement) { + return measurementMap.get(measurement); + } + + public int update(final String[] measurements, final IMeasurementSchema[] schemas) { + int diff = 0; + if (schemas == null) { + return diff; + } + + final int length = measurements.length; + + for (int i = 0; i < length; ++i) { + // Skip this to avoid instance creation/gc for writing performance + if (measurements[i] == null || measurementMap.containsKey(measurements[i])) { + continue; + } + diff += putEntry(measurements[i], schemas[i], null); + } + return diff; + } + + public int update(final List schemaInfoList) { + return schemaInfoList.stream() + .mapToInt( + schemaInfo -> + putEntry(schemaInfo.getName(), schemaInfo.getSchema(), schemaInfo.getTagMap())) + .reduce(0, Integer::sum); + } + + private int putEntry( + final String measurement, final IMeasurementSchema schema, final Map tagMap) { + final SchemaCacheEntry putEntry = new SchemaCacheEntry(schema, tagMap); + final SchemaCacheEntry cachedEntry = measurementMap.put(measurement, putEntry); + return Objects.isNull(cachedEntry) + ? (int) (RamUsageEstimator.sizeOf(measurement) + SchemaCacheEntry.estimateSize(putEntry)) + : SchemaCacheEntry.estimateSize(putEntry) - SchemaCacheEntry.estimateSize(cachedEntry); + } + + @Override + public int estimateSize() { + // Do not need to calculate database because it is interned + return INSTANCE_SIZE + + measurementMap.entrySet().stream() + .mapToInt( + entry -> + Math.toIntExact( + RamUsageEstimator.sizeOf(entry.getKey()) + + SchemaCacheEntry.estimateSize(entry.getValue()))) + .reduce(0, Integer::sum); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceTemplateSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceTemplateSchema.java new file mode 100644 index 0000000000000..583919dd5c12f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceTemplateSchema.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +import org.apache.tsfile.utils.RamUsageEstimator; + +public class TreeDeviceTemplateSchema implements IDeviceSchema { + static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(TreeDeviceTemplateSchema.class); + + private final String database; + private final int templateId; + + TreeDeviceTemplateSchema(final String database, final int templateId) { + this.database = database; + this.templateId = templateId; + } + + public int getTemplateId() { + return templateId; + } + + public String getDatabase() { + return database; + } + + @Override + public int estimateSize() { + // Do not need to calculate database because it is interned + return INSTANCE_SIZE; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCache.java index 397ca18e89fee..1a2d788c9f792 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCache.java @@ -19,14 +19,12 @@ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.TestOnly; - import javax.annotation.concurrent.GuardedBy; -import java.util.List; import java.util.Map; -import java.util.function.Function; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import java.util.function.ToIntFunction; /** * This interfaces defines the behaviour of a dual key cache. A dual key cache supports manage cache @@ -39,33 +37,51 @@ public interface IDualKeyCache { /** Get the cache value with given first key and second key. */ - V get(FK firstKey, SK secondKey); + V get(final FK firstKey, final SK secondKey); - boolean batchGet(final Map> inputMap, final Function mappingFunction); + boolean batchApply( + final Map> inputMap, final BiFunction mappingFunction); /** - * Traverse target cache values via given first key and second keys provided in computation and - * execute the defined computation logic. The computation is read only. + * Update the existing value. The updater shall return the difference caused by the update, + * because we do not want to call "valueSizeComputer" twice, which may include abundant useless + * calculations. + * + *

Warning: This method is without any locks for performance concerns. The caller shall ensure + * the concurrency safety for the value update. + * + * @param createIfNotExists put the value to cache iff it does not exist, */ - void compute(IDualKeyCacheComputation computation); + void update( + final FK firstKey, + final SK secondKey, + final V value, + final ToIntFunction updater, + final boolean createIfNotExists); /** - * Traverse target cache values via given first key and second keys provided in computation and - * execute the defined computation logic. Value can be updated in this computation. + * Update all the existing value with {@link SK} and a the {@link SK}s matching the given + * predicate. The updater shall return the difference caused by the update, because we do not want + * to call "valueSizeComputer" twice, which may include abundant useless calculations. + * + *

Warning: This method is without any locks for performance concerns. The caller shall ensure + * the concurrency safety for the value update. */ - void update(IDualKeyCacheUpdating updating); - - /** put the cache value into cache */ - void put(FK firstKey, SK secondKey, V value); + void update( + final FK firstKey, final Predicate secondKeyChecker, final ToIntFunction updater); /** - * Invalidate last cache in datanode schema cache. Do not invalidate time series cache. + * Update all the existing value with {@link SK} and a the {@link SK}s matching the given + * predicate. The updater shall return the difference caused by the update, because we do not want + * to call "valueSizeComputer" twice, which may include abundant useless calculations. * - * @param partialPathList + *

Warning: This method is without any locks for performance concerns. The caller shall ensure + * the concurrency safety for the value update. */ - void invalidateLastCache(PartialPath partialPath); - - void invalidateDataRegionLastCache(String database); + void update( + final Predicate firstKeyChecker, + final Predicate secondKeyChecker, + final ToIntFunction updater); /** * Invalidate all cache values in the cache and clear related cache keys. The cache status and @@ -74,29 +90,21 @@ public interface IDualKeyCache { @GuardedBy("DataNodeSchemaCache#writeLock") void invalidateAll(); - /** - * Invalidate cache values in the cache and clear related cache keys. The cache status and - * statistics won't be clear and they can still be accessed via cache.stats(). - */ - @GuardedBy("DataNodeSchemaCache#writeLock") - void invalidate(String database); + /** Return all the current cache status and statistics. */ + IDualKeyCacheStats stats(); - /** - * Invalidate cache values in the cache and clear related cache keys. The cache status and - * statistics won't be clear and they can still be accessed via cache.stats(). - */ + /** remove all entries for firstKey */ @GuardedBy("DataNodeSchemaCache#writeLock") - void invalidate(List partialPathList); + void invalidate(final FK firstKey); - /** - * Clean up all data and info of this cache, including cache keys, cache values and cache stats. - */ + /** remove matched entry */ @GuardedBy("DataNodeSchemaCache#writeLock") - void cleanUp(); + void invalidate(final FK firstKey, final SK secondKey); - /** Return all the current cache status and statistics. */ - IDualKeyCacheStats stats(); + @GuardedBy("DataNodeSchemaCache#writeLock") + void invalidate(final FK firstKey, final Predicate secondKeyChecker); - @TestOnly - void evictOneEntry(); + /** remove all entries matching the firstKey and the secondKey */ + @GuardedBy("DataNodeSchemaCache#writeLock") + void invalidate(final Predicate firstKeyChecker, final Predicate secondKeyChecker); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java index 9ed842cc602d5..9247e20a9c48a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java @@ -39,4 +39,10 @@ public interface IDualKeyCacheStats { /** Return current memory usage of dual key cache. */ long memoryUsage(); + + /** Return capacity of dual key cache. */ + long capacity(); + + /** Return entries num of dual key cache */ + long entriesCount(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheEntryGroupImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheEntryGroupImpl.java index b15f6d153fca0..54c53a15d5038 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheEntryGroupImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheEntryGroupImpl.java @@ -19,21 +19,36 @@ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl; +import org.apache.tsfile.utils.RamUsageEstimator; + import java.util.Iterator; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; +import java.util.function.Function; public class CacheEntryGroupImpl> implements ICacheEntryGroup { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(CacheEntryGroupImpl.class) + + RamUsageEstimator.shallowSizeOfInstance(AtomicLong.class) + + RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class) + // Calculate the outer entry of the "firstKeyMap" here + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY; + private final FK firstKey; private final Map cacheEntryMap = new ConcurrentHashMap<>(); + private final ICacheSizeComputer sizeComputer; + private final AtomicLong memory; - CacheEntryGroupImpl(FK firstKey) { + CacheEntryGroupImpl(final FK firstKey, final ICacheSizeComputer sizeComputer) { this.firstKey = firstKey; + this.sizeComputer = sizeComputer; + this.memory = new AtomicLong(INSTANCE_SIZE + sizeComputer.computeFirstKeySize(firstKey)); } @Override @@ -42,7 +57,7 @@ public FK getFirstKey() { } @Override - public T getCacheEntry(SK secondKey) { + public T getCacheEntry(final SK secondKey) { return secondKey == null ? null : cacheEntryMap.get(secondKey); } @@ -52,13 +67,23 @@ public Iterator> getAllCacheEntries() { } @Override - public T computeCacheEntry(SK secondKey, BiFunction computation) { - return cacheEntryMap.compute(secondKey, computation); + public T computeCacheEntry( + final SK secondKey, final Function> computation) { + return cacheEntryMap.compute(secondKey, computation.apply(memory)); } @Override - public T removeCacheEntry(SK secondKey) { - return cacheEntryMap.remove(secondKey); + public long removeCacheEntry(final SK secondKey) { + final T result = cacheEntryMap.remove(secondKey); + if (Objects.nonNull(result)) { + final long delta = + sizeComputer.computeSecondKeySize(result.getSecondKey()) + + sizeComputer.computeValueSize(result.getValue()) + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY; + memory.addAndGet(-delta); + return delta; + } + return 0; } @Override @@ -67,10 +92,24 @@ public boolean isEmpty() { } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CacheEntryGroupImpl that = (CacheEntryGroupImpl) o; + public long getMemory() { + return memory.get(); + } + + @Override + public int getEntriesCount() { + return cacheEntryMap.size(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final CacheEntryGroupImpl that = (CacheEntryGroupImpl) o; return Objects.equals(firstKey, that.firstKey); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java index 817092d3eb909..0628d35db164e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheStats; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; class CacheStats implements IDualKeyCacheStats { @@ -30,25 +31,23 @@ class CacheStats implements IDualKeyCacheStats { private final long memoryThreshold; - private final AtomicLong memoryUsage = new AtomicLong(0); + private final Supplier memoryComputation; + private final Supplier entriesComputation; private final AtomicLong requestCount = new AtomicLong(0); private final AtomicLong hitCount = new AtomicLong(0); - CacheStats(long memoryCapacity) { + CacheStats( + long memoryCapacity, + final Supplier memoryComputation, + final Supplier entriesComputation) { this.memoryThreshold = (long) (memoryCapacity * MEMORY_THRESHOLD_RATIO); + this.memoryComputation = memoryComputation; + this.entriesComputation = entriesComputation; } - void increaseMemoryUsage(int size) { - memoryUsage.getAndAdd(size); - } - - void decreaseMemoryUsage(int size) { - memoryUsage.getAndAdd(-size); - } - - boolean isExceedMemoryCapacity() { - return memoryUsage.get() > memoryThreshold; + long getExceedMemory() { + return memoryUsage() - memoryThreshold; } void recordHit(int num) { @@ -93,16 +92,16 @@ public double hitRate() { @Override public long memoryUsage() { - return memoryUsage.get(); + return memoryComputation.get(); } - void reset() { - resetMemoryUsage(); - hitCount.set(0); - requestCount.set(0); + @Override + public long capacity() { + return memoryThreshold; } - void resetMemoryUsage() { - memoryUsage.set(0); + @Override + public long entriesCount() { + return entriesComputation.get(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java index 5a4024bb31d28..841444f42793f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java @@ -19,25 +19,23 @@ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathPatternUtil; -import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.SchemaCacheEntry; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheComputation; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheStats; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheUpdating; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.DataNodeLastCacheManager; + +import org.apache.tsfile.utils.RamUsageEstimator; + +import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; -import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.ToIntFunction; class DualKeyCacheImpl> implements IDualKeyCache { @@ -52,22 +50,22 @@ class DualKeyCacheImpl> private final CacheStats cacheStats; DualKeyCacheImpl( - ICacheEntryManager cacheEntryManager, - ICacheSizeComputer sizeComputer, - long memoryCapacity) { + final ICacheEntryManager cacheEntryManager, + final ICacheSizeComputer sizeComputer, + final long memoryCapacity) { this.cacheEntryManager = cacheEntryManager; this.sizeComputer = sizeComputer; - this.cacheStats = new CacheStats(memoryCapacity); + this.cacheStats = new CacheStats(memoryCapacity, this::getMemory, this::getEntriesCount); } @Override - public V get(FK firstKey, SK secondKey) { - ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); + public V get(final FK firstKey, final SK secondKey) { + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); if (cacheEntryGroup == null) { cacheStats.recordMiss(1); return null; } else { - T cacheEntry = cacheEntryGroup.getCacheEntry(secondKey); + final T cacheEntry = cacheEntryGroup.getCacheEntry(secondKey); if (cacheEntry == null) { cacheStats.recordMiss(1); return null; @@ -80,8 +78,8 @@ public V get(FK firstKey, SK secondKey) { } @Override - public boolean batchGet( - final Map> inputMap, final Function mappingFunction) { + public boolean batchApply( + final Map> inputMap, final BiFunction mappingFunction) { for (final Map.Entry> fkMapEntry : inputMap.entrySet()) { final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(fkMapEntry.getKey()); if (cacheEntryGroup == null) { @@ -92,317 +90,258 @@ public boolean batchGet( if (cacheEntry == null) { return false; } - skrEntry.setValue(mappingFunction.apply(cacheEntry.getValue())); + if (!mappingFunction.apply(cacheEntry.getValue(), skrEntry.getValue())) { + return false; + } } } return true; } @Override - public void compute(IDualKeyCacheComputation computation) { - FK firstKey = computation.getFirstKey(); + public void update( + final FK firstKey, + final @Nonnull SK secondKey, + final V value, + final ToIntFunction updater, + final boolean createIfNotExists) { + ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); - SK[] secondKeyList = computation.getSecondKeyList(); - if (cacheEntryGroup == null) { - for (int i = 0; i < secondKeyList.length; i++) { - computation.computeValue(i, null); - } - cacheStats.recordMiss(secondKeyList.length); - } else { - T cacheEntry; - int hitCount = 0; - for (int i = 0; i < secondKeyList.length; i++) { - cacheEntry = cacheEntryGroup.getCacheEntry(secondKeyList[i]); - if (cacheEntry == null) { - computation.computeValue(i, null); - } else { - computation.computeValue(i, cacheEntry.getValue()); - cacheEntryManager.access(cacheEntry); - hitCount++; - } + if (Objects.isNull(cacheEntryGroup)) { + if (createIfNotExists) { + cacheEntryGroup = new CacheEntryGroupImpl<>(firstKey, sizeComputer); + firstKeyMap.put(firstKey, cacheEntryGroup); + } else { + return; } - cacheStats.recordHit(hitCount); - cacheStats.recordMiss(secondKeyList.length - hitCount); } - } - @Override - public void update(IDualKeyCacheUpdating updating) { - FK firstKey = updating.getFirstKey(); - ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); - SK[] secondKeyList = updating.getSecondKeyList(); - if (cacheEntryGroup == null) { - for (int i = 0; i < secondKeyList.length; i++) { - updating.updateValue(i, null); - } - cacheStats.recordMiss(secondKeyList.length); - } else { - T cacheEntry; - int hitCount = 0; - for (int i = 0; i < secondKeyList.length; i++) { - cacheEntry = cacheEntryGroup.getCacheEntry(secondKeyList[i]); - if (cacheEntry == null) { - updating.updateValue(i, null); - } else { - int changeSize = 0; - synchronized (cacheEntry) { - if (cacheEntry.getBelongedGroup() != null) { - // Only update the value when the cache entry is not evicted. - // If the cache entry is evicted, getBelongedGroup is null. - // Synchronized is to guarantee the cache entry is not evicted during the update. - changeSize = updating.updateValue(i, cacheEntry.getValue()); - cacheEntryManager.access(cacheEntry); - if (changeSize != 0) { - cacheStats.increaseMemoryUsage(changeSize); + final ICacheEntryGroup finalCacheEntryGroup = cacheEntryGroup; + cacheEntryGroup.computeCacheEntry( + secondKey, + memory -> + (sk, cacheEntry) -> { + if (Objects.isNull(cacheEntry)) { + if (!createIfNotExists) { + return null; + } + cacheEntry = + cacheEntryManager.createCacheEntry(secondKey, value, finalCacheEntryGroup); + cacheEntryManager.put(cacheEntry); + memory.getAndAdd( + sizeComputer.computeSecondKeySize(sk) + + sizeComputer.computeValueSize(cacheEntry.getValue()) + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY); } - } - } - if (changeSize != 0 && cacheStats.isExceedMemoryCapacity()) { - executeCacheEviction(changeSize); - } - hitCount++; - } - } - cacheStats.recordHit(hitCount); - cacheStats.recordMiss(secondKeyList.length - hitCount); - } + memory.getAndAdd(updater.applyAsInt(cacheEntry.getValue())); + return cacheEntry; + }); + + mayEvict(); } @Override - public void put(FK firstKey, SK secondKey, V value) { - int usedMemorySize = putToCache(firstKey, secondKey, value); - cacheStats.increaseMemoryUsage(usedMemorySize); - if (cacheStats.isExceedMemoryCapacity()) { - executeCacheEviction(usedMemorySize); + public void update( + final FK firstKey, final Predicate secondKeyChecker, final ToIntFunction updater) { + final ICacheEntryGroup entryGroup = firstKeyMap.get(firstKey); + if (Objects.nonNull(entryGroup)) { + entryGroup + .getAllCacheEntries() + .forEachRemaining( + entry -> { + if (!secondKeyChecker.test(entry.getKey())) { + return; + } + entryGroup.computeCacheEntry( + entry.getKey(), + memory -> + (secondKey, cacheEntry) -> { + if (Objects.nonNull(cacheEntry)) { + memory.getAndAdd(updater.applyAsInt(cacheEntry.getValue())); + } + return cacheEntry; + }); + }); } + mayEvict(); } - private int putToCache(FK firstKey, SK secondKey, V value) { - AtomicInteger usedMemorySize = new AtomicInteger(0); - firstKeyMap.compute( - firstKey, - (k, cacheEntryGroup) -> { - if (cacheEntryGroup == null) { - cacheEntryGroup = new CacheEntryGroupImpl<>(firstKey); - usedMemorySize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); - } - ICacheEntryGroup finalCacheEntryGroup = cacheEntryGroup; - cacheEntryGroup.computeCacheEntry( - secondKey, - (sk, cacheEntry) -> { - if (cacheEntry == null) { - cacheEntry = - cacheEntryManager.createCacheEntry(secondKey, value, finalCacheEntryGroup); - cacheEntryManager.put(cacheEntry); - usedMemorySize.getAndAdd(sizeComputer.computeSecondKeySize(sk)); - } else { - V existingValue = cacheEntry.getValue(); - if (existingValue != value && !existingValue.equals(value)) { - cacheEntry.replaceValue(value); - usedMemorySize.getAndAdd(-sizeComputer.computeValueSize(existingValue)); + @Override + public void update( + final Predicate firstKeyChecker, + final Predicate secondKeyChecker, + final ToIntFunction updater) { + for (final FK firstKey : firstKeyMap.getAllKeys()) { + if (!firstKeyChecker.test(firstKey)) { + continue; + } + final ICacheEntryGroup entryGroup = firstKeyMap.get(firstKey); + if (Objects.nonNull(entryGroup)) { + entryGroup + .getAllCacheEntries() + .forEachRemaining( + entry -> { + if (!secondKeyChecker.test(entry.getKey())) { + return; } - // update the cache status - cacheEntryManager.access(cacheEntry); - } - usedMemorySize.getAndAdd(sizeComputer.computeValueSize(value)); - return cacheEntry; - }); - return cacheEntryGroup; - }); - return usedMemorySize.get(); + entryGroup.computeCacheEntry( + entry.getKey(), + memory -> + (secondKey, cacheEntry) -> { + memory.getAndAdd(updater.applyAsInt(cacheEntry.getValue())); + return cacheEntry; + }); + }); + } + mayEvict(); + } } - /** - * Each thread putting new cache value only needs to evict cache values, total memory equals that - * the new cache value occupied. - */ - private void executeCacheEviction(int targetSize) { - int evictedSize; - while (targetSize > 0 && cacheStats.memoryUsage() > 0) { - evictedSize = evictOneCacheEntry(); - cacheStats.decreaseMemoryUsage(evictedSize); - targetSize -= evictedSize; + private void mayEvict() { + long exceedMemory; + while ((exceedMemory = cacheStats.getExceedMemory()) > 0) { + // Not compute each time to save time when FK is too many + // The hard-coded size is 100 + do { + exceedMemory -= evictOneCacheEntry(); + } while (exceedMemory > 0 && firstKeyMap.size() > 100); } } - private int evictOneCacheEntry() { - - ICacheEntry evictCacheEntry = cacheEntryManager.evict(); + // The returned delta may have some error, but it's OK + // Because the delta is only for loop round estimation + private long evictOneCacheEntry() { + final ICacheEntry evictCacheEntry = cacheEntryManager.evict(); if (evictCacheEntry == null) { return 0; } - synchronized (evictCacheEntry) { - AtomicInteger evictedSize = new AtomicInteger(0); - evictedSize.getAndAdd(sizeComputer.computeValueSize(evictCacheEntry.getValue())); - - ICacheEntryGroup belongedGroup = evictCacheEntry.getBelongedGroup(); - evictCacheEntry.setBelongedGroup(null); - belongedGroup.removeCacheEntry(evictCacheEntry.getSecondKey()); - evictedSize.getAndAdd(sizeComputer.computeSecondKeySize(evictCacheEntry.getSecondKey())); - - if (belongedGroup.isEmpty()) { - firstKeyMap.compute( - belongedGroup.getFirstKey(), - (firstKey, cacheEntryGroup) -> { - if (cacheEntryGroup == null) { - // has been removed by other threads - return null; - } - if (cacheEntryGroup.isEmpty()) { - evictedSize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); - return null; - } - // some other thread has put value to it - return cacheEntryGroup; - }); + final ICacheEntryGroup belongedGroup = evictCacheEntry.getBelongedGroup(); + evictCacheEntry.setBelongedGroup(null); + + long memory = belongedGroup.removeCacheEntry(evictCacheEntry.getSecondKey()); + + final ICacheEntryGroup cacheEntryGroup = + firstKeyMap.get(belongedGroup.getFirstKey()); + if (Objects.nonNull(cacheEntryGroup) && cacheEntryGroup.isEmpty()) { + if (Objects.nonNull(firstKeyMap.remove(belongedGroup.getFirstKey()))) { + memory += + sizeComputer.computeFirstKeySize(belongedGroup.getFirstKey()) + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY; } - return evictedSize.get(); } + return memory; } @Override - public void invalidateLastCache(PartialPath path) { - String measurement = path.getMeasurement(); - PartialPath devicePath = path.getDevicePath(); - Function deviceFilter = null; - Function measurementFilter = null; - - if (PathPatternUtil.hasWildcard(devicePath.getFullPath())) { - deviceFilter = d -> devicePath.matchFullPath((PartialPath) d); - } - if (PathPatternUtil.isMultiLevelMatchWildcard(measurement)) { - measurementFilter = m -> true; - } - if (deviceFilter == null) { - deviceFilter = d -> d.equals(devicePath); - } - - if (measurementFilter == null) { - measurementFilter = m -> PathPatternUtil.isNodeMatch(measurement, m.toString()); - } + public void invalidateAll() { + firstKeyMap.clear(); + cacheEntryManager.cleanUp(); + } - for (FK device : firstKeyMap.getAllKeys()) { - if (Boolean.TRUE.equals(deviceFilter.apply(device))) { - ICacheEntryGroup entryGroup = firstKeyMap.get(device); - for (Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { - Map.Entry entry = it.next(); - if (Boolean.TRUE.equals(measurementFilter.apply(entry.getKey()))) { - T cacheEntry = entry.getValue(); - synchronized (cacheEntry) { - SchemaCacheEntry schemaCacheEntry = (SchemaCacheEntry) cacheEntry.getValue(); - cacheStats.decreaseMemoryUsage( - DataNodeLastCacheManager.invalidateLastCache(schemaCacheEntry)); - } - } - } - } - } + @Override + public IDualKeyCacheStats stats() { + return cacheStats; } @Override - public void invalidateDataRegionLastCache(String database) { - for (FK device : firstKeyMap.getAllKeys()) { - if (device.toString().startsWith(database)) { - ICacheEntryGroup entryGroup = firstKeyMap.get(device); - for (Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { - Map.Entry entry = it.next(); - T cacheEntry = entry.getValue(); - synchronized (cacheEntry) { - SchemaCacheEntry schemaCacheEntry = (SchemaCacheEntry) cacheEntry.getValue(); - cacheStats.decreaseMemoryUsage( - DataNodeLastCacheManager.invalidateLastCache(schemaCacheEntry)); - } - } + public void invalidate(final FK firstKey) { + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.remove(firstKey); + if (cacheEntryGroup != null) { + for (final Iterator> it = cacheEntryGroup.getAllCacheEntries(); + it.hasNext(); ) { + cacheEntryManager.invalidate(it.next().getValue()); } } } @Override - public void invalidateAll() { - executeInvalidateAll(); + public void invalidate(final FK firstKey, final SK secondKey) { + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); + if (Objects.isNull(cacheEntryGroup)) { + return; + } + + final T entry = cacheEntryGroup.getCacheEntry(secondKey); + if (Objects.nonNull(entry) && cacheEntryManager.invalidate(entry)) { + cacheEntryGroup.removeCacheEntry(entry.getSecondKey()); + } + + if (cacheEntryGroup.isEmpty()) { + firstKeyMap.remove(firstKey); + } } @Override - public void invalidate(String database) { - int estimateSize = 0; - for (FK device : firstKeyMap.getAllKeys()) { - if (device.toString().startsWith(database)) { - estimateSize += sizeComputer.computeFirstKeySize(device); - ICacheEntryGroup entryGroup = firstKeyMap.get(device); - for (Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { - Map.Entry entry = it.next(); - estimateSize += sizeComputer.computeSecondKeySize(entry.getKey()); - estimateSize += sizeComputer.computeValueSize(entry.getValue().getValue()); - cacheEntryManager.invalid(entry.getValue()); - } - firstKeyMap.remove(device); + public void invalidate(final FK firstKey, final Predicate secondKeyChecker) { + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); + if (Objects.isNull(cacheEntryGroup)) { + return; + } + for (final Iterator> it = cacheEntryGroup.getAllCacheEntries(); + it.hasNext(); ) { + final Map.Entry entry = it.next(); + if (secondKeyChecker.test(entry.getKey()) && cacheEntryManager.invalidate(entry.getValue())) { + cacheEntryGroup.removeCacheEntry(entry.getKey()); } } - cacheStats.decreaseMemoryUsage(estimateSize); + + if (cacheEntryGroup.isEmpty()) { + firstKeyMap.remove(firstKey); + } } @Override - public void invalidate(List partialPathList) { - int estimateSize = 0; - for (PartialPath path : partialPathList) { - String measurement = path.getMeasurement(); - Function deviceFilter; - Function measurementFilter; - if (PathPatternUtil.isMultiLevelMatchWildcard(measurement)) { - deviceFilter = d -> d.toString().startsWith(path.getDevicePath().getFullPath()); - measurementFilter = m -> true; - } else { - deviceFilter = d -> d.toString().equals(path.getDevicePath().getFullPath()); - measurementFilter = m -> PathPatternUtil.isNodeMatch(measurement, m.toString()); + public void invalidate( + final Predicate firstKeyChecker, final Predicate secondKeyChecker) { + for (final FK firstKey : firstKeyMap.getAllKeys()) { + if (!firstKeyChecker.test(firstKey)) { + continue; } - for (FK device : firstKeyMap.getAllKeys()) { - if (Boolean.TRUE.equals(deviceFilter.apply(device))) { - boolean allSKInvalid = true; - ICacheEntryGroup entryGroup = firstKeyMap.get(device); - for (Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { - Map.Entry entry = it.next(); - if (Boolean.TRUE.equals(measurementFilter.apply(entry.getKey()))) { - estimateSize += sizeComputer.computeSecondKeySize(entry.getKey()); - estimateSize += sizeComputer.computeValueSize(entry.getValue().getValue()); - cacheEntryManager.invalid(entry.getValue()); - it.remove(); - } else { - allSKInvalid = false; - } - } - if (allSKInvalid) { - firstKeyMap.remove(device); - estimateSize += sizeComputer.computeFirstKeySize(device); - } - } + + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); + if (Objects.isNull(cacheEntryGroup)) { + return; } - } - cacheStats.decreaseMemoryUsage(estimateSize); - } + for (final Iterator> it = cacheEntryGroup.getAllCacheEntries(); + it.hasNext(); ) { + final Map.Entry entry = it.next(); - private void executeInvalidateAll() { - firstKeyMap.clear(); - cacheEntryManager.cleanUp(); - cacheStats.resetMemoryUsage(); - } + if (secondKeyChecker.test(entry.getKey()) + && cacheEntryManager.invalidate(entry.getValue())) { + cacheEntryGroup.removeCacheEntry(entry.getKey()); + } + } - @Override - public void cleanUp() { - executeInvalidateAll(); - cacheStats.reset(); + if (cacheEntryGroup.isEmpty()) { + firstKeyMap.remove(firstKey); + } + } } - @Override - public IDualKeyCacheStats stats() { - return cacheStats; + private long getMemory() { + long memory = 0; + for (final Map> map : firstKeyMap.maps) { + if (Objects.nonNull(map)) { + for (final ICacheEntryGroup group : map.values()) { + memory += group.getMemory(); + } + } + } + return memory; } - @Override - @TestOnly - public void evictOneEntry() { - cacheStats.decreaseMemoryUsage(evictOneCacheEntry()); + private long getEntriesCount() { + long count = 0; + for (final Map> map : firstKeyMap.maps) { + if (Objects.nonNull(map)) { + for (final ICacheEntryGroup group : map.values()) { + count += group.getEntriesCount(); + } + } + } + return count; } /** @@ -412,29 +351,28 @@ public void evictOneEntry() { private static class SegmentedConcurrentHashMap { private static final int SLOT_NUM = 31; + private final Map[] maps = new ConcurrentHashMap[SLOT_NUM]; - V get(K key) { + V get(final K key) { return getBelongedMap(key).get(key); } - V remove(K key) { + V remove(final K key) { return getBelongedMap(key).remove(key); } - V compute(K key, BiFunction remappingFunction) { - return getBelongedMap(key).compute(key, remappingFunction); + V put(final K key, final V value) { + return getBelongedMap(key).put(key, value); } void clear() { synchronized (maps) { - for (int i = 0; i < SLOT_NUM; i++) { - maps[i] = null; - } + Arrays.fill(maps, null); } } - Map getBelongedMap(K key) { + Map getBelongedMap(final K key) { int slotIndex = key.hashCode() % SLOT_NUM; slotIndex = slotIndex < 0 ? slotIndex + SLOT_NUM : slotIndex; Map map = maps[slotIndex]; @@ -450,8 +388,9 @@ Map getBelongedMap(K key) { return map; } + // Copied list, deletion-safe List getAllKeys() { - List res = new ArrayList<>(); + final List res = new ArrayList<>(); Arrays.stream(maps) .iterator() .forEachRemaining( @@ -462,5 +401,15 @@ List getAllKeys() { }); return res; } + + int size() { + int size = 0; + for (final Map map : maps) { + if (Objects.nonNull(map)) { + size += map.size(); + } + } + return size; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/FIFOCacheEntryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/FIFOCacheEntryManager.java index 7d8a07acf4c86..c8dc64e01f363 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/FIFOCacheEntryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/FIFOCacheEntryManager.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; public class FIFOCacheEntryManager @@ -35,26 +36,33 @@ public class FIFOCacheEntryManager @Override public FIFOCacheEntry createCacheEntry( - SK secondKey, V value, ICacheEntryGroup> cacheEntryGroup) { + final SK secondKey, + final V value, + final ICacheEntryGroup> cacheEntryGroup) { return new FIFOCacheEntry<>(secondKey, value, cacheEntryGroup); } @Override - public void access(FIFOCacheEntry cacheEntry) { + public void access(final FIFOCacheEntry cacheEntry) { // do nothing } @Override - public void put(FIFOCacheEntry cacheEntry) { + public void put(final FIFOCacheEntry cacheEntry) { getNextList(cachePutRoundRobinIndex).add(cacheEntry); } @Override - public void invalid(FIFOCacheEntry cacheEntry) { + public boolean invalidate(final FIFOCacheEntry cacheEntry) { + if (cacheEntry.isInvalidated.getAndSet(true)) { + return false; + } + cacheEntry.next.pre = cacheEntry.pre; cacheEntry.pre.next = cacheEntry.next; cacheEntry.next = null; cacheEntry.pre = null; + return true; } @Override @@ -87,7 +95,7 @@ public void cleanUp() { } } - private FIFOLinkedList getNextList(AtomicInteger roundRobinIndex) { + private FIFOLinkedList getNextList(final AtomicInteger roundRobinIndex) { int listIndex = getNextIndex(roundRobinIndex); FIFOLinkedList fifoLinkedList = fifoLinkedLists[listIndex]; if (fifoLinkedList == null) { @@ -102,7 +110,7 @@ private FIFOLinkedList getNextList(AtomicInteger roundRobinIndex) { return fifoLinkedList; } - private int getNextIndex(AtomicInteger roundRobinIndex) { + private int getNextIndex(final AtomicInteger roundRobinIndex) { return roundRobinIndex.getAndUpdate( currentValue -> { currentValue = currentValue + 1; @@ -122,7 +130,10 @@ static class FIFOCacheEntry implements ICacheEntry { private FIFOCacheEntry pre = null; private FIFOCacheEntry next = null; - private FIFOCacheEntry(SK secondKey, V value, ICacheEntryGroup cacheEntryGroup) { + private final AtomicBoolean isInvalidated = new AtomicBoolean(false); + + private FIFOCacheEntry( + final SK secondKey, final V value, final ICacheEntryGroup cacheEntryGroup) { this.secondKey = secondKey; this.value = value; this.cacheEntryGroup = cacheEntryGroup; @@ -144,20 +155,24 @@ public ICacheEntryGroup getBelongedGroup() { } @Override - public void setBelongedGroup(ICacheEntryGroup belongedGroup) { + public void setBelongedGroup(final ICacheEntryGroup belongedGroup) { this.cacheEntryGroup = belongedGroup; } @Override - public void replaceValue(V newValue) { + public void replaceValue(final V newValue) { this.value = newValue; } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FIFOCacheEntry that = (FIFOCacheEntry) o; + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final FIFOCacheEntry that = (FIFOCacheEntry) o; return Objects.equals(secondKey, that.secondKey) && Objects.equals(cacheEntryGroup, that.cacheEntryGroup); } @@ -171,28 +186,40 @@ public int hashCode() { private static class FIFOLinkedList { // head.next is the newest - private final FIFOCacheEntry head; - private final FIFOCacheEntry tail; + private final FIFOCacheEntry head; + private final FIFOCacheEntry tail; public FIFOLinkedList() { - head = new FIFOCacheEntry(null, null, null); - tail = new FIFOCacheEntry(null, null, null); + head = new FIFOCacheEntry<>(null, null, null); + tail = new FIFOCacheEntry<>(null, null, null); head.next = tail; tail.pre = head; } - synchronized void add(FIFOCacheEntry cacheEntry) { - cacheEntry.next = head.next; + synchronized void add(final FIFOCacheEntry cacheEntry) { + FIFOCacheEntry nextEntry; + + do { + nextEntry = head.next; + } while (nextEntry.isInvalidated.get()); + + cacheEntry.next = nextEntry; cacheEntry.pre = head; - head.next.pre = cacheEntry; + nextEntry.pre = cacheEntry; head.next = cacheEntry; } - synchronized FIFOCacheEntry evict() { - if (tail.pre == head) { - return null; - } - FIFOCacheEntry cacheEntry = tail.pre; + synchronized FIFOCacheEntry evict() { + FIFOCacheEntry cacheEntry; + + do { + cacheEntry = tail.pre; + if (cacheEntry == head) { + return null; + } + + } while (cacheEntry.isInvalidated.compareAndSet(false, true)); + cacheEntry.pre.next = cacheEntry.next; cacheEntry.next.pre = cacheEntry.pre; cacheEntry.next = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryGroup.java index 89f562e31329e..d1f8ab9923e22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryGroup.java @@ -21,7 +21,9 @@ import java.util.Iterator; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; +import java.util.function.Function; /** * This interface defines the behaviour of a cache entry group, which is mainly accessed via first @@ -36,13 +38,19 @@ interface ICacheEntryGroup> { FK getFirstKey(); - T getCacheEntry(SK secondKey); + T getCacheEntry(final SK secondKey); Iterator> getAllCacheEntries(); - T computeCacheEntry(SK secondKey, BiFunction computation); + T computeCacheEntry( + final SK secondKey, final Function> computation); - T removeCacheEntry(SK secondKey); + long removeCacheEntry(final SK secondKey); boolean isEmpty(); + + // Metric + long getMemory(); + + int getEntriesCount(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java index a43e612b0e6d4..e1b1f85026ebc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java @@ -30,14 +30,19 @@ */ interface ICacheEntryManager> { - T createCacheEntry(SK secondKey, V value, ICacheEntryGroup cacheEntryGroup); + T createCacheEntry( + final SK secondKey, final V value, final ICacheEntryGroup cacheEntryGroup); - void access(T cacheEntry); + void access(final T cacheEntry); - void put(T cacheEntry); + void put(final T cacheEntry); - void invalid(T cacheEntry); + // A cacheEntry is removed iff the caller has called "invalid" and it returns "true" + // Shall never remove a cacheEntry directly or when the "invalid" returns false + boolean invalidate(final T cacheEntry); + // The "evict" is allowed to be synchronized called + // Inner implementation guarantees that an entry won't be concurrently evicted T evict(); void cleanUp(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/LRUCacheEntryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/LRUCacheEntryManager.java index 726c4489ce2a3..52d2160d6f763 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/LRUCacheEntryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/LRUCacheEntryManager.java @@ -21,6 +21,7 @@ import java.util.Objects; import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; /** * This class implements the cache entry manager with LRU policy. @@ -40,32 +41,39 @@ class LRUCacheEntryManager @Override public LRUCacheEntry createCacheEntry( - SK secondKey, V value, ICacheEntryGroup> cacheEntryGroup) { + final SK secondKey, + final V value, + final ICacheEntryGroup> cacheEntryGroup) { return new LRUCacheEntry<>(secondKey, value, cacheEntryGroup); } @Override - public void access(LRUCacheEntry cacheEntry) { + public void access(final LRUCacheEntry cacheEntry) { getBelongedList(cacheEntry).moveToHead(cacheEntry); } @Override - public void put(LRUCacheEntry cacheEntry) { + public void put(final LRUCacheEntry cacheEntry) { getBelongedList(cacheEntry).add(cacheEntry); } @Override - public void invalid(LRUCacheEntry cacheEntry) { + public boolean invalidate(final LRUCacheEntry cacheEntry) { + if (cacheEntry.isInvalidated.getAndSet(true)) { + return false; + } + cacheEntry.next.pre = cacheEntry.pre; cacheEntry.pre.next = cacheEntry.next; cacheEntry.next = null; cacheEntry.pre = null; + return true; } @Override public LRUCacheEntry evict() { int startIndex = idxGenerator.nextInt(SLOT_NUM); - LRULinkedList lruLinkedList; + LRULinkedList lruLinkedList; LRUCacheEntry cacheEntry; for (int i = 0; i < SLOT_NUM; i++) { if (startIndex == SLOT_NUM) { @@ -119,6 +127,7 @@ static class LRUCacheEntry implements ICacheEntry { private LRUCacheEntry pre; private LRUCacheEntry next; + private final AtomicBoolean isInvalidated = new AtomicBoolean(false); private LRUCacheEntry(SK secondKey, V value, ICacheEntryGroup cacheEntryGroup) { this.secondKey = secondKey; @@ -166,31 +175,43 @@ public int hashCode() { } } - private static class LRULinkedList { + private static class LRULinkedList { // head.next is the most recently used entry - private final LRUCacheEntry head; - private final LRUCacheEntry tail; + private final LRUCacheEntry head; + private final LRUCacheEntry tail; public LRULinkedList() { - head = new LRUCacheEntry(null, null, null); - tail = new LRUCacheEntry(null, null, null); + head = new LRUCacheEntry<>(null, null, null); + tail = new LRUCacheEntry<>(null, null, null); head.next = tail; tail.pre = head; } - synchronized void add(LRUCacheEntry cacheEntry) { + synchronized void add(final LRUCacheEntry cacheEntry) { + LRUCacheEntry nextEntry; + + do { + nextEntry = head.next; + } while (nextEntry.isInvalidated.get()); + cacheEntry.next = head.next; cacheEntry.pre = head; head.next.pre = cacheEntry; head.next = cacheEntry; } - synchronized LRUCacheEntry evict() { - if (tail.pre == head) { - return null; - } - LRUCacheEntry cacheEntry = tail.pre; + synchronized LRUCacheEntry evict() { + LRUCacheEntry cacheEntry; + + do { + cacheEntry = tail.pre; + if (cacheEntry == head) { + return null; + } + + } while (cacheEntry.isInvalidated.compareAndSet(false, true)); + cacheEntry.pre.next = cacheEntry.next; cacheEntry.next.pre = cacheEntry.pre; cacheEntry.next = null; @@ -198,8 +219,8 @@ synchronized LRUCacheEntry evict() { return cacheEntry; } - synchronized void moveToHead(LRUCacheEntry cacheEntry) { - if (cacheEntry.next == null || cacheEntry.pre == null) { + synchronized void moveToHead(final LRUCacheEntry cacheEntry) { + if (cacheEntry.isInvalidated.get()) { // this cache entry has been evicted return; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 698ff13abe6f7..db3a6b9d79e41 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -98,6 +98,7 @@ public enum Metric { MEM("mem"), CACHE("cache"), CACHE_HIT_RATE("cache_hit"), + CACHE_ENTRIES_NUM("cache_entries_num"), QUANTITY("quantity"), LEADER_QUANTITY("leader_quantity"), SCHEMA_REGION("schema_region"), From 43730da65b25ecfc6865054f7f68c3a9cb363501 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 14:28:18 +0800 Subject: [PATCH 02/33] second removal --- .../impl/DataNodeInternalRPCServiceImpl.java | 2 +- .../cache/schema/DataNodeSchemaCache.java | 40 ++----- .../cache/schema/DeviceSchemaCache.java | 104 +----------------- 3 files changed, 15 insertions(+), 131 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index c7c51ed5f4b54..336d6b0ae729e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -537,7 +537,7 @@ public TSStatus invalidatePartitionCache(final TInvalidateCacheReq req) { @Override public TSStatus invalidateLastCache(final String database) { - DataNodeSchemaCache.getInstance().invalidateLastCacheInDataRegion(database); + DataNodeSchemaCache.getInstance().invalidateDatabaseLastCache(database); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index d4ca47349876c..c376e7494d64e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -32,14 +32,13 @@ import org.apache.iotdb.db.schemaengine.template.ITemplateManager; import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; -import javax.annotation.Nonnull; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -68,11 +67,11 @@ private DataNodeSchemaCache() { } public static DataNodeSchemaCache getInstance() { - return TreeDeviceSchemaCacheManagerHolder.INSTANCE; + return DataNodeSchemaCacheHolder.INSTANCE; } /** singleton pattern. */ - private static class TreeDeviceSchemaCacheManagerHolder { + private static class DataNodeSchemaCacheHolder { private static final DataNodeSchemaCache INSTANCE = new DataNodeSchemaCache(); } @@ -288,14 +287,14 @@ public List computeWithTemplate(final ISchemaComputation computation) { new IMeasurementSchemaInfo() { @Override public String getName() { - return schema.getMeasurementName(); + return schema.getMeasurementId(); } @Override public IMeasurementSchema getSchema() { if (isLogicalView()) { return new LogicalViewSchema( - schema.getMeasurementName(), ((LogicalViewSchema) schema).getExpression()); + schema.getMeasurementId(), ((LogicalViewSchema) schema).getExpression()); } else { return this.getSchemaAsMeasurementSchema(); } @@ -304,7 +303,7 @@ public IMeasurementSchema getSchema() { @Override public MeasurementSchema getSchemaAsMeasurementSchema() { return new MeasurementSchema( - schema.getMeasurementName(), + schema.getMeasurementId(), schema.getType(), schema.getEncodingType(), schema.getCompressor()); @@ -353,6 +352,11 @@ public void put(final ClusterSchemaTree tree) { tree.getBelongedDatabase(deviceSchemaInfo.getDevicePath()), deviceSchemaInfo)); } + public boolean getLastCache( + final Map>>> inputMap) { + return deviceSchemaCache.getLastCache(inputMap); + } + public TimeValuePair getLastCache(final MeasurementPath seriesPath) { return deviceSchemaCache.getLastEntry( null, seriesPath.getIDeviceID(), seriesPath.getMeasurement()); @@ -372,28 +376,6 @@ public void invalidateDatabaseLastCache(final String database) { deviceSchemaCache.invalidateLastCache(database); } - /** - * Update the {@link DeviceLastCache} in writing for tree model. If a measurement is with all - * {@code null}s or is an id/attribute column, its {@link TimeValuePair[]} shall be {@code null}. - * For correctness, this will put the {@link TableDeviceCacheEntry} lazily and only update the - * existing {@link DeviceLastCache}s of measurements. - * - * @param database the device's database, WITH "root" - * @param deviceID {@link IDeviceID} - * @param measurements the fetched measurements - * @param timeValuePairs the {@link TimeValuePair}s with indexes corresponding to the measurements - */ - public void updateLastCacheIfExists( - final String database, - final IDeviceID deviceID, - final String[] measurements, - final @Nonnull TimeValuePair[] timeValuePairs, - final boolean isAligned, - final IMeasurementSchema[] measurementSchemas) { - deviceSchemaCache.updateLastCache( - database, deviceID, measurements, timeValuePairs, isAligned, measurementSchemas, false); - } - /** * Update the {@link DeviceLastCache} on query in tree model. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index 827b2e2219ebd..5c861a3320827 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -24,7 +24,6 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.service.metric.MetricService; -import org.apache.iotdb.db.conf.DataNodeMemoryConfig; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; @@ -39,7 +38,6 @@ import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,8 +50,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.ToIntFunction; @@ -83,8 +79,6 @@ public class DeviceSchemaCache { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private static final DataNodeMemoryConfig memoryConfig = - IoTDBDescriptor.getInstance().getMemoryConfig(); private static final Logger logger = LoggerFactory.getLogger(DeviceSchemaCache.class); /** @@ -97,7 +91,7 @@ public class DeviceSchemaCache { */ private final IDualKeyCache dualKeyCache; - private final Map treeModelDatabasePool = new ConcurrentHashMap<>(); + private final Map databasePool = new ConcurrentHashMap<>(); private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); @@ -116,52 +110,6 @@ public class DeviceSchemaCache { /////////////////////////////// Last Cache /////////////////////////////// - /** - * Update the last cache in writing or the second push of last cache query. If a measurement is - * with all {@code null}s or is a tag/attribute column, its {@link TimeValuePair}[] shall be - * {@code null}. For correctness, this will put the cache lazily and only update the existing last - * caches of measurements. - * - * @param database the device's database, without "root" - * @param deviceId {@link IDeviceID} - * @param measurements the fetched measurements - * @param timeValuePairs the {@link TimeValuePair}s with indexes corresponding to the measurements - * @param invalidateNull when true invalidate cache entries where timeValuePairs[i] == null; when - * false ignore cache entries where timeValuePairs[i] == null - */ - public void updateLastCacheIfExists( - final String database, - final IDeviceID deviceId, - final String[] measurements, - final TimeValuePair[] timeValuePairs, - boolean invalidateNull) { - dualKeyCache.update( - new TableId(database, deviceId.getTableName()), - deviceId, - null, - entry -> entry.tryUpdateLastCache(measurements, timeValuePairs, invalidateNull), - false); - } - - /** - * Update the last cache in writing or the second push of last cache query. If a measurement is - * with all {@code null}s or is a tag/attribute column, its {@link TimeValuePair}[] shall be - * {@code null}. For correctness, this will put the cache lazily and only update the existing last - * caches of measurements. - * - * @param database the device's database, without "root" - * @param deviceId {@link IDeviceID} - * @param measurements the fetched measurements - * @param timeValuePairs the {@link TimeValuePair}s with indexes corresponding to the measurements - */ - public void updateLastCacheIfExists( - final String database, - final IDeviceID deviceId, - final String[] measurements, - final TimeValuePair[] timeValuePairs) { - updateLastCacheIfExists(database, deviceId, measurements, timeValuePairs, false); - } - /** * Get the last {@link TimeValuePair} of a measurement, the measurement shall never be "time". * @@ -178,52 +126,6 @@ public TimeValuePair getLastEntry( return Objects.nonNull(entry) ? entry.getTimeValuePair(measurement) : null; } - /** - * Get the last {@link TimeValuePair}s of given measurements, the measurements shall never be - * "time". If you want to get the last of "time", use "" to represent. - * - * @param database the device's database, without "root", {@code null} for tree model - * @param deviceId {@link IDeviceID} - * @param measurements the measurements to get - * @return {@code null} iff cache miss, {@link DeviceLastCache#EMPTY_TIME_VALUE_PAIR} iff cache - * hit but result is {@code null}, and the result value otherwise. - */ - public TimeValuePair[] getLastEntries( - final @Nullable String database, final IDeviceID deviceId, final String[] measurements) { - final TableDeviceCacheEntry entry = - dualKeyCache.get(new TableId(database, deviceId.getTableName()), deviceId); - return Objects.nonNull(entry) - ? Arrays.stream(measurements).map(entry::getTimeValuePair).toArray(TimeValuePair[]::new) - : null; - } - - /** - * Get the last value of measurements last by a target measurement. If the caller wants to last by - * time or get the time last by another source measurement, the measurement shall be "" to - * indicate the time column. - * - * @param database the device's database, without "root" - * @param deviceId {@link IDeviceID} - * @param sourceMeasurement the measurement to get - * @return {@code Optional.empty()} iff the last cache is miss at all; Or the optional of a pair, - * the {@link Pair#left} will be the source measurement's last time, (OptionalLong.empty() iff - * the source measurement is all {@code null}); {@link Pair#right} will be an {@link - * TsPrimitiveType} array, whose element will be {@code null} if cache miss, {@link - * DeviceLastCache#EMPTY_PRIMITIVE_TYPE} iff cache hit and the measurement is without any - * values when last by the source measurement's time, and the result value otherwise. - */ - public Optional> getLastRow( - final String database, - final IDeviceID deviceId, - final String sourceMeasurement, - final List targetMeasurements) { - final TableDeviceCacheEntry entry = - dualKeyCache.get(new TableId(database, deviceId.getTableName()), deviceId); - return Objects.nonNull(entry) - ? entry.getLastRow(sourceMeasurement, targetMeasurements) - : Optional.empty(); - } - /** * Invalidate the last cache of one table. * @@ -255,7 +157,7 @@ public void invalidateLastCache(final String database, final IDeviceID deviceId) public void putDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { final PartialPath devicePath = deviceSchemaInfo.getDevicePath(); final IDeviceID deviceID = devicePath.getIDeviceID(); - final String previousDatabase = treeModelDatabasePool.putIfAbsent(database, database); + final String previousDatabase = databasePool.putIfAbsent(database, database); dualKeyCache.update( new TableId(null, deviceID.getTableName()), @@ -287,7 +189,7 @@ void updateLastCache( final boolean isAligned, final IMeasurementSchema[] measurementSchemas, final boolean initOrInvalidate) { - final String previousDatabase = treeModelDatabasePool.putIfAbsent(database, database); + final String previousDatabase = databasePool.putIfAbsent(database, database); final String database2Use = Objects.nonNull(previousDatabase) ? previousDatabase : database; dualKeyCache.update( From f5a49dd20e56c14681168877c03433f1d2d843d6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 14:33:30 +0800 Subject: [PATCH 03/33] Fix and removal --- .../cache/schema/DataNodeSchemaCache.java | 26 ++++++- .../schema/lastcache/ILastCacheContainer.java | 45 ----------- .../lastcache/value/ILastCacheValue.java | 39 ---------- .../lastcache/value/LastCacheValue.java | 74 ------------------- 4 files changed, 25 insertions(+), 159 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index c376e7494d64e..2bb38bad46df5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -39,6 +39,8 @@ import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; +import javax.annotation.Nonnull; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -357,7 +359,7 @@ public boolean getLastCache( return deviceSchemaCache.getLastCache(inputMap); } - public TimeValuePair getLastCache(final MeasurementPath seriesPath) { + public TimeValuePair getLastCache(final PartialPath seriesPath) { return deviceSchemaCache.getLastEntry( null, seriesPath.getIDeviceID(), seriesPath.getMeasurement()); } @@ -376,6 +378,28 @@ public void invalidateDatabaseLastCache(final String database) { deviceSchemaCache.invalidateLastCache(database); } + /** + * Update the {@link TableDeviceLastCache} in writing for tree model. If a measurement is with all + * {@code null}s or is an id/attribute column, its {@link TimeValuePair[]} shall be {@code null}. + * For correctness, this will put the {@link TableDeviceCacheEntry} lazily and only update the + * existing {@link TableDeviceLastCache}s of measurements. + * + * @param database the device's database, WITH "root" + * @param deviceID {@link IDeviceID} + * @param measurements the fetched measurements + * @param timeValuePairs the {@link TimeValuePair}s with indexes corresponding to the measurements + */ + public void updateLastCacheIfExists( + final String database, + final IDeviceID deviceID, + final String[] measurements, + final @Nonnull TimeValuePair[] timeValuePairs, + final boolean isAligned, + final IMeasurementSchema[] measurementSchemas) { + deviceSchemaCache.updateLastCache( + database, deviceID, measurements, timeValuePairs, isAligned, measurementSchemas, false); + } + /** * Update the {@link DeviceLastCache} on query in tree model. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java index edd0c202b9b06..e69de29bb2d1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache; - -import org.apache.tsfile.read.TimeValuePair; - -/** this interface declares the operations of LastCache data */ -public interface ILastCacheContainer { - - // get lastCache of monad timeseries - TimeValuePair getCachedLast(); - - /** - * update last point cache and enable last cache. - * - * @param timeValuePair last point - * @param highPriorityUpdate whether it's a high priority update - * @param latestFlushedTime latest flushed time - * @return increasing of memory usage - */ - int updateCachedLast( - TimeValuePair timeValuePair, boolean highPriorityUpdate, Long latestFlushedTime); - - /** Invalidate Last cache. */ - int invalidateLastCache(); - - int estimateSize(); -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java index c7782ef260584..e69de29bb2d1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.value; - -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.TsPrimitiveType; - -// this interface declares the simplest storage operation of lastCacheValue -public interface ILastCacheValue { - - long getTimestamp(); - - TsPrimitiveType getValue(); - - void setTimestamp(long timestamp); - - void setValue(TsPrimitiveType value); - - TimeValuePair getTimeValuePair(); - - int estimateSize(); -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java index c7b48da4c0958..e69de29bb2d1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.value; - -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.TsPrimitiveType; - -public class LastCacheValue implements ILastCacheValue { - - private long timestamp; - - private TsPrimitiveType value; - - public LastCacheValue(long timestamp, TsPrimitiveType value) { - this.timestamp = timestamp; - this.value = value; - } - - @Override - public long getTimestamp() { - return timestamp; - } - - @Override - public TsPrimitiveType getValue() { - return value; - } - - @Override - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - @Override - public void setValue(TsPrimitiveType value) { - this.value = value; - } - - @Override - public TimeValuePair getTimeValuePair() { - return new TimeValuePair(timestamp, value); - } - - /** - * Total basic 24B - * - *

    - *
  • LastCacheContainer Object header, 8B - *
  • long timestamp, 8B - *
  • TsPrimitiveType reference, 8B - *
- */ - @Override - public int estimateSize() { - return 24 + (value == null ? 0 : value.getSize()); - } -} From b79b2055e281bac7fb9b0619c41d29022ba349b5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 15:46:46 +0800 Subject: [PATCH 04/33] deletion --- .../DeviceUsingTemplateSchemaCache.java | 235 ---------- .../cache/schema/TimeSeriesSchemaCache.java | 441 ------------------ 2 files changed, 676 deletions(-) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java deleted file mode 100644 index 167242ff20cc3..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; - -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.schema.view.LogicalViewSchema; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; -import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; -import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; -import org.apache.iotdb.db.schemaengine.template.ITemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.Weigher; -import org.apache.tsfile.write.schema.IMeasurementSchema; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class DeviceUsingTemplateSchemaCache { - - private static final Logger logger = - LoggerFactory.getLogger(DeviceUsingTemplateSchemaCache.class); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - - private final Cache cache; - - private final ITemplateManager templateManager; - - DeviceUsingTemplateSchemaCache(ITemplateManager templateManager) { - this.templateManager = templateManager; - // TODO proprietary config parameter expected - cache = - Caffeine.newBuilder() - .maximumWeight(config.getAllocateMemoryForSchemaCache()) - .weigher( - (Weigher) - (key, val) -> (PartialPath.estimateSize(key) + 32)) - .recordStats() - .build(); - } - - public long getHitCount() { - return cache.stats().hitCount(); - } - - public long getRequestCount() { - return cache.stats().requestCount(); - } - - public ClusterSchemaTree get(PartialPath fullPath) { - DeviceCacheEntry deviceCacheEntry = cache.getIfPresent(fullPath.getDevicePath()); - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - if (deviceCacheEntry != null) { - Template template = templateManager.getTemplate(deviceCacheEntry.getTemplateId()); - if (template.getSchema(fullPath.getMeasurement()) != null) { - schemaTree.appendTemplateDevice( - fullPath.getDevicePath(), template.isDirectAligned(), template.getId(), template); - schemaTree.setDatabases(Collections.singleton(deviceCacheEntry.getDatabase())); - } - } - return schemaTree; - } - - public ClusterSchemaTree getMatchedSchemaWithTemplate(PartialPath devicePath) { - DeviceCacheEntry deviceCacheEntry = cache.getIfPresent(devicePath); - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - if (deviceCacheEntry != null) { - Template template = templateManager.getTemplate(deviceCacheEntry.getTemplateId()); - schemaTree.appendTemplateDevice( - devicePath, template.isDirectAligned(), template.getId(), template); - schemaTree.setDatabases(Collections.singleton(deviceCacheEntry.getDatabase())); - } - return schemaTree; - } - - /** - * CONFORM indicates that the provided devicePath had been cached as a template activated path, - * ensuring that the alignment of the device, as well as the name and schema of every measurement - * are consistent with the cache. - * - * @param computation - * @return true if conform to template cache, which means no need to fetch or create anymore - */ - public List compute(ISchemaComputation computation) { - List indexOfMissingMeasurements = new ArrayList<>(); - PartialPath devicePath = computation.getDevicePath(); - String[] measurements = computation.getMeasurements(); - DeviceCacheEntry deviceCacheEntry = cache.getIfPresent(devicePath); - if (deviceCacheEntry == null) { - for (int i = 0; i < measurements.length; i++) { - indexOfMissingMeasurements.add(i); - } - return indexOfMissingMeasurements; - } - - computation.computeDevice( - templateManager.getTemplate(deviceCacheEntry.getTemplateId()).isDirectAligned()); - Map templateSchema = - templateManager.getTemplate(deviceCacheEntry.getTemplateId()).getSchemaMap(); - for (int i = 0; i < measurements.length; i++) { - if (!templateSchema.containsKey(measurements[i])) { - indexOfMissingMeasurements.add(i); - continue; - } - IMeasurementSchema schema = templateSchema.get(measurements[i]); - computation.computeMeasurement( - i, - new IMeasurementSchemaInfo() { - @Override - public String getName() { - return schema.getMeasurementId(); - } - - @Override - public IMeasurementSchema getSchema() { - if (isLogicalView()) { - return new LogicalViewSchema( - schema.getMeasurementId(), ((LogicalViewSchema) schema).getExpression()); - } else { - return this.getSchemaAsMeasurementSchema(); - } - } - - @Override - public MeasurementSchema getSchemaAsMeasurementSchema() { - return new MeasurementSchema( - schema.getMeasurementId(), - schema.getType(), - schema.getEncodingType(), - schema.getCompressor()); - } - - @Override - public LogicalViewSchema getSchemaAsLogicalViewSchema() { - throw new RuntimeException( - new UnsupportedOperationException( - "Function getSchemaAsLogicalViewSchema is not supported in DeviceUsingTemplateSchemaCache.")); - } - - @Override - public Map getTagMap() { - return null; - } - - @Override - public Map getAttributeMap() { - return null; - } - - @Override - public String getAlias() { - return null; - } - - @Override - public boolean isLogicalView() { - return schema.isLogicalView(); - } - }); - } - return indexOfMissingMeasurements; - } - - public void put(PartialPath path, String database, Integer id) { - cache.put(path, new DeviceCacheEntry(database, id)); - } - - public void invalidateCache() { - cache.invalidateAll(); - } - - public void invalidateCache(String database) { - for (PartialPath key : cache.asMap().keySet()) { - if (key.startsWith(database)) { - cache.invalidate(key); - } - } - } - - public void invalidateCache(List partialPathList) { - for (PartialPath path : partialPathList) { - for (PartialPath key : cache.asMap().keySet()) { - if (key.startsWith(path.getDevice())) { - cache.invalidate(key); - } - } - } - } - - private static class DeviceCacheEntry { - - private final String database; - - private final int templateId; - - private DeviceCacheEntry(String database, int templateId) { - this.database = database.intern(); - this.templateId = templateId; - } - - public int getTemplateId() { - return templateId; - } - - public String getDatabase() { - return database; - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java deleted file mode 100644 index 51f22abd1e968..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; - -import org.apache.iotdb.commons.path.MeasurementPath; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.schema.view.LogicalViewSchema; -import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.metadata.view.InsertNonWritableViewException; -import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheComputation; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheUpdating; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.DataNodeLastCacheManager; -import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; - -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.IntFunction; -import java.util.function.IntPredicate; - -public class TimeSeriesSchemaCache { - - private static final Logger logger = LoggerFactory.getLogger(DataNodeSchemaCache.class); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - - // - private final IDualKeyCache dualKeyCache; - - TimeSeriesSchemaCache() { - DualKeyCacheBuilder dualKeyCacheBuilder = - new DualKeyCacheBuilder<>(); - dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy( - DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) - .memoryCapacity(config.getAllocateMemoryForSchemaCache()) - .firstKeySizeComputer(PartialPath::estimateSize) - .secondKeySizeComputer(s -> 32 + 2 * s.length()) - .valueSizeComputer(SchemaCacheEntry::estimateSize) - .build(); - } - - public long getHitCount() { - return dualKeyCache.stats().hitCount(); - } - - public long getRequestCount() { - return dualKeyCache.stats().requestCount(); - } - - /** - * Get SchemaEntity info without auto create schema - * - * @param devicePath should not be measurementPath or AlignedPath - * @param measurements - * @return timeseries partialPath and its SchemaEntity - */ - public ClusterSchemaTree get(PartialPath devicePath, String[] measurements) { - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - Set storageGroupSet = new HashSet<>(); - - dualKeyCache.compute( - new IDualKeyCacheComputation() { - @Override - public PartialPath getFirstKey() { - return devicePath; - } - - @Override - public String[] getSecondKeyList() { - return measurements; - } - - @Override - public void computeValue(int index, SchemaCacheEntry value) { - if (value != null) { - schemaTree.appendSingleMeasurement( - devicePath.concatNode(value.getSchemaEntryId()), - value.getIMeasurementSchema(), - value.getTagMap(), - null, - null, - value.isAligned()); - storageGroupSet.add(value.getStorageGroup()); - } - } - }); - schemaTree.setDatabases(storageGroupSet); - return schemaTree; - } - - public ClusterSchemaTree get(PartialPath fullPath) { - SchemaCacheEntry schemaCacheEntry = - dualKeyCache.get(fullPath.getDevicePath(), fullPath.getMeasurement()); - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - if (schemaCacheEntry != null) { - schemaTree.appendSingleMeasurement( - fullPath, - schemaCacheEntry.getIMeasurementSchema(), - schemaCacheEntry.getTagMap(), - null, - null, - schemaCacheEntry.isAligned()); - schemaTree.setDatabases(Collections.singleton(schemaCacheEntry.getStorageGroup())); - } - return schemaTree; - } - - public List computeAndRecordLogicalView(ISchemaComputation schemaComputation) { - List indexOfMissingMeasurements = new ArrayList<>(); - final AtomicBoolean isFirstNonViewMeasurement = new AtomicBoolean(true); - dualKeyCache.compute( - new IDualKeyCacheComputation() { - @Override - public PartialPath getFirstKey() { - return schemaComputation.getDevicePath(); - } - - @Override - public String[] getSecondKeyList() { - return schemaComputation.getMeasurements(); - } - - @Override - public void computeValue(int index, SchemaCacheEntry value) { - if (value == null) { - indexOfMissingMeasurements.add(index); - } else { - if (isFirstNonViewMeasurement.get() && (!value.isLogicalView())) { - schemaComputation.computeDevice(value.isAligned()); - isFirstNonViewMeasurement.getAndSet(false); - } - schemaComputation.computeMeasurement(index, value); - } - } - }); - return indexOfMissingMeasurements; - } - - public Pair, List> computeSourceOfLogicalView( - ISchemaComputation schemaComputation) { - List indexOfMissingMeasurements = new ArrayList<>(); - List missedPathStringList = new ArrayList<>(); - Pair beginToEnd = schemaComputation.getRangeOfLogicalViewSchemaListRecorded(); - List logicalViewSchemaList = schemaComputation.getLogicalViewSchemaList(); - List indexListOfLogicalViewPaths = schemaComputation.getIndexListOfLogicalViewPaths(); - for (int i = beginToEnd.left; i < beginToEnd.right; i++) { - LogicalViewSchema logicalViewSchema = logicalViewSchemaList.get(i); - final int realIndex = indexListOfLogicalViewPaths.get(i); - final int recordMissingIndex = i; - if (!logicalViewSchema.isWritable()) { - PartialPath path = schemaComputation.getDevicePath(); - path = path.concatNode(schemaComputation.getMeasurements()[realIndex]); - throw new RuntimeException(new InsertNonWritableViewException(path.getFullPath())); - } - PartialPath fullPath = logicalViewSchema.getSourcePathIfWritable(); - dualKeyCache.compute( - new IDualKeyCacheComputation() { - @Override - public PartialPath getFirstKey() { - return fullPath.getDevicePath(); - } - - @Override - public String[] getSecondKeyList() { - return new String[] {fullPath.getMeasurement()}; - } - - @Override - public void computeValue(int index, SchemaCacheEntry value) { - if (value == null) { - indexOfMissingMeasurements.add(recordMissingIndex); - } else { - // Can not call function computeDevice here, because the value is source of one - // view, but schemaComputation is the device in this insert statement. The - // computation between them is miss matched. - if (value.isLogicalView()) { - // does not support views in views - throw new RuntimeException( - new UnsupportedOperationException( - String.format( - "The source of view [%s] is also a view! Nested view is unsupported! " - + "Please check it.", - fullPath))); - } - schemaComputation.computeMeasurementOfView(realIndex, value, value.isAligned()); - } - } - }); - } - for (int index : indexOfMissingMeasurements) { - missedPathStringList.add(logicalViewSchemaList.get(index).getSourcePathStringIfWritable()); - } - return new Pair<>(indexOfMissingMeasurements, missedPathStringList); - } - - public void putSingleMeasurementPath(String storageGroup, MeasurementPath measurementPath) { - SchemaCacheEntry schemaCacheEntry = - new SchemaCacheEntry( - storageGroup, - measurementPath.getMeasurementSchema(), - measurementPath.getTagMap(), - measurementPath.isUnderAlignedEntity()); - dualKeyCache.put( - measurementPath.getDevicePath(), measurementPath.getMeasurement(), schemaCacheEntry); - } - - public TimeValuePair getLastCache(PartialPath seriesPath) { - SchemaCacheEntry entry = - dualKeyCache.get(seriesPath.getDevicePath(), seriesPath.getMeasurement()); - if (null == entry) { - return null; - } - - return DataNodeLastCacheManager.getLastCache(entry); - } - - public boolean getLastCache(final Map> inputMap) { - return dualKeyCache.batchGet(inputMap, DataNodeLastCacheManager::getLastCache); - } - - /** get SchemaCacheEntry and update last cache */ - @TestOnly - public void updateLastCache( - PartialPath devicePath, - String measurement, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - SchemaCacheEntry entry = dualKeyCache.get(devicePath, measurement); - if (null == entry) { - return; - } - - DataNodeLastCacheManager.updateLastCache( - entry, timeValuePair, highPriorityUpdate, latestFlushedTime); - } - - /** get SchemaCacheEntry and update last cache by device */ - public void updateLastCache( - String database, - PartialPath devicePath, - String[] measurements, - MeasurementSchema[] measurementSchemas, - boolean isAligned, - IntFunction timeValuePairProvider, - IntPredicate shouldUpdateProvider, - boolean highPriorityUpdate, - Long latestFlushedTime) { - SchemaCacheEntry entry; - List missingMeasurements = new ArrayList<>(); - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public PartialPath getFirstKey() { - return devicePath; - } - - @Override - public String[] getSecondKeyList() { - return measurements; - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - if (!shouldUpdateProvider.test(index)) { - return 0; - } - if (value == null) { - missingMeasurements.add(index); - return 0; - } else { - return DataNodeLastCacheManager.updateLastCache( - value, timeValuePairProvider.apply(index), highPriorityUpdate, latestFlushedTime); - } - } - }); - - for (int index : missingMeasurements) { - entry = dualKeyCache.get(devicePath, measurements[index]); - if (entry == null) { - synchronized (dualKeyCache) { - entry = dualKeyCache.get(devicePath, measurements[index]); - if (null == entry && measurementSchemas != null) { - entry = new SchemaCacheEntry(database, measurementSchemas[index], null, isAligned); - dualKeyCache.put(devicePath, measurements[index], entry); - } - } - } - } - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public PartialPath getFirstKey() { - return devicePath; - } - - @Override - public String[] getSecondKeyList() { - return missingMeasurements.stream().map(i -> measurements[i]).toArray(String[]::new); - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - return DataNodeLastCacheManager.updateLastCache( - value, - timeValuePairProvider.apply(missingMeasurements.get(index)), - highPriorityUpdate, - latestFlushedTime); - } - }); - } - - /** - * get or create SchemaCacheEntry and update last cache, only support non-aligned sensor or - * aligned sensor without only one sub sensor - */ - public void updateLastCache( - String storageGroup, - MeasurementPath measurementPath, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - PartialPath seriesPath = measurementPath.transformToPartialPath(); - SchemaCacheEntry entry = - dualKeyCache.get(seriesPath.getDevicePath(), seriesPath.getMeasurement()); - if (null == entry) { - synchronized (dualKeyCache) { - entry = dualKeyCache.get(seriesPath.getDevicePath(), seriesPath.getMeasurement()); - if (null == entry) { - entry = - new SchemaCacheEntry( - storageGroup, - measurementPath.getMeasurementSchema(), - measurementPath.getTagMap(), - measurementPath.isUnderAlignedEntity()); - dualKeyCache.put(seriesPath.getDevicePath(), seriesPath.getMeasurement(), entry); - } - } - } - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public PartialPath getFirstKey() { - return measurementPath.getDevicePath(); - } - - @Override - public String[] getSecondKeyList() { - return new String[] {measurementPath.getMeasurement()}; - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - return DataNodeLastCacheManager.updateLastCache( - value, timeValuePair, highPriorityUpdate, latestFlushedTime); - } - }); - } - - public void invalidateAll() { - dualKeyCache.invalidateAll(); - } - - public void invalidate(String database) { - dualKeyCache.invalidate(database); - } - - public void invalidateLastCache(PartialPath path) { - if (!path.hasWildcard()) { - SchemaCacheEntry entry = dualKeyCache.get(path.getDevicePath(), path.getMeasurement()); - if (null == entry) { - return; - } - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public PartialPath getFirstKey() { - return path.getDevicePath(); - } - - @Override - public String[] getSecondKeyList() { - return new String[] {path.getMeasurement()}; - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - return -DataNodeLastCacheManager.invalidateLastCache(value); - } - }); - } else { - dualKeyCache.invalidateLastCache(path); - } - } - - public void invalidateDataRegionLastCache(String database) { - dualKeyCache.invalidateDataRegionLastCache(database); - } - - public void invalidate(List partialPathList) { - dualKeyCache.invalidate(partialPathList); - } - - public void cleanUp() { - dualKeyCache.cleanUp(); - } -} From d7981a7f9c4fdcb0a0ba1793530e5bbe38ff37d2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 16:15:48 +0800 Subject: [PATCH 05/33] partial --- .../cache/schema/DataNodeSchemaCache.java | 6 +- .../cache/DataNodeSchemaCacheTest.java | 213 ++++++++---------- 2 files changed, 91 insertions(+), 128 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 2bb38bad46df5..6a98f63f758f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -379,10 +379,10 @@ public void invalidateDatabaseLastCache(final String database) { } /** - * Update the {@link TableDeviceLastCache} in writing for tree model. If a measurement is with all - * {@code null}s or is an id/attribute column, its {@link TimeValuePair[]} shall be {@code null}. + * Update the {@link DeviceLastCache} in writing for tree model. If a measurement is with all + * {@code null}s or is an id/attribute column, its {@link TimeValuePair}[] shall be {@code null}. * For correctness, this will put the {@link TableDeviceCacheEntry} lazily and only update the - * existing {@link TableDeviceLastCache}s of measurements. + * existing {@link DeviceLastCache}s of measurements. * * @param database the device's database, WITH "root" * @param deviceID {@link IDeviceID} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java index fc1e2f6ff6943..b2aaa910ffc11 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.metadata.cache; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; @@ -30,6 +31,7 @@ import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; @@ -50,11 +52,16 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; public class DataNodeSchemaCacheTest { - DataNodeSchemaCache dataNodeSchemaCache; + private DataNodeSchemaCache dataNodeSchemaCache; private Map s1TagMap; + private boolean lastCacheEnabledDefaultValue; @Before public void setUp() throws Exception { + lastCacheEnabledDefaultValue = CommonDescriptor.getInstance().getConfig().isLastCacheEnable(); + if (!lastCacheEnabledDefaultValue) { + CommonDescriptor.getInstance().getConfig().setLastCacheEnable(true); + } dataNodeSchemaCache = DataNodeSchemaCache.getInstance(); s1TagMap = new HashMap<>(); s1TagMap.put("k1", "v1"); @@ -62,14 +69,15 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { + CommonDescriptor.getInstance().getConfig().setLastCacheEnable(lastCacheEnabledDefaultValue); dataNodeSchemaCache.cleanUp(); ClusterTemplateManager.getInstance().clear(); } @Test public void testGetSchemaEntity() throws IllegalPathException { - PartialPath device1 = new PartialPath("root.sg1.d1"); - String[] measurements = new String[3]; + final PartialPath device1 = new PartialPath("root.sg1.d1"); + final String[] measurements = new String[3]; measurements[0] = "s1"; measurements[1] = "s2"; measurements[2] = "s3"; @@ -82,12 +90,7 @@ public void testGetSchemaEntity() throws IllegalPathException { .collect( Collectors.toMap( o -> new PartialPath(o.getNodes()), - o -> - new SchemaCacheEntry( - "root.sg1", - o.getMeasurementSchema(), - o.getTagMap(), - o.isUnderAlignedEntity()))); + o -> new SchemaCacheEntry(o.getMeasurementSchema(), o.getTagMap()))); Assert.assertEquals( TSDataType.INT32, schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s1")).getTsDataType()); @@ -102,7 +105,7 @@ public void testGetSchemaEntity() throws IllegalPathException { schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s3")).getTsDataType()); Assert.assertNull(schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s3")).getTagMap()); - String[] otherMeasurements = new String[3]; + final String[] otherMeasurements = new String[3]; otherMeasurements[0] = "s3"; otherMeasurements[1] = "s4"; otherMeasurements[2] = "s5"; @@ -115,12 +118,7 @@ public void testGetSchemaEntity() throws IllegalPathException { .collect( Collectors.toMap( o -> new PartialPath(o.getNodes()), - o -> - new SchemaCacheEntry( - "root.sg1", - o.getMeasurementSchema(), - o.getTagMap(), - o.isUnderAlignedEntity()))); + o -> new SchemaCacheEntry(o.getMeasurementSchema(), o.getTagMap()))); Assert.assertEquals( TSDataType.BOOLEAN, schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s3")).getTsDataType()); @@ -135,71 +133,9 @@ public void testGetSchemaEntity() throws IllegalPathException { Assert.assertNull(schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s4")).getTagMap()); } - @Test - public void testLastCache() throws IllegalPathException { - // test no cache - PartialPath devicePath = new PartialPath("root.sg1.d1"); - PartialPath seriesPath1 = new PartialPath("root.sg1.d1.s1"); - PartialPath seriesPath2 = new PartialPath("root.sg1.d1.s2"); - PartialPath seriesPath3 = new PartialPath("root.sg1.d1.s3"); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath1)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - // test no last cache - dataNodeSchemaCache.put((ClusterSchemaTree) generateSchemaTree1()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath1)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - // put cache - long timestamp = 100; - long timestamp2 = 101; - TsPrimitiveType value = TsPrimitiveType.getByType(TSDataType.INT32, 101); - TsPrimitiveType value2 = TsPrimitiveType.getByType(TSDataType.INT32, 100); - TsPrimitiveType value3 = TsPrimitiveType.getByType(TSDataType.INT32, 99); - - // put into last cache when cache not exist - TimeValuePair timeValuePair = new TimeValuePair(timestamp, value); - dataNodeSchemaCache.updateLastCache(devicePath, "s1", timeValuePair, false, 99L); - TimeValuePair cachedTimeValuePair = dataNodeSchemaCache.getLastCache(seriesPath1); - Assert.assertNotNull(cachedTimeValuePair); - Assert.assertEquals(timestamp, cachedTimeValuePair.getTimestamp()); - Assert.assertEquals(value, cachedTimeValuePair.getValue()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - - // same time but low priority - TimeValuePair timeValuePair2 = new TimeValuePair(timestamp, value2); - dataNodeSchemaCache.updateLastCache(devicePath, "s1", timeValuePair2, false, 100L); - TimeValuePair cachedTimeValuePair2 = dataNodeSchemaCache.getLastCache(seriesPath1); - Assert.assertNotNull(cachedTimeValuePair2); - Assert.assertEquals(timestamp, cachedTimeValuePair2.getTimestamp()); - Assert.assertEquals(value, cachedTimeValuePair2.getValue()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - - // same time but high priority - dataNodeSchemaCache.updateLastCache(devicePath, "s1", timeValuePair2, true, 100L); - cachedTimeValuePair2 = dataNodeSchemaCache.getLastCache(seriesPath1); - Assert.assertNotNull(cachedTimeValuePair2); - Assert.assertEquals(timestamp, cachedTimeValuePair2.getTimestamp()); - Assert.assertEquals(value2, cachedTimeValuePair2.getValue()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - - // put into last cache when cache already exist - TimeValuePair timeValuePair3 = new TimeValuePair(timestamp2, value3); - dataNodeSchemaCache.updateLastCache(devicePath, "s1", timeValuePair3, false, 100L); - TimeValuePair cachedTimeValuePair3 = dataNodeSchemaCache.getLastCache(seriesPath1); - Assert.assertNotNull(cachedTimeValuePair3); - Assert.assertEquals(timestamp2, cachedTimeValuePair3.getTimestamp()); - Assert.assertEquals(value3, cachedTimeValuePair3.getValue()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - } - private ISchemaTree generateSchemaTree1() throws IllegalPathException { - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - Map s1TagMap = new HashMap<>(); + final ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + final Map s1TagMap = new HashMap<>(); s1TagMap.put("k1", "v1"); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s1"), @@ -227,7 +163,7 @@ private ISchemaTree generateSchemaTree1() throws IllegalPathException { } private ISchemaTree generateSchemaTree2() throws IllegalPathException { - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + final ClusterSchemaTree schemaTree = new ClusterSchemaTree(); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s3"), @@ -256,58 +192,85 @@ private ISchemaTree generateSchemaTree2() throws IllegalPathException { @Test public void testUpdateLastCache() throws IllegalPathException { - String database = "root.db"; - PartialPath device = new PartialPath("root.db.d"); + final String database = "root.db"; + final PartialPath device = new PartialPath("root.db.d"); + + final String[] measurements = new String[] {"s1", "s2", "s3"}; - String[] measurements = new String[] {"s1", "s2", "s3"}; - MeasurementSchema[] measurementSchemas = - new MeasurementSchema[] { - new MeasurementSchema("s1", TSDataType.INT32), - new MeasurementSchema("s2", TSDataType.INT32), - new MeasurementSchema("s3", TSDataType.INT32) - }; + final MeasurementSchema s1 = new MeasurementSchema("s1", TSDataType.INT32); + final MeasurementSchema s2 = new MeasurementSchema("s2", TSDataType.INT32); + final MeasurementSchema s3 = new MeasurementSchema("s3", TSDataType.INT32); + final TimeValuePair tv1 = new TimeValuePair(1, new TsPrimitiveType.TsInt(1)); + + dataNodeSchemaCache.updateLastCache( + database, new MeasurementPath(device.concatNode("s1"), s1), false); dataNodeSchemaCache.updateLastCache( + database, new MeasurementPath(device.concatNode("s3"), s3), false); + + // Simulate "s1" revert when the query has failed in calculation + dataNodeSchemaCache.updateLastCacheIfExists( database, - device, - measurements, - measurementSchemas, - true, - index -> new TimeValuePair(1, new TsPrimitiveType.TsInt(1)), - index -> index != 1, - true, - 1L); + IDeviceID.Factory.DEFAULT_FACTORY.create( + StringArrayDeviceID.splitDeviceIdString(device.getNodes())), + new String[] {"s1"}, + new TimeValuePair[] { + new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), + }, + false, + new MeasurementSchema[] {s1}); + dataNodeSchemaCache.updateLastCache( + database, new MeasurementPath(device.concatNode("s1"), s1), true); + + // "s2" shall be null since the "null" timeValuePair has not been put + dataNodeSchemaCache.updateLastCacheIfExists( + database, + IDeviceID.Factory.DEFAULT_FACTORY.create( + StringArrayDeviceID.splitDeviceIdString(device.getNodes())), + new String[] {"s2"}, + new TimeValuePair[] {tv1}, + false, + new MeasurementSchema[] {s2}); - Assert.assertNotNull(dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s1"))); - Assert.assertNull(dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s2"))); - Assert.assertNotNull(dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s3"))); + dataNodeSchemaCache.updateLastCacheIfExists( + database, + IDeviceID.Factory.DEFAULT_FACTORY.create( + StringArrayDeviceID.splitDeviceIdString(device.getNodes())), + new String[] {"s3"}, + new TimeValuePair[] {tv1}, + false, + new MeasurementSchema[] {s3}); - dataNodeSchemaCache.updateLastCache( + Assert.assertNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s1"))); + Assert.assertNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s2"))); + Assert.assertNotNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s3"))); + + final MeasurementSchema[] measurementSchemas = new MeasurementSchema[] {s1, s2, s3}; + + dataNodeSchemaCache.updateLastCacheIfExists( database, - device, + IDeviceID.Factory.DEFAULT_FACTORY.create( + StringArrayDeviceID.splitDeviceIdString(device.getNodes())), measurements, - measurementSchemas, - true, - index -> new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), - index -> true, - true, - 1L); + new TimeValuePair[] { + new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), + new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), + new TimeValuePair(2, new TsPrimitiveType.TsInt(2)) + }, + false, + measurementSchemas); + Assert.assertNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s1"))); + Assert.assertNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s2"))); Assert.assertEquals( new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), - dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s1"))); - Assert.assertEquals( - new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), - dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s2"))); - Assert.assertEquals( - new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), - dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s3"))); + dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s3"))); } @Test public void testPut() throws Exception { - ClusterSchemaTree clusterSchemaTree = new ClusterSchemaTree(); - Template template1 = + final ClusterSchemaTree clusterSchemaTree = new ClusterSchemaTree(); + final Template template1 = new Template( "t1", Arrays.asList("s1", "s2"), @@ -315,7 +278,7 @@ public void testPut() throws Exception { Arrays.asList(TSEncoding.RLE, TSEncoding.RLE), Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY)); template1.setId(1); - Template template2 = + final Template template2 = new Template( "t2", Arrays.asList("t1", "t2", "t3"), @@ -331,22 +294,22 @@ public void testPut() throws Exception { clusterSchemaTree.appendSingleMeasurementPath( new MeasurementPath("root.sg1.d3.s1", TSDataType.FLOAT)); dataNodeSchemaCache.put(clusterSchemaTree); - ClusterSchemaTree d1Tree = + final ClusterSchemaTree d1Tree = dataNodeSchemaCache.getMatchedSchemaWithTemplate(new PartialPath("root.sg1.d1")); - ClusterSchemaTree d2Tree = + final ClusterSchemaTree d2Tree = dataNodeSchemaCache.getMatchedSchemaWithTemplate(new PartialPath("root.sg1.d2")); - ClusterSchemaTree d3Tree = - dataNodeSchemaCache.getMatchedSchemaWithoutTemplate(new PartialPath("root.sg1.d3.s1")); + final ClusterSchemaTree d3Tree = + dataNodeSchemaCache.getMatchedSchemaWithoutTemplate(new MeasurementPath("root.sg1.d3.s1")); List measurementPaths = d1Tree.searchMeasurementPaths(ALL_MATCH_PATTERN).left; Assert.assertEquals(2, measurementPaths.size()); - for (MeasurementPath measurementPath : measurementPaths) { + for (final MeasurementPath measurementPath : measurementPaths) { Assert.assertEquals( template1.getSchema(measurementPath.getMeasurement()), measurementPath.getMeasurementSchema()); } measurementPaths = d2Tree.searchMeasurementPaths(ALL_MATCH_PATTERN).left; Assert.assertEquals(3, measurementPaths.size()); - for (MeasurementPath measurementPath : measurementPaths) { + for (final MeasurementPath measurementPath : measurementPaths) { Assert.assertEquals( template2.getSchema(measurementPath.getMeasurement()), measurementPath.getMeasurementSchema()); From f40c14e728611e42c3e79f4edafefd391ac49b3d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 16:56:36 +0800 Subject: [PATCH 06/33] fix-clear-cache --- .../impl/DataNodeInternalRPCServiceImpl.java | 12 ++- .../cache/schema/DataNodeSchemaCache.java | 32 ++++---- .../analyze/cache/schema/DeviceLastCache.java | 47 +---------- ...malSchema.java => DeviceNormalSchema.java} | 6 +- .../cache/schema/DeviceSchemaCache.java | 72 +---------------- ...eSchema.java => DeviceTemplateSchema.java} | 6 +- .../cache/schema/TableDeviceCacheEntry.java | 77 ++++--------------- 7 files changed, 51 insertions(+), 201 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/{TreeDeviceNormalSchema.java => DeviceNormalSchema.java} (94%) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/{TreeDeviceTemplateSchema.java => DeviceTemplateSchema.java} (85%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 336d6b0ae729e..9d837b6f8fb09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -546,7 +546,7 @@ public TSStatus invalidateSchemaCache(final TInvalidateCacheReq req) { DataNodeSchemaCache.getInstance().takeWriteLock(); try { // req.getFullPath() is a database path - DataNodeSchemaCache.getInstance().invalidate(req.getFullPath()); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidate(req.getFullPath()); ClusterTemplateManager.getInstance().invalid(req.getFullPath()); LOGGER.info("Schema cache of {} has been invalidated", req.getFullPath()); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); @@ -1974,12 +1974,16 @@ public TSStatus clearCacheImpl(final Set options) { || options.contains(CacheClearOptions.QUERY)) { storageEngine.clearCache(); } - if (options.contains(CacheClearOptions.TREE_SCHEMA)) { - schemaCache.invalidateAll(); + if (options.contains(CacheClearOptions.QUERY) + && options.contains(CacheClearOptions.TREE_SCHEMA)) { + schemaCache.getDeviceSchemaCache().invalidateAll(); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } if (options.contains(CacheClearOptions.QUERY)) { - schemaCache.invalidateLastCache(new PartialPath("root.**")); + schemaCache.getDeviceSchemaCache().invalidateLastCache(); + } + if (options.contains(CacheClearOptions.TREE_SCHEMA)) { + schemaCache.getDeviceSchemaCache().invalidateTreeSchema(); } } catch (final Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 6a98f63f758f9..98801c48ef68b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -103,10 +103,10 @@ public void releaseWriteLock() { public ClusterSchemaTree get(final PartialPath devicePath, final String[] measurements) { final ClusterSchemaTree tree = new ClusterSchemaTree(); final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getNodes()); - if (!(schema instanceof TreeDeviceNormalSchema)) { + if (!(schema instanceof DeviceNormalSchema)) { return tree; } - final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; + final DeviceNormalSchema treeSchema = (DeviceNormalSchema) schema; for (final String measurement : measurements) { final SchemaCacheEntry entry = treeSchema.getSchemaCacheEntry(measurement); if (Objects.nonNull(entry)) { @@ -132,10 +132,10 @@ public ClusterSchemaTree get(final PartialPath devicePath, final String[] measur public ClusterSchemaTree getMatchedSchemaWithTemplate(final PartialPath devicePath) { final ClusterSchemaTree tree = new ClusterSchemaTree(); final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getNodes()); - if (!(schema instanceof TreeDeviceTemplateSchema)) { + if (!(schema instanceof DeviceTemplateSchema)) { return tree; } - final TreeDeviceTemplateSchema treeSchema = (TreeDeviceTemplateSchema) schema; + final DeviceTemplateSchema treeSchema = (DeviceTemplateSchema) schema; Template template = templateManager.getTemplate(treeSchema.getTemplateId()); tree.appendTemplateDevice(devicePath, template.isDirectAligned(), template.getId(), template); tree.setDatabases(Collections.singleton(treeSchema.getDatabase())); @@ -153,10 +153,10 @@ public ClusterSchemaTree getMatchedSchemaWithoutTemplate(final PartialPath fullP final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema( Arrays.copyOf(fullPath.getNodes(), fullPath.getNodeLength() - 1)); - if (!(schema instanceof TreeDeviceNormalSchema)) { + if (!(schema instanceof DeviceNormalSchema)) { return tree; } - final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; + final DeviceNormalSchema treeSchema = (DeviceNormalSchema) schema; final SchemaCacheEntry entry = treeSchema.getSchemaCacheEntry(fullPath.getMeasurement()); if (Objects.isNull(entry)) { return tree; @@ -173,12 +173,12 @@ public List computeWithoutTemplate(final ISchemaComputation schemaCompu final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(schemaComputation.getDevicePath().getNodes()); - if (!(schema instanceof TreeDeviceNormalSchema)) { + if (!(schema instanceof DeviceNormalSchema)) { return IntStream.range(0, schemaComputation.getMeasurements().length) .boxed() .collect(Collectors.toList()); } - final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; + final DeviceNormalSchema treeSchema = (DeviceNormalSchema) schema; for (int i = 0; i < schemaComputation.getMeasurements().length; i++) { final SchemaCacheEntry value = treeSchema.getSchemaCacheEntry(measurements[i]); @@ -231,12 +231,12 @@ public Pair, List> computeSourceOfLogicalView( final PartialPath fullPath = logicalViewSchema.getSourcePathIfWritable(); final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevicePath().getNodes()); - if (!(schema instanceof TreeDeviceNormalSchema)) { + if (!(schema instanceof DeviceNormalSchema)) { indexOfMissingMeasurements.add(i); continue; } - final TreeDeviceNormalSchema treeSchema = (TreeDeviceNormalSchema) schema; + final DeviceNormalSchema treeSchema = (DeviceNormalSchema) schema; final SchemaCacheEntry value = treeSchema.getSchemaCacheEntry(fullPath.getMeasurement()); if (Objects.isNull(value)) { indexOfMissingMeasurements.add(i); @@ -268,11 +268,11 @@ public List computeWithTemplate(final ISchemaComputation computation) { final IDeviceSchema deviceSchema = deviceSchemaCache.getDeviceSchema(computation.getDevicePath().getNodes()); - if (!(deviceSchema instanceof TreeDeviceTemplateSchema)) { + if (!(deviceSchema instanceof DeviceTemplateSchema)) { return IntStream.range(0, measurements.length).boxed().collect(Collectors.toList()); } - final TreeDeviceTemplateSchema deviceTemplateSchema = (TreeDeviceTemplateSchema) deviceSchema; + final DeviceTemplateSchema deviceTemplateSchema = (DeviceTemplateSchema) deviceSchema; computation.computeDevice( templateManager.getTemplate(deviceTemplateSchema.getTemplateId()).isDirectAligned()); @@ -343,8 +343,8 @@ public boolean isLogicalView() { } /** - * Store the fetched schema in either the {@link TreeDeviceNormalSchema} or {@link - * TreeDeviceTemplateSchema}, depending on its associated device. + * Store the fetched schema in either the {@link DeviceNormalSchema} or {@link + * DeviceTemplateSchema}, depending on its associated device. */ public void put(final ClusterSchemaTree tree) { tree.getAllDevices() @@ -449,6 +449,10 @@ public void invalidate(final List partialPathList) { }); } + public DeviceSchemaCache getDeviceSchemaCache() { + return deviceSchemaCache; + } + public void cleanUp() { deviceSchemaCache.invalidateAll(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java index 4cc6ab9369a13..af99bd921dfb5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java @@ -21,7 +21,6 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; @@ -30,11 +29,8 @@ import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; -import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -78,8 +74,6 @@ public TSDataType getDataType() { } }; - private static final Optional> HIT_AND_ALL_NULL = - Optional.of(new Pair<>(OptionalLong.empty(), null)); public static final TimeValuePair EMPTY_TIME_VALUE_PAIR = new TimeValuePair(Long.MIN_VALUE, EMPTY_PRIMITIVE_TYPE); private static final TimeValuePair PLACEHOLDER_TIME_VALUE_PAIR = @@ -165,15 +159,13 @@ int tryUpdate( } @GuardedBy("DataRegionInsertLock#writeLock") - int invalidate(final String measurement, final boolean isTableModel) { + int invalidate(final String measurement) { final AtomicInteger diff = new AtomicInteger(); final AtomicLong time = new AtomicLong(); measurement2CachedLastMap.computeIfPresent( measurement, (s, timeValuePair) -> { - diff.set( - (isTableModel ? 0 : (int) RamUsageEstimator.sizeOf(s)) - + getTVPairEntrySize(timeValuePair)); + diff.set((int) RamUsageEstimator.sizeOf(s) + getTVPairEntrySize(timeValuePair)); time.set(timeValuePair.getTimestamp()); return null; }); @@ -208,41 +200,6 @@ TimeValuePair getTimeValuePair(final @Nonnull String measurement) { return result != PLACEHOLDER_TIME_VALUE_PAIR ? result : null; } - // Shall pass in "" if last by time - Optional> getLastRow( - final @Nonnull String sourceMeasurement, final List targetMeasurements) { - final TimeValuePair pair = measurement2CachedLastMap.get(sourceMeasurement); - if (Objects.isNull(pair) || pair == PLACEHOLDER_TIME_VALUE_PAIR) { - return Optional.empty(); - } - - if (pair == EMPTY_TIME_VALUE_PAIR) { - return HIT_AND_ALL_NULL; - } - final long alignTime = pair.getTimestamp(); - - return Optional.of( - new Pair<>( - OptionalLong.of(alignTime), - targetMeasurements.stream() - .map( - targetMeasurement -> { - if (!targetMeasurement.isEmpty()) { - final TimeValuePair tvPair = - measurement2CachedLastMap.get(targetMeasurement); - if (Objects.isNull(tvPair)) { - return null; - } - return tvPair.getTimestamp() == alignTime - ? tvPair.getValue() - : EMPTY_PRIMITIVE_TYPE; - } else { - return new TsPrimitiveType.TsLong(alignTime); - } - }) - .toArray(TsPrimitiveType[]::new))); - } - int estimateSize() { return INSTANCE_SIZE + (int) RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY * measurement2CachedLastMap.size() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceNormalSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceNormalSchema.java similarity index 94% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceNormalSchema.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceNormalSchema.java index d944f5f483933..cfad9c9e45e05 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceNormalSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceNormalSchema.java @@ -30,17 +30,17 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -public class TreeDeviceNormalSchema implements IDeviceSchema { +public class DeviceNormalSchema implements IDeviceSchema { static final int INSTANCE_SIZE = - (int) RamUsageEstimator.shallowSizeOfInstance(TreeDeviceTemplateSchema.class) + (int) RamUsageEstimator.shallowSizeOfInstance(DeviceNormalSchema.class) + (int) RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); private final String database; private final boolean isAligned; private final ConcurrentMap measurementMap = new ConcurrentHashMap<>(); - public TreeDeviceNormalSchema(final String database, final boolean isAligned) { + public DeviceNormalSchema(final String database, final boolean isAligned) { this.database = database; this.isAligned = isAligned; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index 5c861a3320827..deb292f21acde 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.commons.path.ExtendedPartialPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.service.metric.MetricService; @@ -46,16 +45,11 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; -import java.util.Arrays; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.ToIntFunction; -import java.util.stream.Collectors; - -import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; /** * The {@link DeviceSchemaCache} caches some of the devices and their: attributes of tables / @@ -126,17 +120,6 @@ public TimeValuePair getLastEntry( return Objects.nonNull(entry) ? entry.getTimeValuePair(measurement) : null; } - /** - * Invalidate the last cache of one table. - * - * @param database the device's database, without "root" - * @param table tableName - */ - public void invalidateLastCache(final String database, final String table) { - dualKeyCache.update( - new TableId(database, table), deviceId -> true, entry -> -entry.invalidateLastCache()); - } - /** * Invalidate the last cache of one device. * @@ -219,7 +202,7 @@ void invalidateLastCache(final PartialPath devicePath, final String measurement) final ToIntFunction updateFunction = PathPatternUtil.hasWildcard(measurement) ? entry -> -entry.invalidateLastCache() - : entry -> -entry.invalidateLastCache(measurement, false); + : entry -> -entry.invalidateLastCache(measurement); if (!devicePath.hasWildcard()) { final IDeviceID deviceID = devicePath.getIDeviceID(); @@ -353,56 +336,6 @@ public void invalidate(final @Nonnull String database) { } } - // The fuzzy filters are not considered because: - // 1. We can actually invalidate more cache entries than we need. - // 2. Constructing the filterOperators may require some time and complication - // 3. The fuzzy filters may contain attributes, which may not exist or be stale - public void invalidate( - final String database, final String tableName, final List patterns) { - readWriteLock.writeLock().lock(); - try { - final TableId firstKey = new TableId(database, tableName); - if (patterns.isEmpty()) { - dualKeyCache.invalidate(firstKey); - } else { - final List multiMatchList = - patterns.stream() - .filter( - idFilter -> { - if (!idFilter.hasWildcard()) { - final IDeviceID deviceId = - IDeviceID.Factory.DEFAULT_FACTORY.create( - Arrays.copyOfRange( - idFilter.getNodes(), 2, idFilter.getNodeLength())); - dualKeyCache.invalidate(firstKey, deviceId); - return false; - } - return true; - }) - .collect(Collectors.toList()); - - dualKeyCache.invalidate( - firstKey, - deviceId -> { - final String[] segments = (String[]) deviceId.getSegments(); - for (int i = 1; i < segments.length; ++i) { - for (final PartialPath path : multiMatchList) { - final int pathIndex = i + 2; - if (path.getNodes()[pathIndex].equals(segments[i]) - || path.getNodes()[pathIndex].equals(ONE_LEVEL_PATH_WILDCARD) - && ((ExtendedPartialPath) path).match(pathIndex, segments[i])) { - return true; - } - } - } - return false; - }); - } - } finally { - readWriteLock.writeLock().unlock(); - } - } - public void invalidateLastCache() { readWriteLock.writeLock().lock(); try { @@ -415,8 +348,7 @@ public void invalidateLastCache() { public void invalidateTreeSchema() { readWriteLock.writeLock().lock(); try { - dualKeyCache.update( - tableId -> true, deviceID -> true, entry -> -entry.invalidateTreeSchema()); + dualKeyCache.update(tableId -> true, deviceID -> true, entry -> -entry.invalidateSchema()); } finally { readWriteLock.writeLock().unlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceTemplateSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceTemplateSchema.java similarity index 85% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceTemplateSchema.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceTemplateSchema.java index 583919dd5c12f..5a9b054bd7cad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TreeDeviceTemplateSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceTemplateSchema.java @@ -21,14 +21,14 @@ import org.apache.tsfile.utils.RamUsageEstimator; -public class TreeDeviceTemplateSchema implements IDeviceSchema { +public class DeviceTemplateSchema implements IDeviceSchema { static final int INSTANCE_SIZE = - (int) RamUsageEstimator.shallowSizeOfInstance(TreeDeviceTemplateSchema.class); + (int) RamUsageEstimator.shallowSizeOfInstance(DeviceTemplateSchema.class); private final String database; private final int templateId; - TreeDeviceTemplateSchema(final String database, final int templateId) { + DeviceTemplateSchema(final String database, final int templateId) { this.database = database; this.templateId = templateId; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java index a5f375d37bddd..6468062007d6f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java @@ -25,17 +25,13 @@ import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.schema.IMeasurementSchema; import javax.annotation.Nonnull; import javax.annotation.concurrent.ThreadSafe; -import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -56,53 +52,23 @@ public class TableDeviceCacheEntry { private final AtomicReference deviceSchema = new AtomicReference<>(); private final AtomicReference lastCache = new AtomicReference<>(); - /////////////////////////////// Attribute /////////////////////////////// - - int invalidateAttribute() { - final AtomicInteger size = new AtomicInteger(0); - deviceSchema.updateAndGet( - schema -> { - if (schema instanceof TableAttributeSchema) { - size.set(schema.estimateSize()); - return null; - } - return schema; - }); - return size.get(); - } - - int invalidateAttributeColumn(final String attribute) { - final AtomicInteger size = new AtomicInteger(0); - deviceSchema.updateAndGet( - schema -> { - if (schema instanceof TableAttributeSchema) { - size.set(((TableAttributeSchema) schema).removeAttribute(attribute)); - return schema; - } - return schema; - }); - return size.get(); - } - - /////////////////////////////// Tree model /////////////////////////////// - int setDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { // Safe here because tree schema is invalidated by the whole entry if (deviceSchemaInfo.getTemplateId() == NON_TEMPLATE) { final int result = (deviceSchema.compareAndSet( - null, new TreeDeviceNormalSchema(database, deviceSchemaInfo.isAligned())) - ? TreeDeviceNormalSchema.INSTANCE_SIZE + null, new DeviceNormalSchema(database, deviceSchemaInfo.isAligned())) + ? DeviceNormalSchema.INSTANCE_SIZE : 0); - return deviceSchema.get() instanceof TreeDeviceNormalSchema + return deviceSchema.get() instanceof DeviceNormalSchema ? result - + ((TreeDeviceNormalSchema) deviceSchema.get()) + + ((DeviceNormalSchema) deviceSchema.get()) .update(deviceSchemaInfo.getMeasurementSchemaInfoList()) : 0; } else { return deviceSchema.compareAndSet( - null, new TreeDeviceTemplateSchema(database, deviceSchemaInfo.getTemplateId())) - ? TreeDeviceTemplateSchema.INSTANCE_SIZE + null, new DeviceTemplateSchema(database, deviceSchemaInfo.getTemplateId())) + ? DeviceTemplateSchema.INSTANCE_SIZE : 0; } } @@ -115,13 +81,13 @@ int setMeasurementSchema( if (schemas == null) { return 0; } - // Safe here because tree schema is invalidated by the whole entry + // Safe here because schema is invalidated by the whole entry final int result = - (deviceSchema.compareAndSet(null, new TreeDeviceNormalSchema(database, isAligned)) - ? TreeDeviceNormalSchema.INSTANCE_SIZE + (deviceSchema.compareAndSet(null, new DeviceNormalSchema(database, isAligned)) + ? DeviceNormalSchema.INSTANCE_SIZE : 0); - return deviceSchema.get() instanceof TreeDeviceNormalSchema - ? result + ((TreeDeviceNormalSchema) deviceSchema.get()).update(measurements, schemas) + return deviceSchema.get() instanceof DeviceNormalSchema + ? result + ((DeviceNormalSchema) deviceSchema.get()).update(measurements, schemas) : 0; } @@ -129,15 +95,11 @@ IDeviceSchema getDeviceSchema() { return deviceSchema.get(); } - int invalidateTreeSchema() { + int invalidateSchema() { final AtomicInteger size = new AtomicInteger(0); deviceSchema.updateAndGet( schema -> { - if (schema instanceof TreeDeviceNormalSchema - || schema instanceof TreeDeviceTemplateSchema) { - size.set(schema.estimateSize()); - return null; - } + size.set(schema.estimateSize()); return schema; }); return size.get(); @@ -167,9 +129,9 @@ int tryUpdateLastCache(final String[] measurements, final TimeValuePair[] timeVa return tryUpdateLastCache(measurements, timeValuePairs, false); } - int invalidateLastCache(final String measurement, final boolean isTableModel) { + int invalidateLastCache(final String measurement) { final DeviceLastCache cache = lastCache.get(); - final int result = Objects.nonNull(cache) ? cache.invalidate(measurement, isTableModel) : 0; + final int result = Objects.nonNull(cache) ? cache.invalidate(measurement) : 0; return Objects.nonNull(lastCache.get()) ? result : 0; } @@ -190,15 +152,6 @@ boolean updateInputMap(final @Nonnull Map> getLastRow( - final String sourceMeasurement, final List targetMeasurements) { - final DeviceLastCache cache = lastCache.get(); - return Objects.nonNull(cache) - ? cache.getLastRow(sourceMeasurement, targetMeasurements) - : Optional.empty(); - } - int invalidateLastCache() { final AtomicInteger size = new AtomicInteger(0); lastCache.updateAndGet( From 2f3bb363bc872c2e6af851a1f0d7531a0ae33a58 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 18:08:11 +0800 Subject: [PATCH 07/33] first-cleaning --- .../thrift/impl/ClientRPCServiceImpl.java | 2 +- .../cache/schema/DataNodeSchemaCache.java | 33 ++-- .../analyze/cache/schema/DeviceLastCache.java | 8 +- .../cache/schema/DeviceSchemaCache.java | 151 +++++++++--------- .../cache/schema/TableDeviceCacheEntry.java | 6 +- .../storageengine/dataregion/DataRegion.java | 24 ++- 6 files changed, 102 insertions(+), 122 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 64f5c0209fade..4803023531c3d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -830,7 +830,7 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath( } // 2.DATA_NODE_SCHEMA_CACHE.getLastCache() - if (!DataNodeSchemaCache.getInstance().getLastCache(resultMap)) { + if (!DataNodeSchemaCache.getInstance().getDeviceSchemaCache().getLastCache(resultMap)) { // 2.1 any sensor miss cache, construct last query sql, then return return executeLastDataQueryInternal(convert(req), SELECT_RESULT); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 98801c48ef68b..163e4944d0e33 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -32,7 +32,6 @@ import org.apache.iotdb.db.schemaengine.template.ITemplateManager; import org.apache.iotdb.db.schemaengine.template.Template; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.utils.Pair; @@ -42,7 +41,6 @@ import javax.annotation.Nonnull; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -102,7 +100,7 @@ public void releaseWriteLock() { */ public ClusterSchemaTree get(final PartialPath devicePath, final String[] measurements) { final ClusterSchemaTree tree = new ClusterSchemaTree(); - final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getNodes()); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getFullPath()); if (!(schema instanceof DeviceNormalSchema)) { return tree; } @@ -131,7 +129,7 @@ public ClusterSchemaTree get(final PartialPath devicePath, final String[] measur */ public ClusterSchemaTree getMatchedSchemaWithTemplate(final PartialPath devicePath) { final ClusterSchemaTree tree = new ClusterSchemaTree(); - final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getNodes()); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getFullPath()); if (!(schema instanceof DeviceTemplateSchema)) { return tree; } @@ -150,9 +148,7 @@ public ClusterSchemaTree getMatchedSchemaWithTemplate(final PartialPath devicePa */ public ClusterSchemaTree getMatchedSchemaWithoutTemplate(final PartialPath fullPath) { final ClusterSchemaTree tree = new ClusterSchemaTree(); - final IDeviceSchema schema = - deviceSchemaCache.getDeviceSchema( - Arrays.copyOf(fullPath.getNodes(), fullPath.getNodeLength() - 1)); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevice()); if (!(schema instanceof DeviceNormalSchema)) { return tree; } @@ -172,7 +168,7 @@ public List computeWithoutTemplate(final ISchemaComputation schemaCompu final String[] measurements = schemaComputation.getMeasurements(); final IDeviceSchema schema = - deviceSchemaCache.getDeviceSchema(schemaComputation.getDevicePath().getNodes()); + deviceSchemaCache.getDeviceSchema(schemaComputation.getDevicePath().getFullPath()); if (!(schema instanceof DeviceNormalSchema)) { return IntStream.range(0, schemaComputation.getMeasurements().length) .boxed() @@ -229,8 +225,7 @@ public Pair, List> computeSourceOfLogicalView( } final PartialPath fullPath = logicalViewSchema.getSourcePathIfWritable(); - final IDeviceSchema schema = - deviceSchemaCache.getDeviceSchema(fullPath.getDevicePath().getNodes()); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevice()); if (!(schema instanceof DeviceNormalSchema)) { indexOfMissingMeasurements.add(i); continue; @@ -266,7 +261,7 @@ public List computeWithTemplate(final ISchemaComputation computation) { final List indexOfMissingMeasurements = new ArrayList<>(); final String[] measurements = computation.getMeasurements(); final IDeviceSchema deviceSchema = - deviceSchemaCache.getDeviceSchema(computation.getDevicePath().getNodes()); + deviceSchemaCache.getDeviceSchema(computation.getDevicePath().getFullPath()); if (!(deviceSchema instanceof DeviceTemplateSchema)) { return IntStream.range(0, measurements.length).boxed().collect(Collectors.toList()); @@ -354,17 +349,11 @@ public void put(final ClusterSchemaTree tree) { tree.getBelongedDatabase(deviceSchemaInfo.getDevicePath()), deviceSchemaInfo)); } - public boolean getLastCache( - final Map>>> inputMap) { - return deviceSchemaCache.getLastCache(inputMap); - } - public TimeValuePair getLastCache(final PartialPath seriesPath) { - return deviceSchemaCache.getLastEntry( - null, seriesPath.getIDeviceID(), seriesPath.getMeasurement()); + return deviceSchemaCache.getLastEntry(seriesPath.getDevice(), seriesPath.getMeasurement()); } - public void invalidateLastCache(final MeasurementPath path) { + public void invalidateLastCache(final PartialPath path) { if (!CommonDescriptor.getInstance().getConfig().isLastCacheEnable()) { return; } @@ -391,7 +380,7 @@ public void invalidateDatabaseLastCache(final String database) { */ public void updateLastCacheIfExists( final String database, - final IDeviceID deviceID, + final String deviceID, final String[] measurements, final @Nonnull TimeValuePair[] timeValuePairs, final boolean isAligned, @@ -411,7 +400,7 @@ public void updateLastCacheIfExists( * stale value is written, but it won't affect the eventual consistency. * *

- Second time put the calculated {@link TimeValuePair}, and use {@link - * #updateLastCacheIfExists(String, IDeviceID, String[], TimeValuePair[], boolean, + * #updateLastCacheIfExists(String, String, String[], TimeValuePair[], boolean, * IMeasurementSchema[])}. The input {@link TimeValuePair} shall never be or contain {@code null}, * if the measurement is with all {@code null}s, its {@link TimeValuePair} shall be {@link * DeviceLastCache#EMPTY_TIME_VALUE_PAIR}. This method is not supposed to update time column. @@ -429,7 +418,7 @@ public void updateLastCache( final String database, final MeasurementPath measurementPath, final boolean isInvalidate) { deviceSchemaCache.updateLastCache( database, - measurementPath.getIDeviceID(), + measurementPath.getDevice(), new String[] {measurementPath.getMeasurement()}, isInvalidate ? new TimeValuePair[] {null} : null, measurementPath.isUnderAlignedEntity(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java index af99bd921dfb5..8d3d7e754a23b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java @@ -82,8 +82,7 @@ public TSDataType getDataType() { // Time is seen as "" as a measurement private final Map measurement2CachedLastMap = new ConcurrentHashMap<>(); - int initOrInvalidate( - final String tableName, final String[] measurements, final boolean isInvalidate) { + int initOrInvalidate(final String[] measurements, final boolean isInvalidate) { final AtomicInteger diff = new AtomicInteger(0); for (final String measurement : measurements) { @@ -98,13 +97,12 @@ int initOrInvalidate( (measurementKey, tvPair) -> { if (Objects.isNull(newPair)) { diff.addAndGet( - -((false ? 0 : (int) RamUsageEstimator.sizeOf(measurement)) - + getTVPairEntrySize(tvPair))); + -((int) RamUsageEstimator.sizeOf(measurement) + getTVPairEntrySize(tvPair))); return null; } if (Objects.isNull(tvPair)) { diff.addAndGet( - (false ? 0 : (int) RamUsageEstimator.sizeOf(measurement)) + (int) RamUsageEstimator.sizeOf(measurement) + (int) RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY); return newPair; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index deb292f21acde..ca1487842f171 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -29,14 +29,14 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; -import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegion; +import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +56,7 @@ * measurement info / template info. The last value of one device is also cached here. Here are the * semantics of attributes, other semantics are omitted: * - *

1. If a deviceId misses cache, it does not necessarily mean that the device does not exist, + *

1. If a device misses cache, it does not necessarily mean that the device does not exist, * Since the cache records only part of the devices. * *

2. If a device is in cache, the attributes will be finally identical to the {@link @@ -76,14 +76,10 @@ public class DeviceSchemaCache { private static final Logger logger = LoggerFactory.getLogger(DeviceSchemaCache.class); /** - * In table model: {@literal <}{@link QualifiedObjectName}, {@link IDeviceID}, lastCache / - * attributes{@literal >} - * - *

In tree model: {@literal <}Pair{@literal <}{@code null}, tableName(translated){@literal >}, - * {@link IDeviceID}(translated), Map{@literal <}Measurement, Schema{@literal + * Leading_Segment, {@link IDeviceID}, Map{@literal <}Measurement, Schema{@literal * >}/templateInfo{@literal >} */ - private final IDualKeyCache dualKeyCache; + private final IDualKeyCache dualKeyCache; private final Map databasePool = new ConcurrentHashMap<>(); @@ -91,12 +87,12 @@ public class DeviceSchemaCache { DeviceSchemaCache() { dualKeyCache = - new DualKeyCacheBuilder() + new DualKeyCacheBuilder() .cacheEvictionPolicy( DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) .memoryCapacity(config.getAllocateMemoryForSchemaCache()) - .firstKeySizeComputer(TableId::estimateSize) - .secondKeySizeComputer(deviceID -> (int) deviceID.ramBytesUsed()) + .firstKeySizeComputer(segment -> (int) RamUsageEstimator.sizeOf(segment)) + .secondKeySizeComputer(device -> (int) RamUsageEstimator.sizeOf(device)) .valueSizeComputer(TableDeviceCacheEntry::estimateSize) .build(); MetricService.getInstance().addMetricSet(new DataNodeSchemaCacheMetrics(this)); @@ -107,44 +103,36 @@ public class DeviceSchemaCache { /** * Get the last {@link TimeValuePair} of a measurement, the measurement shall never be "time". * - * @param database the device's database, without "root", {@code null} for tree model - * @param deviceId {@link IDeviceID} + * @param device {@link IDeviceID} * @param measurement the measurement to get * @return {@code null} iff cache miss, {@link DeviceLastCache#EMPTY_TIME_VALUE_PAIR} iff cache * hit but result is {@code null}, and the result value otherwise. */ - public TimeValuePair getLastEntry( - final @Nullable String database, final IDeviceID deviceId, final String measurement) { - final TableDeviceCacheEntry entry = - dualKeyCache.get(new TableId(database, deviceId.getTableName()), deviceId); + public TimeValuePair getLastEntry(final String device, final String measurement) { + final TableDeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); return Objects.nonNull(entry) ? entry.getTimeValuePair(measurement) : null; } /** * Invalidate the last cache of one device. * - * @param database the device's database, without "root" - * @param deviceId IDeviceID + * @param device IDeviceID */ - public void invalidateLastCache(final String database, final IDeviceID deviceId) { + public void invalidateDeviceLastCache(final String device) { dualKeyCache.update( - new TableId(database, deviceId.getTableName()), - deviceId, - null, - entry -> -entry.invalidateLastCache(), - false); + getLeadingSegment(device), device, null, entry -> -entry.invalidateLastCache(), false); } /////////////////////////////// Tree model /////////////////////////////// public void putDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { final PartialPath devicePath = deviceSchemaInfo.getDevicePath(); - final IDeviceID deviceID = devicePath.getIDeviceID(); + final String device = devicePath.getFullPath(); final String previousDatabase = databasePool.putIfAbsent(database, database); dualKeyCache.update( - new TableId(null, deviceID.getTableName()), - deviceID, + getLeadingSegment(device), + device, new TableDeviceCacheEntry(), entry -> entry.setDeviceSchema( @@ -152,21 +140,14 @@ public void putDeviceSchema(final String database, final DeviceSchemaInfo device true); } - public IDeviceSchema getDeviceSchema(final String[] devicePath) { - return getDeviceSchema( - IDeviceID.Factory.DEFAULT_FACTORY.create( - StringArrayDeviceID.splitDeviceIdString(devicePath))); - } - - public IDeviceSchema getDeviceSchema(final IDeviceID deviceID) { - final TableDeviceCacheEntry entry = - dualKeyCache.get(new TableId(null, deviceID.getTableName()), deviceID); + public IDeviceSchema getDeviceSchema(final String device) { + final TableDeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); return Objects.nonNull(entry) ? entry.getDeviceSchema() : null; } void updateLastCache( final String database, - final IDeviceID deviceID, + final String device, final String[] measurements, final @Nullable TimeValuePair[] timeValuePairs, final boolean isAligned, @@ -176,15 +157,14 @@ void updateLastCache( final String database2Use = Objects.nonNull(previousDatabase) ? previousDatabase : database; dualKeyCache.update( - new TableId(null, deviceID.getTableName()), - deviceID, + getLeadingSegment(device), + device, new TableDeviceCacheEntry(), initOrInvalidate ? entry -> entry.setMeasurementSchema( database2Use, isAligned, measurements, measurementSchemas) - + entry.initOrInvalidateLastCache( - deviceID.getTableName(), measurements, Objects.nonNull(timeValuePairs)) + + entry.initOrInvalidateLastCache(measurements, Objects.nonNull(timeValuePairs)) : entry -> entry.setMeasurementSchema( database2Use, isAligned, measurements, measurementSchemas) @@ -193,7 +173,7 @@ void updateLastCache( } public boolean getLastCache( - final Map>>> inputMap) { + final Map>>> inputMap) { return dualKeyCache.batchApply(inputMap, TableDeviceCacheEntry::updateInputMap); } @@ -205,22 +185,21 @@ void invalidateLastCache(final PartialPath devicePath, final String measurement) : entry -> -entry.invalidateLastCache(measurement); if (!devicePath.hasWildcard()) { - final IDeviceID deviceID = devicePath.getIDeviceID(); - dualKeyCache.update( - new TableId(null, deviceID.getTableName()), deviceID, null, updateFunction, false); + final String device = devicePath.getFullPath(); + dualKeyCache.update(getLeadingSegment(device), device, null, updateFunction, false); } else { // This may take quite a long time to perform, yet we assume that the "invalidateLastCache" is // only called by deletions, which has a low frequency; and it has avoided that // the un-related paths being cleared, like "root.*.b.c.**" affects // "root.*.d.c.**", thereby lower the query performance. dualKeyCache.update( - tableId -> { + segment -> { try { - return devicePath.matchPrefixPath(new PartialPath(tableId.getTableName())); + return devicePath.matchPrefixPath(new PartialPath(segment)); } catch (final IllegalPathException e) { logger.warn( - "Illegal tableID {} found in cache when invalidating by path {}, invalidate it anyway", - tableId.getTableName(), + "Illegal segmentID {} found in cache when invalidating by path {}, invalidate it anyway", + segment, devicePath); return true; } @@ -230,7 +209,7 @@ void invalidateLastCache(final PartialPath devicePath, final String measurement) return new PartialPath(cachedDeviceID).matchFullPath(devicePath); } catch (final IllegalPathException e) { logger.warn( - "Illegal deviceID {} found in cache when invalidating by path {}, invalidate it anyway", + "Illegal device {} found in cache when invalidating by path {}, invalidate it anyway", cachedDeviceID, devicePath); return true; @@ -244,21 +223,21 @@ void invalidateLastCache(final PartialPath devicePath, final String measurement) void invalidateCache( final @Nonnull PartialPath devicePath, final boolean isMultiLevelWildcardMeasurement) { if (!devicePath.hasWildcard()) { - final IDeviceID deviceID = devicePath.getIDeviceID(); - dualKeyCache.invalidate(new TableId(null, deviceID.getTableName()), deviceID); + final String device = devicePath.getFullPath(); + dualKeyCache.invalidate(getLeadingSegment(device), device); } else { // This may take quite a long time to perform, yet we assume that the "invalidateLastCache" is // only called by deletions, which has a low frequency; and it has avoided that // the un-related paths being cleared, like "root.*.b.c.**" affects // "root.*.d.c.**", thereby lower the query performance. dualKeyCache.invalidate( - tableId -> { + segment -> { try { - return devicePath.matchPrefixPath(new PartialPath(tableId.getTableName())); + return devicePath.matchPrefixPath(new PartialPath(segment)); } catch (final IllegalPathException e) { logger.warn( - "Illegal tableID {} found in cache when invalidating by path {}, invalidate it anyway", - tableId.getTableName(), + "Illegal segmentID {} found in cache when invalidating by path {}, invalidate it anyway", + segment, devicePath); return true; } @@ -270,7 +249,7 @@ void invalidateCache( : devicePath.matchFullPath(new PartialPath(cachedDeviceID)); } catch (final IllegalPathException e) { logger.warn( - "Illegal deviceID {} found in cache when invalidating by path {}, invalidate it anyway", + "Illegal device {} found in cache when invalidating by path {}, invalidate it anyway", cachedDeviceID, devicePath); return true; @@ -301,36 +280,30 @@ long entriesCount() { return dualKeyCache.stats().entriesCount(); } + // Note: It might be very slow if the database is too long void invalidateLastCache(final @Nonnull String database) { readWriteLock.writeLock().lock(); try { dualKeyCache.update( - tableId -> - Objects.isNull(tableId.getDatabase()) && tableId.getTableName().startsWith(database), - deviceID -> true, + segment -> segment.startsWith(database), + device -> true, entry -> -entry.invalidateLastCache()); dualKeyCache.update( - tableId -> - Objects.isNull(tableId.getDatabase()) && database.startsWith(tableId.getTableName()), - deviceID -> deviceID.matchDatabaseName(database), + database::startsWith, + device -> device.startsWith(database), entry -> -entry.invalidateLastCache()); } finally { readWriteLock.writeLock().unlock(); } } + // Note: It might be very slow if the database is too long public void invalidate(final @Nonnull String database) { readWriteLock.writeLock().lock(); try { - dualKeyCache.invalidate( - tableId -> - Objects.isNull(tableId.getDatabase()) && tableId.getTableName().startsWith(database), - deviceID -> true); - dualKeyCache.invalidate( - tableId -> - Objects.isNull(tableId.getDatabase()) && database.startsWith(tableId.getTableName()), - deviceID -> deviceID.matchDatabaseName(database)); + dualKeyCache.invalidate(segment -> segment.startsWith(database), device -> true); + dualKeyCache.invalidate(database::startsWith, device -> device.startsWith(database)); } finally { readWriteLock.writeLock().unlock(); } @@ -339,7 +312,7 @@ public void invalidate(final @Nonnull String database) { public void invalidateLastCache() { readWriteLock.writeLock().lock(); try { - dualKeyCache.update(tableId -> true, deviceID -> true, entry -> -entry.invalidateLastCache()); + dualKeyCache.update(segment -> true, device -> true, entry -> -entry.invalidateLastCache()); } finally { readWriteLock.writeLock().unlock(); } @@ -348,7 +321,7 @@ public void invalidateLastCache() { public void invalidateTreeSchema() { readWriteLock.writeLock().lock(); try { - dualKeyCache.update(tableId -> true, deviceID -> true, entry -> -entry.invalidateSchema()); + dualKeyCache.update(segment -> true, device -> true, entry -> -entry.invalidateSchema()); } finally { readWriteLock.writeLock().unlock(); } @@ -362,4 +335,32 @@ public void invalidateAll() { readWriteLock.writeLock().unlock(); } } + + // Utils of leading segment + + private static String getLeadingSegment(final String device) { + final String segment; + int lastSeparatorPos = -1; + int separatorNum = 0; + + for (int i = 0; i < device.length(); i++) { + if (device.charAt(i) == TsFileConstant.PATH_SEPARATOR_CHAR) { + lastSeparatorPos = i; + separatorNum++; + if (separatorNum == 3) { + break; + } + } + } + if (lastSeparatorPos == -1) { + // not find even one separator, probably during a test, use the device as the tableName + segment = device; + } else { + // use the first DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME segments or all segments but the last + // one as the table name + segment = device.substring(0, lastSeparatorPos); + } + + return segment; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java index 6468062007d6f..580406a89a786 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java @@ -107,13 +107,11 @@ int invalidateSchema() { /////////////////////////////// Last Cache /////////////////////////////// - int initOrInvalidateLastCache( - final String tableName, final String[] measurements, final boolean isInvalidate) { + int initOrInvalidateLastCache(final String[] measurements, final boolean isInvalidate) { int result = lastCache.compareAndSet(null, new DeviceLastCache()) ? DeviceLastCache.INSTANCE_SIZE : 0; final DeviceLastCache cache = lastCache.get(); - result += - Objects.nonNull(cache) ? cache.initOrInvalidate(tableName, measurements, isInvalidate) : 0; + result += Objects.nonNull(cache) ? cache.initOrInvalidate(measurements, isInvalidate) : 0; return Objects.nonNull(lastCache.get()) ? result : 0; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index c1518fe5dadbc..90870c22018ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3255,32 +3255,26 @@ private void onTsFileLoaded( // The inner cache is shared by TreeDeviceSchemaCacheManager and // TableDeviceSchemaCacheManager, // so cleaning either of them is enough - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); break; case CLEAN_DEVICE: ITimeIndex timeIndex = newTsFileResource.getTimeIndex(); if (timeIndex instanceof DeviceTimeIndex) { DeviceTimeIndex deviceTimeIndex = (DeviceTimeIndex) timeIndex; for (IDeviceID deviceID : deviceTimeIndex.getDevices()) { - try { - DataNodeSchemaCache.getInstance() - .invalidateLastCache(new PartialPath(deviceID, "**")); - } catch (IllegalPathException e) { - logger.error( - "Failed to construct path for invalidating last cache of {}", deviceID, e); - DataNodeSchemaCache.getInstance().invalidateAll(); - return; - } + DataNodeSchemaCache.getInstance() + .getDeviceSchemaCache() + .invalidateDeviceLastCache(((PlainDeviceID) deviceID).toStringID()); } } else { - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); } break; default: logger.warn( "Unrecognized LastCacheLoadStrategy: {}, fall back to CLEAN_ALL", IoTDBDescriptor.getInstance().getConfig().getLastCacheLoadStrategy()); - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); break; } } @@ -3312,7 +3306,7 @@ private void updateLastCache(TsFileResource newTsFileResource, TsFileLastReader 0L); } catch (IllegalPathException e) { logger.error("Failed to construct path for invalidating last cache of {}", deviceID, e); - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); return; } } @@ -3342,12 +3336,12 @@ private void updateLastCache(TsFileResource newTsFileResource, TsFileLastReader 0L); } catch (IllegalPathException e) { logger.error("Failed to construct path for invalidating last cache of {}", deviceID, e); - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); return; } } } else { - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); } } From 8874a5c517e5378f60974239408f678e745ed545 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 18:27:19 +0800 Subject: [PATCH 08/33] use-partial --- .../thrift/impl/ClientRPCServiceImpl.java | 30 ++++++---- .../cache/schema/DataNodeSchemaCache.java | 20 +++---- .../cache/schema/DeviceSchemaCache.java | 56 +++++++------------ .../cache/schema/TableDeviceCacheEntry.java | 6 +- .../schemaregion/ISchemaRegion.java | 3 +- .../storageengine/dataregion/DataRegion.java | 13 ++++- 6 files changed, 63 insertions(+), 65 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 4803023531c3d..46e3e79ef2d07 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -817,7 +817,7 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath( "The \"executeFastLastDataQueryForOnePrefixPath\" dos not support wildcards.")); } - final Map> resultMap = new HashMap<>(); + final Map>> resultMap = new HashMap<>(); int sensorNum = 0; final String prefixString = prefixPath.toString(); @@ -838,18 +838,24 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath( // 2.2 all sensors hit cache, return response ~= 20ms final TsBlockBuilder builder = LastQueryUtil.createTsBlockBuilder(sensorNum); - for (final Map.Entry> result : resultMap.entrySet()) { - final String deviceWithPrefix = result.getKey() + TsFileConstant.PATH_SEPARATOR; - for (final Map.Entry measurementLastEntry : + for (final Map.Entry>> result : + resultMap.entrySet()) { + for (final Map.Entry> device2MeasurementLastEntry : result.getValue().entrySet()) { - final TimeValuePair tvPair = measurementLastEntry.getValue(); - LastQueryUtil.appendLastValue( - builder, - tvPair.getTimestamp(), - new Binary( - deviceWithPrefix + measurementLastEntry.getKey(), TSFileConfig.STRING_CHARSET), - tvPair.getValue().getStringValue(), - tvPair.getValue().getDataType().name()); + final String deviceWithSeparator = + device2MeasurementLastEntry.getKey() + TsFileConstant.PATH_SEPARATOR; + for (final Map.Entry measurementLastEntry : + result.getValue().entrySet()) { + final TimeValuePair tvPair = measurementLastEntry.getValue(); + LastQueryUtil.appendLastValue( + builder, + tvPair.getTimestamp(), + new Binary( + deviceWithSeparator + measurementLastEntry.getKey(), + TSFileConfig.STRING_CHARSET), + tvPair.getValue().getStringValue(), + tvPair.getValue().getDataType().name()); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 163e4944d0e33..483aa09df9a05 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -100,7 +100,7 @@ public void releaseWriteLock() { */ public ClusterSchemaTree get(final PartialPath devicePath, final String[] measurements) { final ClusterSchemaTree tree = new ClusterSchemaTree(); - final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getFullPath()); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath); if (!(schema instanceof DeviceNormalSchema)) { return tree; } @@ -129,7 +129,7 @@ public ClusterSchemaTree get(final PartialPath devicePath, final String[] measur */ public ClusterSchemaTree getMatchedSchemaWithTemplate(final PartialPath devicePath) { final ClusterSchemaTree tree = new ClusterSchemaTree(); - final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath.getFullPath()); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath); if (!(schema instanceof DeviceTemplateSchema)) { return tree; } @@ -148,7 +148,7 @@ public ClusterSchemaTree getMatchedSchemaWithTemplate(final PartialPath devicePa */ public ClusterSchemaTree getMatchedSchemaWithoutTemplate(final PartialPath fullPath) { final ClusterSchemaTree tree = new ClusterSchemaTree(); - final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevice()); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevicePath()); if (!(schema instanceof DeviceNormalSchema)) { return tree; } @@ -168,7 +168,7 @@ public List computeWithoutTemplate(final ISchemaComputation schemaCompu final String[] measurements = schemaComputation.getMeasurements(); final IDeviceSchema schema = - deviceSchemaCache.getDeviceSchema(schemaComputation.getDevicePath().getFullPath()); + deviceSchemaCache.getDeviceSchema(schemaComputation.getDevicePath()); if (!(schema instanceof DeviceNormalSchema)) { return IntStream.range(0, schemaComputation.getMeasurements().length) .boxed() @@ -225,7 +225,7 @@ public Pair, List> computeSourceOfLogicalView( } final PartialPath fullPath = logicalViewSchema.getSourcePathIfWritable(); - final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevice()); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevicePath()); if (!(schema instanceof DeviceNormalSchema)) { indexOfMissingMeasurements.add(i); continue; @@ -261,7 +261,7 @@ public List computeWithTemplate(final ISchemaComputation computation) { final List indexOfMissingMeasurements = new ArrayList<>(); final String[] measurements = computation.getMeasurements(); final IDeviceSchema deviceSchema = - deviceSchemaCache.getDeviceSchema(computation.getDevicePath().getFullPath()); + deviceSchemaCache.getDeviceSchema(computation.getDevicePath()); if (!(deviceSchema instanceof DeviceTemplateSchema)) { return IntStream.range(0, measurements.length).boxed().collect(Collectors.toList()); @@ -350,7 +350,7 @@ public void put(final ClusterSchemaTree tree) { } public TimeValuePair getLastCache(final PartialPath seriesPath) { - return deviceSchemaCache.getLastEntry(seriesPath.getDevice(), seriesPath.getMeasurement()); + return deviceSchemaCache.getLastEntry(seriesPath.getDevicePath(), seriesPath.getMeasurement()); } public void invalidateLastCache(final PartialPath path) { @@ -380,7 +380,7 @@ public void invalidateDatabaseLastCache(final String database) { */ public void updateLastCacheIfExists( final String database, - final String deviceID, + final PartialPath deviceID, final String[] measurements, final @Nonnull TimeValuePair[] timeValuePairs, final boolean isAligned, @@ -400,7 +400,7 @@ public void updateLastCacheIfExists( * stale value is written, but it won't affect the eventual consistency. * *

- Second time put the calculated {@link TimeValuePair}, and use {@link - * #updateLastCacheIfExists(String, String, String[], TimeValuePair[], boolean, + * #updateLastCacheIfExists(String, PartialPath, String[], TimeValuePair[], boolean, * IMeasurementSchema[])}. The input {@link TimeValuePair} shall never be or contain {@code null}, * if the measurement is with all {@code null}s, its {@link TimeValuePair} shall be {@link * DeviceLastCache#EMPTY_TIME_VALUE_PAIR}. This method is not supposed to update time column. @@ -418,7 +418,7 @@ public void updateLastCache( final String database, final MeasurementPath measurementPath, final boolean isInvalidate) { deviceSchemaCache.updateLastCache( database, - measurementPath.getDevice(), + measurementPath.getDevicePath(), new String[] {measurementPath.getMeasurement()}, isInvalidate ? new TimeValuePair[] {null} : null, measurementPath.isUnderAlignedEntity(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index ca1487842f171..a4aad825140df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -32,10 +32,8 @@ import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegion; import org.apache.tsfile.common.constant.TsFileConstant; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; @@ -79,7 +77,7 @@ public class DeviceSchemaCache { * Leading_Segment, {@link IDeviceID}, Map{@literal <}Measurement, Schema{@literal * >}/templateInfo{@literal >} */ - private final IDualKeyCache dualKeyCache; + private final IDualKeyCache dualKeyCache; private final Map databasePool = new ConcurrentHashMap<>(); @@ -87,12 +85,12 @@ public class DeviceSchemaCache { DeviceSchemaCache() { dualKeyCache = - new DualKeyCacheBuilder() + new DualKeyCacheBuilder() .cacheEvictionPolicy( DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) .memoryCapacity(config.getAllocateMemoryForSchemaCache()) .firstKeySizeComputer(segment -> (int) RamUsageEstimator.sizeOf(segment)) - .secondKeySizeComputer(device -> (int) RamUsageEstimator.sizeOf(device)) + .secondKeySizeComputer(PartialPath::estimateSize) .valueSizeComputer(TableDeviceCacheEntry::estimateSize) .build(); MetricService.getInstance().addMetricSet(new DataNodeSchemaCacheMetrics(this)); @@ -108,7 +106,7 @@ public class DeviceSchemaCache { * @return {@code null} iff cache miss, {@link DeviceLastCache#EMPTY_TIME_VALUE_PAIR} iff cache * hit but result is {@code null}, and the result value otherwise. */ - public TimeValuePair getLastEntry(final String device, final String measurement) { + public TimeValuePair getLastEntry(final PartialPath device, final String measurement) { final TableDeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); return Objects.nonNull(entry) ? entry.getTimeValuePair(measurement) : null; } @@ -118,7 +116,7 @@ public TimeValuePair getLastEntry(final String device, final String measurement) * * @param device IDeviceID */ - public void invalidateDeviceLastCache(final String device) { + public void invalidateDeviceLastCache(final PartialPath device) { dualKeyCache.update( getLeadingSegment(device), device, null, entry -> -entry.invalidateLastCache(), false); } @@ -127,12 +125,11 @@ public void invalidateDeviceLastCache(final String device) { public void putDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { final PartialPath devicePath = deviceSchemaInfo.getDevicePath(); - final String device = devicePath.getFullPath(); final String previousDatabase = databasePool.putIfAbsent(database, database); dualKeyCache.update( - getLeadingSegment(device), - device, + getLeadingSegment(devicePath), + devicePath, new TableDeviceCacheEntry(), entry -> entry.setDeviceSchema( @@ -140,14 +137,14 @@ public void putDeviceSchema(final String database, final DeviceSchemaInfo device true); } - public IDeviceSchema getDeviceSchema(final String device) { + public IDeviceSchema getDeviceSchema(final PartialPath device) { final TableDeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); return Objects.nonNull(entry) ? entry.getDeviceSchema() : null; } void updateLastCache( final String database, - final String device, + final PartialPath device, final String[] measurements, final @Nullable TimeValuePair[] timeValuePairs, final boolean isAligned, @@ -173,7 +170,7 @@ void updateLastCache( } public boolean getLastCache( - final Map>>> inputMap) { + final Map>> inputMap) { return dualKeyCache.batchApply(inputMap, TableDeviceCacheEntry::updateInputMap); } @@ -185,8 +182,7 @@ void invalidateLastCache(final PartialPath devicePath, final String measurement) : entry -> -entry.invalidateLastCache(measurement); if (!devicePath.hasWildcard()) { - final String device = devicePath.getFullPath(); - dualKeyCache.update(getLeadingSegment(device), device, null, updateFunction, false); + dualKeyCache.update(getLeadingSegment(devicePath), devicePath, null, updateFunction, false); } else { // This may take quite a long time to perform, yet we assume that the "invalidateLastCache" is // only called by deletions, which has a low frequency; and it has avoided that @@ -204,17 +200,7 @@ void invalidateLastCache(final PartialPath devicePath, final String measurement) return true; } }, - cachedDeviceID -> { - try { - return new PartialPath(cachedDeviceID).matchFullPath(devicePath); - } catch (final IllegalPathException e) { - logger.warn( - "Illegal device {} found in cache when invalidating by path {}, invalidate it anyway", - cachedDeviceID, - devicePath); - return true; - } - }, + cachedDeviceID -> cachedDeviceID.matchFullPath(devicePath), updateFunction); } } @@ -223,8 +209,7 @@ void invalidateLastCache(final PartialPath devicePath, final String measurement) void invalidateCache( final @Nonnull PartialPath devicePath, final boolean isMultiLevelWildcardMeasurement) { if (!devicePath.hasWildcard()) { - final String device = devicePath.getFullPath(); - dualKeyCache.invalidate(getLeadingSegment(device), device); + dualKeyCache.invalidate(getLeadingSegment(devicePath), devicePath); } else { // This may take quite a long time to perform, yet we assume that the "invalidateLastCache" is // only called by deletions, which has a low frequency; and it has avoided that @@ -245,8 +230,8 @@ void invalidateCache( cachedDeviceID -> { try { return isMultiLevelWildcardMeasurement - ? devicePath.matchPrefixPath(new PartialPath(cachedDeviceID)) - : devicePath.matchFullPath(new PartialPath(cachedDeviceID)); + ? devicePath.matchPrefixPath(cachedDeviceID) + : devicePath.matchFullPath(cachedDeviceID); } catch (final IllegalPathException e) { logger.warn( "Illegal device {} found in cache when invalidating by path {}, invalidate it anyway", @@ -338,13 +323,14 @@ public void invalidateAll() { // Utils of leading segment - private static String getLeadingSegment(final String device) { + private static String getLeadingSegment(final PartialPath device) { final String segment; int lastSeparatorPos = -1; int separatorNum = 0; - for (int i = 0; i < device.length(); i++) { - if (device.charAt(i) == TsFileConstant.PATH_SEPARATOR_CHAR) { + final String deviceStr = device.getFullPath(); + for (int i = 0; i < deviceStr.length(); i++) { + if (deviceStr.charAt(i) == TsFileConstant.PATH_SEPARATOR_CHAR) { lastSeparatorPos = i; separatorNum++; if (separatorNum == 3) { @@ -354,11 +340,11 @@ private static String getLeadingSegment(final String device) { } if (lastSeparatorPos == -1) { // not find even one separator, probably during a test, use the device as the tableName - segment = device; + segment = deviceStr; } else { // use the first DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME segments or all segments but the last // one as the table name - segment = device.substring(0, lastSeparatorPos); + segment = deviceStr.substring(0, lastSeparatorPos); } return segment; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java index 580406a89a786..55e17700a5489 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java @@ -21,9 +21,7 @@ import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; -import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -138,14 +136,14 @@ TimeValuePair getTimeValuePair(final String measurement) { return Objects.nonNull(cache) ? cache.getTimeValuePair(measurement) : null; } - boolean updateInputMap(final @Nonnull Map> updateMap) { + boolean updateInputMap(final @Nonnull Map updateMap) { // Shall only call this for original table device for (final String measurement : updateMap.keySet()) { final TimeValuePair result = getTimeValuePair(measurement); if (result == null) { return false; } - updateMap.get(measurement).setRight(result); + updateMap.put(measurement, result); } return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 6b050438062b4..167d110d0e96d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -314,7 +314,8 @@ long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) throws MetadataException; int fillLastQueryMap( - final PartialPath pattern, final Map> mapToFill) + final PartialPath pattern, + final Map>> mapToFill) throws MetadataException; // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 90870c22018ee..fb9b875fa5014 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3262,9 +3262,16 @@ private void onTsFileLoaded( if (timeIndex instanceof DeviceTimeIndex) { DeviceTimeIndex deviceTimeIndex = (DeviceTimeIndex) timeIndex; for (IDeviceID deviceID : deviceTimeIndex.getDevices()) { - DataNodeSchemaCache.getInstance() - .getDeviceSchemaCache() - .invalidateDeviceLastCache(((PlainDeviceID) deviceID).toStringID()); + try { + DataNodeSchemaCache.getInstance() + .getDeviceSchemaCache() + .invalidateDeviceLastCache(new PartialPath(deviceID)); + } catch (IllegalPathException e) { + logger.error( + "Failed to construct path for invalidating last cache of {}", deviceID, e); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); + return; + } } } else { DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); From 9ea3f96d276a94d1e54e4d04baa90cc5b0d56fe0 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 18:28:40 +0800 Subject: [PATCH 09/33] continue-kun --- .../analyze/cache/schema/DeviceSchemaCache.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index a4aad825140df..0f2fb86b6b0b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -227,19 +227,10 @@ void invalidateCache( return true; } }, - cachedDeviceID -> { - try { - return isMultiLevelWildcardMeasurement + cachedDeviceID -> + isMultiLevelWildcardMeasurement ? devicePath.matchPrefixPath(cachedDeviceID) - : devicePath.matchFullPath(cachedDeviceID); - } catch (final IllegalPathException e) { - logger.warn( - "Illegal device {} found in cache when invalidating by path {}, invalidate it anyway", - cachedDeviceID, - devicePath); - return true; - } - }); + : devicePath.matchFullPath(cachedDeviceID)); } } From 9d42f72d578d686674e17959c3025b3d8aeaa004 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 15 Sep 2025 19:00:28 +0800 Subject: [PATCH 10/33] kunkun --- .../rest/v2/impl/RestApiServiceImpl.java | 25 +++++++++++-------- .../thrift/impl/ClientRPCServiceImpl.java | 8 +++--- .../cache/schema/DeviceSchemaCache.java | 2 +- .../schemaregion/ISchemaRegion.java | 2 +- .../impl/SchemaRegionMemoryImpl.java | 3 ++- .../impl/SchemaRegionPBTreeImpl.java | 3 ++- .../impl/mem/MTreeBelowSGMemoryImpl.java | 9 +++++-- 7 files changed, 32 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java index ffaecaea7b5e0..8712cbb8f38d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java @@ -110,7 +110,7 @@ public Response executeFastLastQueryStatement( PartialPath prefixPath = new PartialPath(prefixPathList.getPrefixPaths().toArray(new String[0])); - Map> resultMap = new HashMap<>(); + final Map>> resultMap = new HashMap<>(); final String prefixString = prefixPath.toString(); for (ISchemaRegion region : SchemaEngine.getInstance().getAllSchemaRegions()) { @@ -121,7 +121,7 @@ public Response executeFastLastQueryStatement( region.fillLastQueryMap(prefixPath, resultMap); } // Check cache first - if (!DataNodeSchemaCache.getInstance().getLastCache(resultMap)) { + if (!DataNodeSchemaCache.getInstance().getDeviceSchemaCache().getLastCache(resultMap)) { IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); TSLastDataQueryReq tsLastDataQueryReq = FastLastHandler.createTSLastDataQueryReq(clientSession, prefixPathList); @@ -169,14 +169,19 @@ public Response executeFastLastQueryStatement( List timeseries = new ArrayList<>(); List valueList = new ArrayList<>(); List dataTypeList = new ArrayList<>(); - for (Map.Entry> entry : resultMap.entrySet()) { - final String deviceWithPrefix = entry.getKey() + TsFileConstant.PATH_SEPARATOR; - for (Map.Entry measurementEntry : entry.getValue().entrySet()) { - final TimeValuePair tvPair = measurementEntry.getValue(); - valueList.add(tvPair.getValue().getStringValue()); - dataTypeList.add(tvPair.getValue().getDataType().name()); - targetDataSet.addTimestampsItem(tvPair.getTimestamp()); - timeseries.add(deviceWithPrefix + measurementEntry.getKey()); + for (Map.Entry>> entry : + resultMap.entrySet()) { + for (final Map.Entry> device2MeasurementLastEntry : + entry.getValue().entrySet()) { + final String deviceWithPrefix = entry.getKey() + TsFileConstant.PATH_SEPARATOR; + for (Map.Entry measurementEntry : + device2MeasurementLastEntry.getValue().entrySet()) { + final TimeValuePair tvPair = measurementEntry.getValue(); + valueList.add(tvPair.getValue().getStringValue()); + dataTypeList.add(tvPair.getValue().getDataType().name()); + targetDataSet.addTimestampsItem(tvPair.getTimestamp()); + timeseries.add(deviceWithPrefix + measurementEntry.getKey()); + } } } if (!timeseries.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 46e3e79ef2d07..95dbcb1655472 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -817,7 +817,7 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath( "The \"executeFastLastDataQueryForOnePrefixPath\" dos not support wildcards.")); } - final Map>> resultMap = new HashMap<>(); + final Map>> resultMap = new HashMap<>(); int sensorNum = 0; final String prefixString = prefixPath.toString(); @@ -838,14 +838,14 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath( // 2.2 all sensors hit cache, return response ~= 20ms final TsBlockBuilder builder = LastQueryUtil.createTsBlockBuilder(sensorNum); - for (final Map.Entry>> result : + for (final Map.Entry>> result : resultMap.entrySet()) { - for (final Map.Entry> device2MeasurementLastEntry : + for (final Map.Entry> device2MeasurementLastEntry : result.getValue().entrySet()) { final String deviceWithSeparator = device2MeasurementLastEntry.getKey() + TsFileConstant.PATH_SEPARATOR; for (final Map.Entry measurementLastEntry : - result.getValue().entrySet()) { + device2MeasurementLastEntry.getValue().entrySet()) { final TimeValuePair tvPair = measurementLastEntry.getValue(); LastQueryUtil.appendLastValue( builder, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index 0f2fb86b6b0b6..4ba49378314ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -314,7 +314,7 @@ public void invalidateAll() { // Utils of leading segment - private static String getLeadingSegment(final PartialPath device) { + public static String getLeadingSegment(final PartialPath device) { final String segment; int lastSeparatorPos = -1; int separatorNum = 0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 167d110d0e96d..12630a147c0d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -315,7 +315,7 @@ long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) int fillLastQueryMap( final PartialPath pattern, - final Map>> mapToFill) + final Map>> mapToFill) throws MetadataException; // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 95feb33e52a91..06a00c64e0330 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1325,7 +1325,8 @@ public long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) @Override public int fillLastQueryMap( - final PartialPath pattern, final Map> mapToFill) + final PartialPath pattern, + final Map>> mapToFill) throws MetadataException { return mtree.fillLastQueryMap(pattern, mapToFill); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 6ae88e4946823..7c05e864b81f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1430,7 +1430,8 @@ public long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) @Override public int fillLastQueryMap( - PartialPath pattern, Map> mapToFill) { + final PartialPath pattern, + final Map>> mapToFill) { throw new UnsupportedOperationException("Not implemented"); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 9f6316583d58a..60e7f99f74f4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.exception.metadata.template.TemplateIsInUseException; import org.apache.iotdb.db.exception.quota.ExceedQuotaException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DeviceSchemaCache; import org.apache.iotdb.db.schemaengine.metric.SchemaRegionMemMetric; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; @@ -1076,7 +1077,8 @@ public IDeviceSchemaInfo next() { } public int fillLastQueryMap( - final PartialPath prefixPath, final Map> mapToFill) + final PartialPath prefixPath, + final Map>> mapToFill) throws MetadataException { final int[] sensorNum = {0}; try (final EntityUpdater updater = @@ -1091,7 +1093,10 @@ protected void updateEntity(final IDeviceMNode node) { measurementMap.put(child.getName(), null); } } - mapToFill.put(node.getPartialPath(), measurementMap); + final PartialPath path = node.getPartialPath(); + mapToFill + .computeIfAbsent(DeviceSchemaCache.getLeadingSegment(path), o -> new HashMap<>()) + .put(path, measurementMap); sensorNum[0] += measurementMap.size(); } }) { From bcb7c228ec2261120be1937b0041770112da9f94 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 16 Sep 2025 09:47:56 +0800 Subject: [PATCH 11/33] partial --- .../planner/plan/node/write/InsertNode.java | 15 +++++++++++ .../plan/node/write/InsertRowNode.java | 17 ++++++++++++ .../storageengine/dataregion/DataRegion.java | 26 ++----------------- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index f013dd6755695..c664bbb9ddfaa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -295,6 +295,21 @@ public boolean allMeasurementFailed() { return true; } + public String[] getRawMeasurements() { + String[] measurements = getMeasurements(); + MeasurementSchema[] measurementSchemas = getMeasurementSchemas(); + String[] rawMeasurements = new String[measurements.length]; + for (int i = 0; i < measurements.length; i++) { + if (measurementSchemas[i] != null) { + // get raw measurement rather than alias + rawMeasurements[i] = measurementSchemas[i].getMeasurementId(); + } else { + rawMeasurements[i] = measurements[i]; + } + } + return rawMeasurements; + } + // endregion // region progress index diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index 29a163b2ec504..8cbf48b62ac88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -860,4 +861,20 @@ public TimeValuePair composeTimeValuePair(int columnIndex) { Object value = values[columnIndex]; return new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)); } + + public void updateLastCache(String databaseName) { + String[] rawMeasurements = getRawMeasurements(); + TimeValuePair[] timeValuePairs = new TimeValuePair[rawMeasurements.length]; + for (int i = 0; i < rawMeasurements.length; i++) { + timeValuePairs[i] = composeTimeValuePair(i); + } + DataNodeSchemaCache.getInstance() + .updateLastCacheIfExists( + databaseName, + devicePath, + rawMeasurements, + timeValuePairs, + isAligned, + measurementSchemas); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index fb9b875fa5014..4c8e70f6258af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -1324,29 +1324,7 @@ private TsFileProcessor insertToTsFileProcessor( } private void tryToUpdateInsertRowLastCache(InsertRowNode node) { - long latestFlushedTime = lastFlushTimeMap.getGlobalFlushedTime(node.getDeviceID()); - String[] measurements = node.getMeasurements(); - MeasurementSchema[] measurementSchemas = node.getMeasurementSchemas(); - String[] rawMeasurements = new String[measurements.length]; - for (int i = 0; i < measurements.length; i++) { - if (measurementSchemas[i] != null) { - // get raw measurement rather than alias - rawMeasurements[i] = measurementSchemas[i].getMeasurementId(); - } else { - rawMeasurements[i] = measurements[i]; - } - } - DataNodeSchemaCache.getInstance() - .updateLastCache( - getDatabaseName(), - node.getDevicePath(), - rawMeasurements, - node.getMeasurementSchemas(), - node.isAligned(), - node::composeTimeValuePair, - index -> node.getValues()[index] != null, - true, - latestFlushedTime); + node.updateLastCache(databaseName); } private List insertToTsFileProcessors( @@ -2487,7 +2465,7 @@ public void deleteDataDirectly( if (deleted) { return; } - DataNodeSchemaCache.getInstance().invalidateLastCacheInDataRegion(getDatabaseName()); + DataNodeSchemaCache.getInstance().invalidateDatabaseLastCache(getDatabaseName()); // write log to impacted working TsFileProcessors List walListeners = logDeletionInWAL(startTime, endTime, searchIndex, pathToDelete); From a66f0de2b9b6c3141527620af8ac0d616fea2cd3 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:12:33 +0800 Subject: [PATCH 12/33] may-complete --- .../last/AbstractUpdateLastCacheOperator.java | 25 ++++++- .../operator/process/last/LastQueryUtil.java | 4 +- .../plan/planner/OperatorTreeGenerator.java | 32 ++++++++ .../plan/node/write/InsertTabletNode.java | 17 +++++ .../storageengine/dataregion/DataRegion.java | 74 +++---------------- 5 files changed, 81 insertions(+), 71 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java index 30d305bcec7ea..9d718198622c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.execution.operator.process.ProcessOperator; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DeviceLastCache; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; @@ -35,9 +36,11 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.schema.IMeasurementSchema; import javax.annotation.Nullable; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractUpdateLastCacheOperator implements ProcessOperator { @@ -117,8 +120,17 @@ protected void mayUpdateLastCache( } if (!deviceInMultiRegion) { - lastCache.updateLastCache( - getDatabaseName(), fullPath, new TimeValuePair(time, value), false, Long.MIN_VALUE); + lastCache.updateLastCacheIfExists( + getDatabaseName(), + fullPath.getDevicePath(), + new String[] {fullPath.getMeasurement()}, + new TimeValuePair[] { + Objects.nonNull(value) + ? new TimeValuePair(time, value) + : needUpdateNullEntry ? DeviceLastCache.EMPTY_TIME_VALUE_PAIR : null + }, + fullPath.isUnderAlignedEntity(), + new IMeasurementSchema[] {fullPath.getMeasurementSchema()}); return; } // update cache in DataNodeQueryContext @@ -127,8 +139,13 @@ protected void mayUpdateLastCache( } if (seriesScanInfo.left.decrementAndGet() == 0) { - lastCache.updateLastCache( - getDatabaseName(), fullPath, seriesScanInfo.right, false, Long.MIN_VALUE); + lastCache.updateLastCacheIfExists( + getDatabaseName(), + fullPath.getDevicePath(), + new String[] {fullPath.getMeasurement()}, + new TimeValuePair[] {seriesScanInfo.right}, + fullPath.isUnderAlignedEntity(), + new IMeasurementSchema[] {fullPath.getMeasurementSchema()}); } } finally { dataNodeQueryContext.unLock(deviceInMultiRegion); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java index 98525791f6dd5..f6578c7ef70b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java @@ -166,8 +166,8 @@ public static List createAggregators(TSDataType dataType, int valueC public static boolean needUpdateCache(Filter timeFilter) { // Update the cache only when, the filter is gt (greater than) or ge (greater than or equal to) - return CACHE_ENABLED && (timeFilter == null || timeFilter instanceof TimeGtEq) - || (timeFilter instanceof TimeGt); + return CACHE_ENABLED + && (timeFilter == null || timeFilter instanceof TimeGtEq || timeFilter instanceof TimeGt); } public static boolean needUpdateNullEntry(Filter timeFilter) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 2e6a6be8d1adf..d4a2a8d70ad60 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -2774,6 +2774,17 @@ private UpdateLastCacheOperator createUpdateLastCacheOperator( node.getPlanNodeId(), UpdateLastCacheOperator.class.getSimpleName()); + final boolean isNeedUpdateLastCache = context.isNeedUpdateLastCache(); + if (isNeedUpdateLastCache) { + DataNodeSchemaCache.getInstance() + .updateLastCache( + ((DataDriverContext) operatorContext.getDriverContext()) + .getDataRegion() + .getDatabaseName(), + fullPath, + false); + } + return Objects.isNull(node.getOutputViewPath()) ? new UpdateLastCacheOperator( operatorContext, @@ -2811,6 +2822,27 @@ private AlignedUpdateLastCacheOperator createAlignedUpdateLastCacheOperator( planNodeId, AlignedUpdateLastCacheOperator.class.getSimpleName()); + final boolean isNeedUpdateLastCache = context.isNeedUpdateLastCache(); + if (isNeedUpdateLastCache) { + final int size = unCachedPath.getColumnNum(); + final PartialPath devicePath = unCachedPath.getDevicePath(); + final String databaseName = + ((DataDriverContext) operatorContext.getDriverContext()) + .getDataRegion() + .getDatabaseName(); + + for (int i = 0; i < size; ++i) { + DataNodeSchemaCache.getInstance() + .updateLastCache( + databaseName, + new MeasurementPath( + devicePath.concatNode(unCachedPath.getMeasurementList().get(i)), + unCachedPath.getSchemaList().get(i), + true), + false); + } + } + return Objects.isNull(outputViewPath) ? new AlignedUpdateLastCacheOperator( operatorContext, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 790d2b7987b03..eb5015a93f44f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -1131,4 +1132,20 @@ public TimeValuePair composeLastTimeValuePair(int measurementIndex) { } return new TimeValuePair(times[lastIdx], value); } + + public void updateLastCache(final String databaseName) { + final String[] rawMeasurements = getRawMeasurements(); + final TimeValuePair[] timeValuePairs = new TimeValuePair[rawMeasurements.length]; + for (int i = 0; i < rawMeasurements.length; i++) { + timeValuePairs[i] = composeLastTimeValuePair(i); + } + DataNodeSchemaCache.getInstance() + .updateLastCacheIfExists( + databaseName, + devicePath, + rawMeasurements, + timeValuePairs, + isAligned, + measurementSchemas); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 4c8e70f6258af..a16bb396b5c88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -138,7 +138,6 @@ import org.apache.tsfile.read.reader.TsFileLastReader; import org.apache.tsfile.utils.FSUtils; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.writer.RestorableTsFileIOWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1282,29 +1281,7 @@ private boolean insertTabletToTsFileProcessor( } private void tryToUpdateInsertTabletLastCache(InsertTabletNode node) { - long latestFlushedTime = lastFlushTimeMap.getGlobalFlushedTime(node.getDeviceID()); - String[] measurements = node.getMeasurements(); - MeasurementSchema[] measurementSchemas = node.getMeasurementSchemas(); - String[] rawMeasurements = new String[measurements.length]; - for (int i = 0; i < measurements.length; i++) { - if (measurementSchemas[i] != null) { - // get raw measurement rather than alias - rawMeasurements[i] = measurementSchemas[i].getMeasurementId(); - } else { - rawMeasurements[i] = measurements[i]; - } - } - DataNodeSchemaCache.getInstance() - .updateLastCache( - getDatabaseName(), - node.getDevicePath(), - rawMeasurements, - node.getMeasurementSchemas(), - node.isAligned(), - node::composeLastTimeValuePair, - index -> node.getColumns()[index] != null, - true, - latestFlushedTime); + node.updateLastCache(getDatabaseName()); } private TsFileProcessor insertToTsFileProcessor( @@ -1394,35 +1371,8 @@ private List insertToTsFileProcessors( } private void tryToUpdateInsertRowsLastCache(List nodeList) { - DataNodeSchemaCache.getInstance().takeReadLock(); - try { - for (InsertRowNode node : nodeList) { - long latestFlushedTime = lastFlushTimeMap.getGlobalFlushedTime(node.getDeviceID()); - String[] measurements = node.getMeasurements(); - MeasurementSchema[] measurementSchemas = node.getMeasurementSchemas(); - String[] rawMeasurements = new String[measurements.length]; - for (int i = 0; i < measurements.length; i++) { - if (measurementSchemas[i] != null) { - // get raw measurement rather than alias - rawMeasurements[i] = measurementSchemas[i].getMeasurementId(); - } else { - rawMeasurements[i] = measurements[i]; - } - } - DataNodeSchemaCache.getInstance() - .updateLastCacheWithoutLock( - getDatabaseName(), - node.getDevicePath(), - rawMeasurements, - node.getMeasurementSchemas(), - node.isAligned(), - node::composeTimeValuePair, - index -> node.getValues()[index] != null, - true, - latestFlushedTime); - } - } finally { - DataNodeSchemaCache.getInstance().releaseReadLock(); + for (InsertRowNode node : nodeList) { + node.updateLastCache(databaseName); } } @@ -3279,16 +3229,13 @@ private void updateLastCache(TsFileResource newTsFileResource, TsFileLastReader try { // we do not update schema here, so aligned is not relevant DataNodeSchemaCache.getInstance() - .updateLastCache( + .updateLastCacheIfExists( databaseName, new PartialPath(deviceID), measurements, - null, + timeValuePairs, false, - value -> timeValuePairs[value], - i -> true, - true, - 0L); + null); } catch (IllegalPathException e) { logger.error("Failed to construct path for invalidating last cache of {}", deviceID, e); DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); @@ -3309,16 +3256,13 @@ private void updateLastCache(TsFileResource newTsFileResource, TsFileLastReader try { // we do not update schema here, so aligned is not relevant DataNodeSchemaCache.getInstance() - .updateLastCache( + .updateLastCacheIfExists( databaseName, new PartialPath(deviceID), measurements, - null, + timeValuePairs, false, - value -> timeValuePairs[value], - i -> true, - true, - 0L); + null); } catch (IllegalPathException e) { logger.error("Failed to construct path for invalidating last cache of {}", deviceID, e); DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); From a4f2fee9e8b1a49875373b20725a698c0a1a293f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:17:15 +0800 Subject: [PATCH 13/33] fix --- .../plan/analyze/cache/schema/DeviceSchemaCache.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index 4ba49378314ae..0a0b54cf74ab8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -76,6 +76,12 @@ public class DeviceSchemaCache { /** * Leading_Segment, {@link IDeviceID}, Map{@literal <}Measurement, Schema{@literal * >}/templateInfo{@literal >} + * + *

The segment is used to: + * + *

1. Keep abreast of the newer versions. + * + *

2. Optimize the speed in invalidation by databases for most scenarios. */ private final IDualKeyCache dualKeyCache; From 86348e4944169d1c75bdc65f24cacef5fa22ca84 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:24:01 +0800 Subject: [PATCH 14/33] Remove useless --- .../dualkeycache/IDualKeyCacheUpdating.java | 45 -- .../lastcache/DataNodeLastCacheManager.java | 72 --- .../schema/lastcache/ILastCacheContainer.java | 0 .../schema/lastcache/LastCacheContainer.java | 103 ---- .../lastcache/value/ILastCacheValue.java | 0 .../lastcache/value/LastCacheValue.java | 0 .../cache/dualkeycache/DualKeyCacheTest.java | 491 ------------------ 7 files changed, 711 deletions(-) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheUpdating.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/DataNodeLastCacheManager.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java delete mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/dualkeycache/DualKeyCacheTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheUpdating.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheUpdating.java deleted file mode 100644 index 7ebdd566c9eff..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheUpdating.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache; - -/** - * This interfaces defines the behaviour needed be implemented and executed during cache value - * traverse. - * - * @param The first key of target cache values - * @param The second key of one target cache value - * @param The cache value - */ -public interface IDualKeyCacheUpdating { - - /** Return the first key of target cache values. */ - FK getFirstKey(); - - /** Return the second key list of target cache values. */ - SK[] getSecondKeyList(); - - /** - * Compute each target cache value. The index is the second key's position in second key list. The - * value here can be updated. - * - * @return increasing memory usage - */ - int updateValue(int index, V value); -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/DataNodeLastCacheManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/DataNodeLastCacheManager.java deleted file mode 100644 index da31de2235546..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/DataNodeLastCacheManager.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache; - -import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.SchemaCacheEntry; - -import org.apache.tsfile.read.TimeValuePair; - -public class DataNodeLastCacheManager { - - private static final boolean CACHE_ENABLED = - CommonDescriptor.getInstance().getConfig().isLastCacheEnable(); - - /** - * get the last cache value from time series - * - * @param entry schema cache entry in DataNodeSchemaCache - * @return the last cache value - */ - public static TimeValuePair getLastCache(SchemaCacheEntry entry) { - if (!CACHE_ENABLED || null == entry) { - return null; - } - ILastCacheContainer lastCacheContainer = entry.getLastCacheContainer(); - return lastCacheContainer == null ? null : lastCacheContainer.getCachedLast(); - } - - /** - * update the last cache value of time series - * - * @param entry schema cache entry in DataNodeSchemaCache - * @param timeValuePair the latest point value - * @param highPriorityUpdate the last value from insertPlan is high priority - * @param latestFlushedTime latest flushed time - * @return increasing of memory usage - */ - public static int updateLastCache( - SchemaCacheEntry entry, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - if (!CACHE_ENABLED || null == entry) { - return 0; - } - return entry.updateLastCache(timeValuePair, highPriorityUpdate, latestFlushedTime); - } - - public static int invalidateLastCache(SchemaCacheEntry entry) { - if (!CACHE_ENABLED || null == entry) { - return 0; - } - return entry.invalidateLastCache(); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java deleted file mode 100644 index 8730990333541..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache; - -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.value.ILastCacheValue; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.value.LastCacheValue; - -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.TsPrimitiveType; - -/** - * This class possesses the ILastCacheValue and implements the basic last cache operations. - * - *

The ILastCacheValue may be extended to ILastCacheValue List in future to support batched last - * value cache. - */ -public class LastCacheContainer implements ILastCacheContainer { - - ILastCacheValue lastCacheValue; - - @Override - public TimeValuePair getCachedLast() { - return lastCacheValue == null ? null : lastCacheValue.getTimeValuePair(); - } - - @Override - public synchronized int updateCachedLast( - TimeValuePair timeValuePair, boolean highPriorityUpdate, Long latestFlushedTime) { - if (highPriorityUpdate) { // for write, we won't cache null value - if (timeValuePair == null || timeValuePair.getValue() == null) { - return 0; - } - } else { // for read, we need to cache null value, because it means that there is no data for - // this time series - if (timeValuePair == null) { - return 0; - } - } - - if (lastCacheValue == null) { - // If no cached last, (1) a last query (2) an unseq insertion or (3) a seq insertion will - // update cache. - if (!highPriorityUpdate || latestFlushedTime <= timeValuePair.getTimestamp()) { - lastCacheValue = new LastCacheValue(timeValuePair.getTimestamp(), timeValuePair.getValue()); - return lastCacheValue.estimateSize(); - } - } else if (timeValuePair.getTimestamp() > lastCacheValue.getTimestamp() - || (timeValuePair.getTimestamp() == lastCacheValue.getTimestamp() && highPriorityUpdate)) { - TsPrimitiveType oldValue = lastCacheValue.getValue(); - lastCacheValue.setTimestamp(timeValuePair.getTimestamp()); - lastCacheValue.setValue(timeValuePair.getValue()); - return getDiffSize(oldValue, timeValuePair.getValue()); - } - return 0; - } - - @Override - public int invalidateLastCache() { - int size = lastCacheValue.estimateSize(); - lastCacheValue = null; - return size; - } - - private int getDiffSize(TsPrimitiveType oldValue, TsPrimitiveType newValue) { - if (oldValue == null) { - return newValue == null ? 0 : newValue.getSize(); - } else if (oldValue instanceof TsPrimitiveType.TsBinary) { - return (newValue == null ? 0 : newValue.getSize()) - oldValue.getSize(); - } else { - return newValue == null ? -oldValue.getSize() : 0; - } - } - - /** - * Total basic 100B - * - *

    - *
  • LastCacheContainer Object header, 8B - *
  • ILastCacheValue reference, 8B - *
- */ - @Override - public int estimateSize() { - return 16 + (lastCacheValue == null ? 0 : lastCacheValue.estimateSize()); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/dualkeycache/DualKeyCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/dualkeycache/DualKeyCacheTest.java deleted file mode 100644 index 17ec1bcfb0ccf..0000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/dualkeycache/DualKeyCacheTest.java +++ /dev/null @@ -1,491 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.metadata.cache.dualkeycache; - -import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.SchemaCacheEntry; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheComputation; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheUpdating; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.DataNodeLastCacheManager; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -@RunWith(Parameterized.class) -public class DualKeyCacheTest { - - private final String policy; - - public DualKeyCacheTest(String policy) { - this.policy = policy; - } - - @Parameterized.Parameters - public static List getTestModes() { - return Arrays.asList("FIFO", "LRU"); - } - - @Test - public void testBasicReadPut() { - DualKeyCacheBuilder dualKeyCacheBuilder = new DualKeyCacheBuilder<>(); - IDualKeyCache dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy(DualKeyCachePolicy.valueOf(policy)) - .memoryCapacity(300) - .firstKeySizeComputer(this::computeStringSize) - .secondKeySizeComputer(this::computeStringSize) - .valueSizeComputer(this::computeStringSize) - .build(); - - String[] firstKeyList = new String[] {"root.db.d1", "root.db.d2"}; - String[] secondKeyList = new String[] {"s1", "s2"}; - String[][] valueTable = - new String[][] {new String[] {"1-1", "1-2"}, new String[] {"2-1", "2-2"}}; - - for (int i = 0; i < firstKeyList.length; i++) { - for (int j = 0; j < secondKeyList.length; j++) { - dualKeyCache.put(firstKeyList[i], secondKeyList[j], valueTable[i][j]); - } - } - - int firstKeyOfMissingEntry = -1, secondKeyOfMissingEntry = -1; - for (int i = 0; i < firstKeyList.length; i++) { - for (int j = 0; j < secondKeyList.length; j++) { - String value = dualKeyCache.get(firstKeyList[i], secondKeyList[j]); - if (value == null) { - if (firstKeyOfMissingEntry == -1) { - firstKeyOfMissingEntry = i; - secondKeyOfMissingEntry = j; - } else { - Assert.fail(); - } - } else { - Assert.assertEquals(valueTable[i][j], value); - } - } - } - - Assert.assertEquals(230, dualKeyCache.stats().memoryUsage()); - Assert.assertEquals(4, dualKeyCache.stats().requestCount()); - Assert.assertEquals(3, dualKeyCache.stats().hitCount()); - - dualKeyCache.put( - firstKeyList[firstKeyOfMissingEntry], - secondKeyList[secondKeyOfMissingEntry], - valueTable[firstKeyOfMissingEntry][secondKeyOfMissingEntry]); - Assert.assertEquals(230, dualKeyCache.stats().memoryUsage()); - - for (int i = 0; i < firstKeyList.length; i++) { - int finalI = i; - dualKeyCache.compute( - new IDualKeyCacheComputation() { - @Override - public String getFirstKey() { - return firstKeyList[finalI]; - } - - @Override - public String[] getSecondKeyList() { - return secondKeyList; - } - - @Override - public void computeValue(int index, String value) { - if (value != null) { - Assert.assertEquals(valueTable[finalI][index], value); - } - } - }); - } - - Assert.assertEquals(8, dualKeyCache.stats().requestCount()); - Assert.assertEquals(6, dualKeyCache.stats().hitCount()); - } - - private int computeStringSize(String string) { - return 8 + 8 + 4 + 2 * string.length(); - } - - @Test - public void testComputeAndUpdateSize() { - DualKeyCacheBuilder dualKeyCacheBuilder = - new DualKeyCacheBuilder<>(); - IDualKeyCache dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy(DualKeyCachePolicy.valueOf(policy)) - .memoryCapacity(500) - .firstKeySizeComputer(this::computeStringSize) - .secondKeySizeComputer(this::computeStringSize) - .valueSizeComputer(SchemaCacheEntry::estimateSize) - .build(); - String firstKey = "root.db.d1"; - String[] secondKeyList = new String[] {"s1", "s2"}; - for (String s : secondKeyList) { - dualKeyCache.put( - firstKey, - s, - new SchemaCacheEntry( - "root.db", - new MeasurementSchema(s, TSDataType.INT32), - Collections.emptyMap(), - false)); - } - SchemaCacheEntry schemaCacheEntry = - new SchemaCacheEntry( - "root.db", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false); - int expectedSize = - computeStringSize("root.db.d1") - + computeStringSize("s1") * 2 - + SchemaCacheEntry.estimateSize(schemaCacheEntry) * 2; - Assert.assertEquals(expectedSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public String getFirstKey() { - return firstKey; - } - - @Override - public String[] getSecondKeyList() { - return secondKeyList; - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - return DataNodeLastCacheManager.updateLastCache( - value, new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - } - }); - int tmp = SchemaCacheEntry.estimateSize(schemaCacheEntry); - schemaCacheEntry.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - expectedSize += (SchemaCacheEntry.estimateSize(schemaCacheEntry) - tmp) * 2; - Assert.assertEquals(expectedSize, dualKeyCache.stats().memoryUsage()); - } - - @Test - public void testInvalidDatabase() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateCache(); - dualKeyCache.invalidate("root.db1"); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1"), "s11")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db2.d1")) - + computeStringSize("s1") * 2 - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)) - * 2; - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.evictOneEntry(); - expectSize = - PartialPath.estimateSize(new PartialPath("root.db2.d1")) - + computeStringSize("s1") - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.evictOneEntry(); - Assert.assertEquals(0, dualKeyCache.stats().memoryUsage()); - } - - @Test - public void testInvalidPathPattern1() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateCache(); - dualKeyCache.invalidate( - Arrays.asList(new PartialPath("root.db2.**"), new PartialPath("root.db1.d1.s1"))); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1"), "s11")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db1.d1")) - + PartialPath.estimateSize(new PartialPath("root.db1")) - + computeStringSize("s1") - + computeStringSize("s11") - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)) - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s11", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.evictOneEntry(); - dualKeyCache.evictOneEntry(); - Assert.assertEquals(0, dualKeyCache.stats().memoryUsage()); - } - - @Test - public void testInvalidPathPattern2() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateCache(); - dualKeyCache.invalidate( - Arrays.asList(new PartialPath("root.db1.**"), new PartialPath("root.db2.d1.*1"))); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1"), "s11")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db2.d1")) - + computeStringSize("s2") - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.evictOneEntry(); - Assert.assertEquals(0, dualKeyCache.stats().memoryUsage()); - } - - private IDualKeyCache generateCache() - throws IllegalPathException { - DualKeyCacheBuilder dualKeyCacheBuilder = - new DualKeyCacheBuilder<>(); - IDualKeyCache dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy(DualKeyCachePolicy.valueOf(policy)) - .memoryCapacity(2000) // actual threshold is 1600 - .firstKeySizeComputer(PartialPath::estimateSize) - .secondKeySizeComputer(this::computeStringSize) - .valueSizeComputer(SchemaCacheEntry::estimateSize) - .build(); - dualKeyCache.put( - new PartialPath("root.db1.d1"), - "s1", - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db1.d1"), - "s2", - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db1"), - "s11", - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s11", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db1.d1"), - "s2", - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db2.d1"), - "s1", - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db2.d1"), - "s2", - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1"), "s11")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db1.d1")) * 2 - + PartialPath.estimateSize(new PartialPath("root.db1")) - + computeStringSize("s1") * 4 - + computeStringSize("s11") - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)) - * 4 - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s11", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - return dualKeyCache; - } - - private IDualKeyCache generateLastCache() - throws IllegalPathException { - DualKeyCacheBuilder dualKeyCacheBuilder = - new DualKeyCacheBuilder<>(); - IDualKeyCache dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy(DualKeyCachePolicy.valueOf(policy)) - .memoryCapacity(2000) // actual threshold is 1600 - .firstKeySizeComputer(PartialPath::estimateSize) - .secondKeySizeComputer(this::computeStringSize) - .valueSizeComputer(SchemaCacheEntry::estimateSize) - .build(); - SchemaCacheEntry cacheEntry1 = - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry1.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db1.d1"), "s1", cacheEntry1); - - SchemaCacheEntry cacheEntry2 = - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry2.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db1.d1"), "s2", cacheEntry2); - - SchemaCacheEntry cacheEntry3 = - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry3.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db2.d1"), "s2", cacheEntry3); - - SchemaCacheEntry cacheEntry4 = - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry4.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db2.d1"), "s1", cacheEntry4); - - SchemaCacheEntry cacheEntry5 = - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry5.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db1"), "s2", cacheEntry5); - - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1"), "s2")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db1.d1")) * 2 - + PartialPath.estimateSize(new PartialPath("root.db1")) - + computeStringSize("s1") * 5 - + SchemaCacheEntry.estimateSize(cacheEntry1) * 5; - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - return dualKeyCache; - } - - @Test - public void testInvalidateSimpleTimeseriesAndDataRegion() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateLastCache(); - long memUse = dualKeyCache.stats().memoryUsage(); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db1.d1.s1")); - SchemaCacheEntry cacheEntry = dualKeyCache.get(new PartialPath("root.db1.d1"), "s1"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db1.d1.*")); - cacheEntry = dualKeyCache.get(new PartialPath("root.db1.d1"), "s2"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db2.d1.**")); - cacheEntry = dualKeyCache.get(new PartialPath("root.db2.d1"), "s2"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - dualKeyCache.invalidateDataRegionLastCache("root.db2"); - cacheEntry = dualKeyCache.get(new PartialPath("root.db2.d1"), "s1"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - cacheEntry = dualKeyCache.get(new PartialPath("root.db1"), "s2"); - // last cache container' estimateSize(): header 8b + Ilastcachevalueref 8b + lastcache's size - // invalidate operation: make Ilastcachevalueref = null. - // So the amount of change in size is estimateSize() - 8b - 8b - int size = cacheEntry.getLastCacheContainer().estimateSize() - 16; - Assert.assertEquals(memUse - size * 4, dualKeyCache.stats().memoryUsage()); - } - - @Test - public void testComplexInvalidate() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateLastCache(); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db1.*.s1")); - SchemaCacheEntry cacheEntry = dualKeyCache.get(new PartialPath("root.db1.d1"), "s1"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db1.**.s2")); - cacheEntry = dualKeyCache.get(new PartialPath("root.db1.d1"), "s2"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - } -} From 42362507025a232fca7c3032238eb87a375fb968 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:30:08 +0800 Subject: [PATCH 15/33] fix --- .../plan/analyze/cache/schema/SchemaCacheEntry.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java index 1b46e2c0faf65..d048692962281 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java @@ -78,13 +78,13 @@ public TSDataType getTsDataType() { public static int estimateSize(final SchemaCacheEntry schemaCacheEntry) { return INSTANCE_SIZE - + 2 * schemaCacheEntry.getSchema().getMeasurementName().length() + + 2 * schemaCacheEntry.getSchema().getMeasurementId().length() + (int) RamUsageEstimator.sizeOfMap(schemaCacheEntry.getTagMap()); } @Override public String getName() { - return iMeasurementSchema.getMeasurementName(); + return iMeasurementSchema.getMeasurementId(); } @Override From cf4b84f0f983d6f7f86f45395149e39fd199ad43 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:38:35 +0800 Subject: [PATCH 16/33] fix --- .../plan/analyze/cache/schema/DataNodeSchemaCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 483aa09df9a05..78b4f98f688c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -426,7 +426,7 @@ public void updateLastCache( true); } - public void invalidate(final List partialPathList) { + public void invalidate(final List partialPathList) { // Currently invalidate by device partialPathList.forEach( measurementPath -> { From e9ee7621292a5ae4e4cde815d74ab53d60f27747 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:42:48 +0800 Subject: [PATCH 17/33] fix --- .../db/metadata/cache/DataNodeSchemaCacheTest.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java index b2aaa910ffc11..decc17be1ae0e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java @@ -31,7 +31,6 @@ import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; @@ -211,8 +210,7 @@ public void testUpdateLastCache() throws IllegalPathException { // Simulate "s1" revert when the query has failed in calculation dataNodeSchemaCache.updateLastCacheIfExists( database, - IDeviceID.Factory.DEFAULT_FACTORY.create( - StringArrayDeviceID.splitDeviceIdString(device.getNodes())), + device, new String[] {"s1"}, new TimeValuePair[] { new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), @@ -225,8 +223,7 @@ public void testUpdateLastCache() throws IllegalPathException { // "s2" shall be null since the "null" timeValuePair has not been put dataNodeSchemaCache.updateLastCacheIfExists( database, - IDeviceID.Factory.DEFAULT_FACTORY.create( - StringArrayDeviceID.splitDeviceIdString(device.getNodes())), + device, new String[] {"s2"}, new TimeValuePair[] {tv1}, false, @@ -234,8 +231,7 @@ public void testUpdateLastCache() throws IllegalPathException { dataNodeSchemaCache.updateLastCacheIfExists( database, - IDeviceID.Factory.DEFAULT_FACTORY.create( - StringArrayDeviceID.splitDeviceIdString(device.getNodes())), + device, new String[] {"s3"}, new TimeValuePair[] {tv1}, false, @@ -249,8 +245,7 @@ public void testUpdateLastCache() throws IllegalPathException { dataNodeSchemaCache.updateLastCacheIfExists( database, - IDeviceID.Factory.DEFAULT_FACTORY.create( - StringArrayDeviceID.splitDeviceIdString(device.getNodes())), + device, measurements, new TimeValuePair[] { new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), From 42d916474fee99aa16fff9dc6ce73fbdfb3c2d39 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:54:58 +0800 Subject: [PATCH 18/33] fix --- .../cache/schema/DataNodeSchemaCache.java | 4 +- ...eCacheEntry.java => DeviceCacheEntry.java} | 4 +- .../cache/schema/DeviceSchemaCache.java | 42 +++++-------------- 3 files changed, 15 insertions(+), 35 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/{TableDeviceCacheEntry.java => DeviceCacheEntry.java} (98%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 78b4f98f688c1..1e07cce7548bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -370,8 +370,8 @@ public void invalidateDatabaseLastCache(final String database) { /** * Update the {@link DeviceLastCache} in writing for tree model. If a measurement is with all * {@code null}s or is an id/attribute column, its {@link TimeValuePair}[] shall be {@code null}. - * For correctness, this will put the {@link TableDeviceCacheEntry} lazily and only update the - * existing {@link DeviceLastCache}s of measurements. + * For correctness, this will put the {@link DeviceCacheEntry} lazily and only update the existing + * {@link DeviceLastCache}s of measurements. * * @param database the device's database, WITH "root" * @param deviceID {@link IDeviceID} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java similarity index 98% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java index 55e17700a5489..8dff7b5456168 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TableDeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java @@ -36,10 +36,10 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; @ThreadSafe -public class TableDeviceCacheEntry { +public class DeviceCacheEntry { private static final long INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(TableDeviceCacheEntry.class) + RamUsageEstimator.shallowSizeOfInstance(DeviceCacheEntry.class) + 2 * RamUsageEstimator.shallowSizeOfInstance(AtomicReference.class); // the cached attributeMap may not be the latest, but there won't be any correctness problems diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index 0a0b54cf74ab8..c73926158b0e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -29,7 +29,6 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; -import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegion; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.file.metadata.IDeviceID; @@ -50,22 +49,8 @@ import java.util.function.ToIntFunction; /** - * The {@link DeviceSchemaCache} caches some of the devices and their: attributes of tables / - * measurement info / template info. The last value of one device is also cached here. Here are the - * semantics of attributes, other semantics are omitted: - * - *

1. If a device misses cache, it does not necessarily mean that the device does not exist, - * Since the cache records only part of the devices. - * - *

2. If a device is in cache, the attributes will be finally identical to the {@link - * SchemaRegion}'s version. In reading this may temporarily return false result, and in writing when - * the new attribute differs from the original ones, we send the new attributes to the {@link - * SchemaRegion} anyway. This may not update the attributes in {@link SchemaRegion} since the - * attributes in cache may be stale, but it's okay and {@link SchemaRegion} will just do nothing. - * - *

3. When the attributeMap does not contain an attributeKey, then the value is {@code null}. - * Note that we do not tell whether an attributeKey exists here, and it shall be judged from table - * schema. + * The {@link DeviceSchemaCache} caches some of the devices and their: measurement info / template + * info. The last value of one device is also cached here. */ @ThreadSafe public class DeviceSchemaCache { @@ -83,7 +68,7 @@ public class DeviceSchemaCache { * *

2. Optimize the speed in invalidation by databases for most scenarios. */ - private final IDualKeyCache dualKeyCache; + private final IDualKeyCache dualKeyCache; private final Map databasePool = new ConcurrentHashMap<>(); @@ -91,13 +76,13 @@ public class DeviceSchemaCache { DeviceSchemaCache() { dualKeyCache = - new DualKeyCacheBuilder() + new DualKeyCacheBuilder() .cacheEvictionPolicy( DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) .memoryCapacity(config.getAllocateMemoryForSchemaCache()) .firstKeySizeComputer(segment -> (int) RamUsageEstimator.sizeOf(segment)) .secondKeySizeComputer(PartialPath::estimateSize) - .valueSizeComputer(TableDeviceCacheEntry::estimateSize) + .valueSizeComputer(DeviceCacheEntry::estimateSize) .build(); MetricService.getInstance().addMetricSet(new DataNodeSchemaCacheMetrics(this)); } @@ -113,7 +98,7 @@ public class DeviceSchemaCache { * hit but result is {@code null}, and the result value otherwise. */ public TimeValuePair getLastEntry(final PartialPath device, final String measurement) { - final TableDeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); + final DeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); return Objects.nonNull(entry) ? entry.getTimeValuePair(measurement) : null; } @@ -136,7 +121,7 @@ public void putDeviceSchema(final String database, final DeviceSchemaInfo device dualKeyCache.update( getLeadingSegment(devicePath), devicePath, - new TableDeviceCacheEntry(), + new DeviceCacheEntry(), entry -> entry.setDeviceSchema( Objects.nonNull(previousDatabase) ? previousDatabase : database, deviceSchemaInfo), @@ -144,7 +129,7 @@ public void putDeviceSchema(final String database, final DeviceSchemaInfo device } public IDeviceSchema getDeviceSchema(final PartialPath device) { - final TableDeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); + final DeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); return Objects.nonNull(entry) ? entry.getDeviceSchema() : null; } @@ -162,7 +147,7 @@ void updateLastCache( dualKeyCache.update( getLeadingSegment(device), device, - new TableDeviceCacheEntry(), + new DeviceCacheEntry(), initOrInvalidate ? entry -> entry.setMeasurementSchema( @@ -177,12 +162,11 @@ void updateLastCache( public boolean getLastCache( final Map>> inputMap) { - return dualKeyCache.batchApply(inputMap, TableDeviceCacheEntry::updateInputMap); + return dualKeyCache.batchApply(inputMap, DeviceCacheEntry::updateInputMap); } - // WARNING: This is not guaranteed to affect table model's cache void invalidateLastCache(final PartialPath devicePath, final String measurement) { - final ToIntFunction updateFunction = + final ToIntFunction updateFunction = PathPatternUtil.hasWildcard(measurement) ? entry -> -entry.invalidateLastCache() : entry -> -entry.invalidateLastCache(measurement); @@ -211,7 +195,6 @@ void invalidateLastCache(final PartialPath devicePath, final String measurement) } } - // WARNING: This is not guaranteed to affect table model's cache void invalidateCache( final @Nonnull PartialPath devicePath, final boolean isMultiLevelWildcardMeasurement) { if (!devicePath.hasWildcard()) { @@ -336,11 +319,8 @@ public static String getLeadingSegment(final PartialPath device) { } } if (lastSeparatorPos == -1) { - // not find even one separator, probably during a test, use the device as the tableName segment = deviceStr; } else { - // use the first DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME segments or all segments but the last - // one as the table name segment = deviceStr.substring(0, lastSeparatorPos); } From 3de1d65f09e6004de2860e3286f6dbc381e2a8b1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 16 Sep 2025 18:56:15 +0800 Subject: [PATCH 19/33] de-table --- .../queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java | 1 - .../queryengine/plan/analyze/cache/schema/DeviceLastCache.java | 1 - 2 files changed, 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java index 8dff7b5456168..a12b9c7661b79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java @@ -137,7 +137,6 @@ TimeValuePair getTimeValuePair(final String measurement) { } boolean updateInputMap(final @Nonnull Map updateMap) { - // Shall only call this for original table device for (final String measurement : updateMap.keySet()) { final TimeValuePair result = getTimeValuePair(measurement); if (result == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java index 8d3d7e754a23b..e038feedf1393 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java @@ -86,7 +86,6 @@ int initOrInvalidate(final String[] measurements, final boolean isInvalidate) { final AtomicInteger diff = new AtomicInteger(0); for (final String measurement : measurements) { - // Removing table measurement, do not put cache if (Objects.isNull(measurement)) { continue; } From 3a3464f8a32eac7747b5180cc112a0259726beb2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 17 Sep 2025 10:25:49 +0800 Subject: [PATCH 20/33] fix --- .../thrift/impl/ClientRPCServiceImpl.java | 19 +++++++++++-------- .../last/AbstractUpdateLastCacheOperator.java | 7 ++++++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 95dbcb1655472..3e1d68ac3f32c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -70,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DeviceLastCache; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; @@ -847,14 +848,16 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath( for (final Map.Entry measurementLastEntry : device2MeasurementLastEntry.getValue().entrySet()) { final TimeValuePair tvPair = measurementLastEntry.getValue(); - LastQueryUtil.appendLastValue( - builder, - tvPair.getTimestamp(), - new Binary( - deviceWithSeparator + measurementLastEntry.getKey(), - TSFileConfig.STRING_CHARSET), - tvPair.getValue().getStringValue(), - tvPair.getValue().getDataType().name()); + if (tvPair != DeviceLastCache.EMPTY_TIME_VALUE_PAIR) { + LastQueryUtil.appendLastValue( + builder, + tvPair.getTimestamp(), + new Binary( + deviceWithSeparator + measurementLastEntry.getKey(), + TSFileConfig.STRING_CHARSET), + tvPair.getValue().getStringValue(), + tvPair.getValue().getDataType().name()); + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java index 9d718198622c3..654e881307db5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java @@ -134,8 +134,13 @@ protected void mayUpdateLastCache( return; } // update cache in DataNodeQueryContext + // update cache in DataNodeQueryContext if (seriesScanInfo.right == null || time > seriesScanInfo.right.getTimestamp()) { - seriesScanInfo.right = new TimeValuePair(time, value); + if (Objects.nonNull(value)) { + seriesScanInfo.right = new TimeValuePair(time, value); + } else { + seriesScanInfo.right = needUpdateNullEntry ? DeviceLastCache.EMPTY_TIME_VALUE_PAIR : null; + } } if (seriesScanInfo.left.decrementAndGet() == 0) { From b44482efe8b2b82392311503ec88de7811cb938f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 17 Sep 2025 10:26:37 +0800 Subject: [PATCH 21/33] fix --- .../operator/process/last/AbstractUpdateLastCacheOperator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java index 654e881307db5..ce8d61625ee79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java @@ -134,7 +134,6 @@ protected void mayUpdateLastCache( return; } // update cache in DataNodeQueryContext - // update cache in DataNodeQueryContext if (seriesScanInfo.right == null || time > seriesScanInfo.right.getTimestamp()) { if (Objects.nonNull(value)) { seriesScanInfo.right = new TimeValuePair(time, value); From 48739f476f4b4ab4d2be797057900d1411b677ce Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 17 Sep 2025 12:26:41 +0800 Subject: [PATCH 22/33] Fix --- .../analyze/cache/schema/DeviceSchemaCache.java | 14 ++++++++++---- .../planner/plan/node/write/InsertRowNode.java | 6 ++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index c73926158b0e8..f97a5cd2bc050 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -46,6 +46,7 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Predicate; import java.util.function.ToIntFunction; /** @@ -247,17 +248,22 @@ long entriesCount() { // Note: It might be very slow if the database is too long void invalidateLastCache(final @Nonnull String database) { - readWriteLock.writeLock().lock(); + Predicate secondKeyChecker; + try { + final PartialPath databasePath = new PartialPath(database); + secondKeyChecker = device -> device.matchPrefixPath(databasePath); + } catch (final Exception ignored) { + secondKeyChecker = device -> device.startsWith(database); + } + readWriteLock.writeLock().lock(); try { dualKeyCache.update( segment -> segment.startsWith(database), device -> true, entry -> -entry.invalidateLastCache()); dualKeyCache.update( - database::startsWith, - device -> device.startsWith(database), - entry -> -entry.invalidateLastCache()); + database::startsWith, secondKeyChecker, entry -> -entry.invalidateLastCache()); } finally { readWriteLock.writeLock().unlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index 8cbf48b62ac88..6518385c48a86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -855,11 +855,13 @@ public R accept(PlanVisitor visitor, C context) { } public TimeValuePair composeTimeValuePair(int columnIndex) { - if (columnIndex >= values.length) { + if (columnIndex >= values.length || Objects.isNull(dataTypes[columnIndex])) { return null; } Object value = values[columnIndex]; - return new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)); + return Objects.nonNull(value) + ? new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)) + : null; } public void updateLastCache(String databaseName) { From 0e531adaa2c015785003ce6f0b01c81a221d831b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 10:25:54 +0800 Subject: [PATCH 23/33] fix --- .../cache/schema/DataNodeSchemaCache.java | 14 +++++------ .../cache/schema/DeviceSchemaCache.java | 24 +++++++++---------- .../dualkeycache/impl/ICacheEntryManager.java | 6 ++--- .../LoadTsFileDataTypeConverter.java | 2 +- .../db/utils/datastructure/AlignedTVList.java | 2 +- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 1e07cce7548bf..1c65e5ebb51fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -269,10 +269,9 @@ public List computeWithTemplate(final ISchemaComputation computation) { final DeviceTemplateSchema deviceTemplateSchema = (DeviceTemplateSchema) deviceSchema; - computation.computeDevice( - templateManager.getTemplate(deviceTemplateSchema.getTemplateId()).isDirectAligned()); - final Map templateSchema = - templateManager.getTemplate(deviceTemplateSchema.getTemplateId()).getSchemaMap(); + final Template template = templateManager.getTemplate(deviceTemplateSchema.getTemplateId()); + computation.computeDevice(template.isDirectAligned()); + final Map templateSchema = template.getSchemaMap(); for (int i = 0; i < measurements.length; i++) { if (!templateSchema.containsKey(measurements[i])) { indexOfMissingMeasurements.add(i); @@ -368,10 +367,9 @@ public void invalidateDatabaseLastCache(final String database) { } /** - * Update the {@link DeviceLastCache} in writing for tree model. If a measurement is with all - * {@code null}s or is an id/attribute column, its {@link TimeValuePair}[] shall be {@code null}. - * For correctness, this will put the {@link DeviceCacheEntry} lazily and only update the existing - * {@link DeviceLastCache}s of measurements. + * Update the {@link DeviceLastCache} in writing. If a measurement is with all {@code null}s, its + * {@link TimeValuePair}[] shall be {@code null}. For correctness, this will put the {@link + * DeviceCacheEntry} lazily and only update the existing {@link DeviceLastCache}s of measurements. * * @param database the device's database, WITH "root" * @param deviceID {@link IDeviceID} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index f97a5cd2bc050..0ab796071ca18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -45,7 +45,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; import java.util.function.ToIntFunction; @@ -73,7 +73,7 @@ public class DeviceSchemaCache { private final Map databasePool = new ConcurrentHashMap<>(); - private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); + private final ReentrantLock lock = new ReentrantLock(false); DeviceSchemaCache() { dualKeyCache = @@ -256,7 +256,7 @@ void invalidateLastCache(final @Nonnull String database) { secondKeyChecker = device -> device.startsWith(database); } - readWriteLock.writeLock().lock(); + lock.lock(); try { dualKeyCache.update( segment -> segment.startsWith(database), @@ -265,45 +265,45 @@ void invalidateLastCache(final @Nonnull String database) { dualKeyCache.update( database::startsWith, secondKeyChecker, entry -> -entry.invalidateLastCache()); } finally { - readWriteLock.writeLock().unlock(); + lock.unlock(); } } // Note: It might be very slow if the database is too long public void invalidate(final @Nonnull String database) { - readWriteLock.writeLock().lock(); + lock.lock(); try { dualKeyCache.invalidate(segment -> segment.startsWith(database), device -> true); dualKeyCache.invalidate(database::startsWith, device -> device.startsWith(database)); } finally { - readWriteLock.writeLock().unlock(); + lock.unlock(); } } public void invalidateLastCache() { - readWriteLock.writeLock().lock(); + lock.lock(); try { dualKeyCache.update(segment -> true, device -> true, entry -> -entry.invalidateLastCache()); } finally { - readWriteLock.writeLock().unlock(); + lock.unlock(); } } public void invalidateTreeSchema() { - readWriteLock.writeLock().lock(); + lock.lock(); try { dualKeyCache.update(segment -> true, device -> true, entry -> -entry.invalidateSchema()); } finally { - readWriteLock.writeLock().unlock(); + lock.unlock(); } } public void invalidateAll() { - readWriteLock.writeLock().lock(); + lock.lock(); try { dualKeyCache.invalidateAll(); } finally { - readWriteLock.writeLock().unlock(); + lock.unlock(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java index e1b1f85026ebc..0117f836ba926 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java @@ -37,11 +37,11 @@ T createCacheEntry( void put(final T cacheEntry); - // A cacheEntry is removed iff the caller has called "invalid" and it returns "true" - // Shall never remove a cacheEntry directly or when the "invalid" returns false + // A cacheEntry is removed iff the caller has called "invalidate" and it returns "true" + // Shall never remove a cacheEntry directly or when the "invalidate" returns false boolean invalidate(final T cacheEntry); - // The "evict" is allowed to be synchronized called + // The "evict" is allowed to be concurrently called // Inner implementation guarantees that an entry won't be concurrently evicted T evict(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java index 81a1f0639beba..883de7958dca1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java @@ -68,7 +68,7 @@ public Optional convertForTreeModel(final LoadTsFileStatement loadTsFi return loadTsFileTreeStatement.accept(treeStatementDataTypeConvertExecutionVisitor, null); } catch (Exception e) { LOGGER.warn( - "Failed to convert data types for tree model statement {}.", loadTsFileTreeStatement, e); + "Failed to convert data types for load statement {}.", loadTsFileTreeStatement, e); return Optional.of( new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage())); } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 4971775fefa34..117cb3a89615e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -79,7 +79,7 @@ public abstract class AlignedTVList extends TVList { // Index relation: columnIndex(dataTypeIndex) -> arrayIndex -> elementIndex protected List> bitMaps; - // not null when constructed by queries for tree model + // not null when constructed by queries BitMap allValueColDeletedMap; private final AlignedTVList outer = this; From a84a16c5655ba760656f579fa1680f894c545943 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 10:29:16 +0800 Subject: [PATCH 24/33] fix --- .../plan/analyze/cache/schema/DataNodeSchemaCache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 1c65e5ebb51fe..717209618a1eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -409,8 +409,8 @@ public void updateLastCacheIfExists( * * @param database the device's database, WITH "root" * @param measurementPath the fetched {@link MeasurementPath} - * @param isInvalidate {@code true} if invalidate the first pushed cache, or {@code null} for the - * first fetch. + * @param isInvalidate {@code true} if invalidate the first pushed cache, or {@code false} for the + * first push. */ public void updateLastCache( final String database, final MeasurementPath measurementPath, final boolean isInvalidate) { From ea3146c16a8fd5b13dc2d21eb8b1790fd8aec4c8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 11:15:12 +0800 Subject: [PATCH 25/33] fix --- .../plan/analyze/cache/schema/DeviceSchemaCache.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index 0ab796071ca18..722de7b541b30 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -117,15 +117,13 @@ public void invalidateDeviceLastCache(final PartialPath device) { public void putDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { final PartialPath devicePath = deviceSchemaInfo.getDevicePath(); - final String previousDatabase = databasePool.putIfAbsent(database, database); + final String databaseToUse = databasePool.computeIfAbsent(database, k -> database); dualKeyCache.update( getLeadingSegment(devicePath), devicePath, new DeviceCacheEntry(), - entry -> - entry.setDeviceSchema( - Objects.nonNull(previousDatabase) ? previousDatabase : database, deviceSchemaInfo), + entry -> entry.setDeviceSchema(databaseToUse, deviceSchemaInfo), true); } @@ -142,8 +140,7 @@ void updateLastCache( final boolean isAligned, final IMeasurementSchema[] measurementSchemas, final boolean initOrInvalidate) { - final String previousDatabase = databasePool.putIfAbsent(database, database); - final String database2Use = Objects.nonNull(previousDatabase) ? previousDatabase : database; + final String database2Use = databasePool.computeIfAbsent(database, k -> database); dualKeyCache.update( getLeadingSegment(device), From c778c5d94caf69c448da4acaca5a6ca24db46c00 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 11:22:51 +0800 Subject: [PATCH 26/33] fix --- .../protocol/rest/v2/impl/RestApiServiceImpl.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java index 8712cbb8f38d7..0a0eb119b1313 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DeviceLastCache; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; @@ -173,14 +174,17 @@ public Response executeFastLastQueryStatement( resultMap.entrySet()) { for (final Map.Entry> device2MeasurementLastEntry : entry.getValue().entrySet()) { - final String deviceWithPrefix = entry.getKey() + TsFileConstant.PATH_SEPARATOR; + final String deviceWithSeparator = + device2MeasurementLastEntry.getKey() + TsFileConstant.PATH_SEPARATOR; for (Map.Entry measurementEntry : device2MeasurementLastEntry.getValue().entrySet()) { final TimeValuePair tvPair = measurementEntry.getValue(); - valueList.add(tvPair.getValue().getStringValue()); - dataTypeList.add(tvPair.getValue().getDataType().name()); - targetDataSet.addTimestampsItem(tvPair.getTimestamp()); - timeseries.add(deviceWithPrefix + measurementEntry.getKey()); + if (tvPair != DeviceLastCache.EMPTY_TIME_VALUE_PAIR) { + valueList.add(tvPair.getValue().getStringValue()); + dataTypeList.add(tvPair.getValue().getDataType().name()); + targetDataSet.addTimestampsItem(tvPair.getTimestamp()); + timeseries.add(deviceWithSeparator + measurementEntry.getKey()); + } } } } From 9a9502f647c3d264251ae3bcf687375d743cc36f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 11:52:13 +0800 Subject: [PATCH 27/33] Refactor --- .../plan/analyze/cache/schema/DataNodeSchemaCache.java | 4 ++-- .../plan/analyze/cache/schema/DeviceLastCache.java | 3 +++ .../iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java | 6 +++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 717209618a1eb..fd59d46f3610d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -127,7 +127,7 @@ public ClusterSchemaTree get(final PartialPath devicePath, final String[] measur * @param devicePath full path of the device * @return empty if cache miss or the device path is not a template activated path */ - public ClusterSchemaTree getMatchedSchemaWithTemplate(final PartialPath devicePath) { + public ClusterSchemaTree getMatchedTemplateSchema(final PartialPath devicePath) { final ClusterSchemaTree tree = new ClusterSchemaTree(); final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath); if (!(schema instanceof DeviceTemplateSchema)) { @@ -146,7 +146,7 @@ public ClusterSchemaTree getMatchedSchemaWithTemplate(final PartialPath devicePa * @param fullPath full path * @return empty if cache miss */ - public ClusterSchemaTree getMatchedSchemaWithoutTemplate(final PartialPath fullPath) { + public ClusterSchemaTree getMatchedNormalSchema(final PartialPath fullPath) { final ClusterSchemaTree tree = new ClusterSchemaTree(); final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevicePath()); if (!(schema instanceof DeviceNormalSchema)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java index e038feedf1393..8b5003eca0502 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java @@ -74,8 +74,11 @@ public TSDataType getDataType() { } }; + /** This means that the tv pair has been put, and the value is null */ public static final TimeValuePair EMPTY_TIME_VALUE_PAIR = new TimeValuePair(Long.MIN_VALUE, EMPTY_PRIMITIVE_TYPE); + + /** This means that the tv pair has been put once, and is ready for the next put. */ private static final TimeValuePair PLACEHOLDER_TIME_VALUE_PAIR = new TimeValuePair(Long.MIN_VALUE, EMPTY_PRIMITIVE_TYPE); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java index decc17be1ae0e..c652f813c6c6f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java @@ -290,11 +290,11 @@ public void testPut() throws Exception { new MeasurementPath("root.sg1.d3.s1", TSDataType.FLOAT)); dataNodeSchemaCache.put(clusterSchemaTree); final ClusterSchemaTree d1Tree = - dataNodeSchemaCache.getMatchedSchemaWithTemplate(new PartialPath("root.sg1.d1")); + dataNodeSchemaCache.getMatchedTemplateSchema(new PartialPath("root.sg1.d1")); final ClusterSchemaTree d2Tree = - dataNodeSchemaCache.getMatchedSchemaWithTemplate(new PartialPath("root.sg1.d2")); + dataNodeSchemaCache.getMatchedTemplateSchema(new PartialPath("root.sg1.d2")); final ClusterSchemaTree d3Tree = - dataNodeSchemaCache.getMatchedSchemaWithoutTemplate(new MeasurementPath("root.sg1.d3.s1")); + dataNodeSchemaCache.getMatchedNormalSchema(new MeasurementPath("root.sg1.d3.s1")); List measurementPaths = d1Tree.searchMeasurementPaths(ALL_MATCH_PATTERN).left; Assert.assertEquals(2, measurementPaths.size()); for (final MeasurementPath measurementPath : measurementPaths) { From fb9ae10b72f7a34944d7dde18eef14d8296b5776 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 11:59:18 +0800 Subject: [PATCH 28/33] fix --- .../queryengine/plan/analyze/schema/ClusterSchemaFetcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java index 13a9f5fc97a6d..cb6ae8ec89719 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java @@ -119,7 +119,7 @@ public ClusterSchemaTree fetchSchema( Set storageGroupSet = new HashSet<>(); if (!explicitDevicePatternList.isEmpty()) { for (PartialPath explicitDevicePattern : explicitDevicePatternList) { - cachedSchema = schemaCache.getMatchedSchemaWithTemplate(explicitDevicePattern); + cachedSchema = schemaCache.getMatchedTemplateSchema(explicitDevicePattern); if (cachedSchema.isEmpty()) { isAllCached = false; break; @@ -132,7 +132,7 @@ public ClusterSchemaTree fetchSchema( if (isAllCached && !explicitPathList.isEmpty()) { for (PartialPath fullPath : explicitPathList) { - cachedSchema = schemaCache.getMatchedSchemaWithoutTemplate(fullPath); + cachedSchema = schemaCache.getMatchedNormalSchema(fullPath); if (cachedSchema.isEmpty()) { isAllCached = false; break; From 6aa1c07b1ca51c7a4680eae1f1a46489dafc50bd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:07:11 +0800 Subject: [PATCH 29/33] fix --- .../cache/schema/DataNodeSchemaCache.java | 114 +++++++++--------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index fd59d46f3610d..03b9fd2ee79b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -278,60 +278,7 @@ public List computeWithTemplate(final ISchemaComputation computation) { continue; } final IMeasurementSchema schema = templateSchema.get(measurements[i]); - computation.computeMeasurement( - i, - new IMeasurementSchemaInfo() { - @Override - public String getName() { - return schema.getMeasurementId(); - } - - @Override - public IMeasurementSchema getSchema() { - if (isLogicalView()) { - return new LogicalViewSchema( - schema.getMeasurementId(), ((LogicalViewSchema) schema).getExpression()); - } else { - return this.getSchemaAsMeasurementSchema(); - } - } - - @Override - public MeasurementSchema getSchemaAsMeasurementSchema() { - return new MeasurementSchema( - schema.getMeasurementId(), - schema.getType(), - schema.getEncodingType(), - schema.getCompressor()); - } - - @Override - public LogicalViewSchema getSchemaAsLogicalViewSchema() { - throw new RuntimeException( - new UnsupportedOperationException( - "Function getSchemaAsLogicalViewSchema is not supported in DeviceUsingTemplateSchemaCache.")); - } - - @Override - public Map getTagMap() { - return null; - } - - @Override - public Map getAttributeMap() { - return null; - } - - @Override - public String getAlias() { - return null; - } - - @Override - public boolean isLogicalView() { - return schema.isLogicalView(); - } - }); + computation.computeMeasurement(i, new WrappedSchemaInfo(schema)); } return indexOfMissingMeasurements; } @@ -443,4 +390,63 @@ public DeviceSchemaCache getDeviceSchemaCache() { public void cleanUp() { deviceSchemaCache.invalidateAll(); } + + private static class WrappedSchemaInfo implements IMeasurementSchemaInfo { + private final IMeasurementSchema schema; + + public WrappedSchemaInfo(final IMeasurementSchema schema) { + this.schema = schema; + } + + @Override + public String getName() { + return schema.getMeasurementId(); + } + + @Override + public IMeasurementSchema getSchema() { + if (isLogicalView()) { + return new LogicalViewSchema( + schema.getMeasurementId(), ((LogicalViewSchema) schema).getExpression()); + } else { + return this.getSchemaAsMeasurementSchema(); + } + } + + @Override + public MeasurementSchema getSchemaAsMeasurementSchema() { + return new MeasurementSchema( + schema.getMeasurementId(), + schema.getType(), + schema.getEncodingType(), + schema.getCompressor()); + } + + @Override + public LogicalViewSchema getSchemaAsLogicalViewSchema() { + throw new RuntimeException( + new UnsupportedOperationException( + "Function getSchemaAsLogicalViewSchema is not supported in DeviceUsingTemplateSchemaCache.")); + } + + @Override + public Map getTagMap() { + return null; + } + + @Override + public Map getAttributeMap() { + return null; + } + + @Override + public String getAlias() { + return null; + } + + @Override + public boolean isLogicalView() { + return schema.isLogicalView(); + } + } } From 2df33fe06b786a393046ae72b863395f2b69663e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:16:07 +0800 Subject: [PATCH 30/33] f --- .../IoTDBRemoveConfigNodeITFramework.java | 2 +- .../IoTDBRemoveDataNodeITFramework.java | 2 +- .../thrift/IoTDBDataNodeReceiver.java | 3 +- .../cache/schema/DataNodeSchemaCache.java | 37 +++++++++++++------ .../cache/schema/DeviceSchemaCache.java | 2 - .../plan/analyze/load/LoadTsFileAnalyzer.java | 2 +- .../plan/planner/OperatorTreeGenerator.java | 10 ++--- .../cache/DataNodeSchemaCacheTest.java | 12 +++--- 8 files changed, 39 insertions(+), 31 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java index 561e99ec3da39..4cca7ceb9fa9f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java @@ -99,7 +99,7 @@ public void testRemoveConfigNode( SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - // Insert data in tree model + // Insert data statement.execute(TREE_MODEL_INSERTION); Map> regionMap = getDataRegionMap(statement); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java index 73d5fd6fac353..b12c15283ed3c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java @@ -154,7 +154,7 @@ public void testRemoveDataNode( SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - // Insert data in tree model + // Insert data statement.execute(TREE_MODEL_INSERTION); Map> regionMap = getDataRegionMap(statement); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index 44ba9eaedfb96..f2e71ba319f39 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -715,8 +715,7 @@ private TSStatus executeStatementWithRetryOnDataTypeMismatch(final Statement sta final TSStatus status = executeStatement(statement); - // Try to convert data type if the statement is a tree model statement - // and the status code is not success + // Try to convert the data type and the status code is not success return shouldConvertDataTypeOnTypeMismatch && ((statement instanceof InsertBaseStatement && ((InsertBaseStatement) statement).hasFailedMeasurements()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 03b9fd2ee79b0..a015f6ebb502f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -335,14 +335,14 @@ public void updateLastCacheIfExists( } /** - * Update the {@link DeviceLastCache} on query in tree model. + * Update the {@link DeviceLastCache} on query. * *

Note: The query shall put the {@link DeviceLastCache} twice: * - *

- First time set the "isCommit" to {@code false} before the query accesses data. It is just - * to allow the writing to update the cache, then avoid that the query put a stale value to cache - * and break the consistency. WARNING: The writing may temporarily put a stale value in cache if a - * stale value is written, but it won't affect the eventual consistency. + *

- First time call this before the query accesses data. It is just to allow the writing to + * update the cache, then avoid that the query put a stale value to cache and break the + * consistency. WARNING: The writing may temporarily put a stale value in cache if a stale value + * is written, but it won't affect the eventual consistency. * *

- Second time put the calculated {@link TimeValuePair}, and use {@link * #updateLastCacheIfExists(String, PartialPath, String[], TimeValuePair[], boolean, @@ -350,22 +350,35 @@ public void updateLastCacheIfExists( * if the measurement is with all {@code null}s, its {@link TimeValuePair} shall be {@link * DeviceLastCache#EMPTY_TIME_VALUE_PAIR}. This method is not supposed to update time column. * + * @param database the device's database, WITH "root" + * @param measurementPath the fetched {@link MeasurementPath} + */ + public void declareLastCache(final String database, final MeasurementPath measurementPath) { + deviceSchemaCache.updateLastCache( + database, + measurementPath.getDevicePath(), + new String[] {measurementPath.getMeasurement()}, + null, + measurementPath.isUnderAlignedEntity(), + new IMeasurementSchema[] {measurementPath.getMeasurementSchema()}, + true); + } + + /** + * Update the {@link DeviceLastCache} on query. + * *

If the query has ended abnormally, it shall call this to invalidate the entry it has pushed - * in the first time, to avoid the stale writing damaging the eventual consistency. In this case - * and the "isInvalidate" shall be {@code true}. + * in the first time, to avoid the stale writing damaging the eventual consistency. * * @param database the device's database, WITH "root" * @param measurementPath the fetched {@link MeasurementPath} - * @param isInvalidate {@code true} if invalidate the first pushed cache, or {@code false} for the - * first push. */ - public void updateLastCache( - final String database, final MeasurementPath measurementPath, final boolean isInvalidate) { + public void invalidateLastCache(final String database, final MeasurementPath measurementPath) { deviceSchemaCache.updateLastCache( database, measurementPath.getDevicePath(), new String[] {measurementPath.getMeasurement()}, - isInvalidate ? new TimeValuePair[] {null} : null, + new TimeValuePair[] {null}, measurementPath.isUnderAlignedEntity(), new IMeasurementSchema[] {measurementPath.getMeasurementSchema()}, true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index 722de7b541b30..8d66cb3d619ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -113,8 +113,6 @@ public void invalidateDeviceLastCache(final PartialPath device) { getLeadingSegment(device), device, null, entry -> -entry.invalidateLastCache(), false); } - /////////////////////////////// Tree model /////////////////////////////// - public void putDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { final PartialPath devicePath = deviceSchemaInfo.getDevicePath(); final String databaseToUse = databasePool.computeIfAbsent(database, k -> database); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index a02393d390699..e621c2b445292 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -767,7 +767,7 @@ private void executeSetDatabaseStatement(Statement statement) if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode() && result.status.code != TSStatusCode.DATABASE_ALREADY_EXISTS - .getStatusCode() // In tree model, if the user creates a conflict database + .getStatusCode() // If the user creates a conflict database // concurrently, for instance, the // database created by user is root.db.ss.a, the auto-creation failed database is root.db, // we wait till "getOrCreatePartition" to judge if the time series (like root.db.ss.a.e / diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index d4a2a8d70ad60..2f1b9671e31f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -2777,12 +2777,11 @@ private UpdateLastCacheOperator createUpdateLastCacheOperator( final boolean isNeedUpdateLastCache = context.isNeedUpdateLastCache(); if (isNeedUpdateLastCache) { DataNodeSchemaCache.getInstance() - .updateLastCache( + .declareLastCache( ((DataDriverContext) operatorContext.getDriverContext()) .getDataRegion() .getDatabaseName(), - fullPath, - false); + fullPath); } return Objects.isNull(node.getOutputViewPath()) @@ -2833,13 +2832,12 @@ private AlignedUpdateLastCacheOperator createAlignedUpdateLastCacheOperator( for (int i = 0; i < size; ++i) { DataNodeSchemaCache.getInstance() - .updateLastCache( + .declareLastCache( databaseName, new MeasurementPath( devicePath.concatNode(unCachedPath.getMeasurementList().get(i)), unCachedPath.getSchemaList().get(i), - true), - false); + true)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java index c652f813c6c6f..d2f11d65d1fea 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java @@ -202,10 +202,10 @@ public void testUpdateLastCache() throws IllegalPathException { final TimeValuePair tv1 = new TimeValuePair(1, new TsPrimitiveType.TsInt(1)); - dataNodeSchemaCache.updateLastCache( - database, new MeasurementPath(device.concatNode("s1"), s1), false); - dataNodeSchemaCache.updateLastCache( - database, new MeasurementPath(device.concatNode("s3"), s3), false); + dataNodeSchemaCache.declareLastCache( + database, new MeasurementPath(device.concatNode("s1"), s1)); + dataNodeSchemaCache.declareLastCache( + database, new MeasurementPath(device.concatNode("s3"), s3)); // Simulate "s1" revert when the query has failed in calculation dataNodeSchemaCache.updateLastCacheIfExists( @@ -217,8 +217,8 @@ public void testUpdateLastCache() throws IllegalPathException { }, false, new MeasurementSchema[] {s1}); - dataNodeSchemaCache.updateLastCache( - database, new MeasurementPath(device.concatNode("s1"), s1), true); + dataNodeSchemaCache.invalidateLastCache( + database, new MeasurementPath(device.concatNode("s1"), s1)); // "s2" shall be null since the "null" timeValuePair has not been put dataNodeSchemaCache.updateLastCacheIfExists( From df8b3a3391a4954679598722fb2eef938b15fad4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:18:26 +0800 Subject: [PATCH 31/33] declared --- .../common/schematree/DeviceSchemaInfo.java | 12 ++++-------- .../plan/analyze/cache/schema/DeviceLastCache.java | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/DeviceSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/DeviceSchemaInfo.java index 8bb8a8f4ab045..fc36f38941cdf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/DeviceSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/DeviceSchemaInfo.java @@ -30,16 +30,12 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; - public class DeviceSchemaInfo { - private PartialPath devicePath; - private boolean isAligned; - private List measurementSchemaInfoList; - private int templateId = NON_TEMPLATE; - - private DeviceSchemaInfo() {} + private final PartialPath devicePath; + private final boolean isAligned; + private final List measurementSchemaInfoList; + private final int templateId; public DeviceSchemaInfo( PartialPath devicePath, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java index 8b5003eca0502..a2f7696b6dfb5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java @@ -78,7 +78,7 @@ public TSDataType getDataType() { public static final TimeValuePair EMPTY_TIME_VALUE_PAIR = new TimeValuePair(Long.MIN_VALUE, EMPTY_PRIMITIVE_TYPE); - /** This means that the tv pair has been put once, and is ready for the next put. */ + /** This means that the tv pair has been declared, and is ready for the next put. */ private static final TimeValuePair PLACEHOLDER_TIME_VALUE_PAIR = new TimeValuePair(Long.MIN_VALUE, EMPTY_PRIMITIVE_TYPE); From 05e3ed1b3e906d2b3bee28254cfc14d0f9938572 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:44:09 +0800 Subject: [PATCH 32/33] fix --- .../queryengine/plan/analyze/schema/ClusterSchemaFetcher.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java index cb6ae8ec89719..c9914e0d7e9af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java @@ -132,6 +132,10 @@ public ClusterSchemaTree fetchSchema( if (isAllCached && !explicitPathList.isEmpty()) { for (PartialPath fullPath : explicitPathList) { + // no path length <= 2 + if (fullPath.getNodeLength() <= 2) { + continue; + } cachedSchema = schemaCache.getMatchedNormalSchema(fullPath); if (cachedSchema.isEmpty()) { isAllCached = false; From 0a5f17d1c65596364a0efaf925b41da71f7ea48b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 18 Sep 2025 18:42:55 +0800 Subject: [PATCH 33/33] comment --- .../cache/schema/dualkeycache/impl/DualKeyCacheImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java index 841444f42793f..5e614fa5fd351 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java @@ -224,6 +224,8 @@ private long evictOneCacheEntry() { final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(belongedGroup.getFirstKey()); if (Objects.nonNull(cacheEntryGroup) && cacheEntryGroup.isEmpty()) { + // The removal is non-atomic, but it's ok because it's just a cache and does not affect the + // consistency if you evicts some entries being added if (Objects.nonNull(firstKeyMap.remove(belongedGroup.getFirstKey()))) { memory += sizeComputer.computeFirstKeySize(belongedGroup.getFirstKey())