diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java index 561e99ec3da39..4cca7ceb9fa9f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removeconfignode/IoTDBRemoveConfigNodeITFramework.java @@ -99,7 +99,7 @@ public void testRemoveConfigNode( SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - // Insert data in tree model + // Insert data statement.execute(TREE_MODEL_INSERTION); Map> regionMap = getDataRegionMap(statement); diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java index 73d5fd6fac353..b12c15283ed3c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java +++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java @@ -154,7 +154,7 @@ public void testRemoveDataNode( SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) { - // Insert data in tree model + // Insert data statement.execute(TREE_MODEL_INSERTION); Map> regionMap = getDataRegionMap(statement); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index 44ba9eaedfb96..f2e71ba319f39 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -715,8 +715,7 @@ private TSStatus executeStatementWithRetryOnDataTypeMismatch(final Statement sta final TSStatus status = executeStatement(statement); - // Try to convert data type if the statement is a tree model statement - // and the status code is not success + // Try to convert the data type and the status code is not success return shouldConvertDataTypeOnTypeMismatch && ((statement instanceof InsertBaseStatement && ((InsertBaseStatement) statement).hasFailedMeasurements()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java index ffaecaea7b5e0..0a0eb119b1313 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DeviceLastCache; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; @@ -110,7 +111,7 @@ public Response executeFastLastQueryStatement( PartialPath prefixPath = new PartialPath(prefixPathList.getPrefixPaths().toArray(new String[0])); - Map> resultMap = new HashMap<>(); + final Map>> resultMap = new HashMap<>(); final String prefixString = prefixPath.toString(); for (ISchemaRegion region : SchemaEngine.getInstance().getAllSchemaRegions()) { @@ -121,7 +122,7 @@ public Response executeFastLastQueryStatement( region.fillLastQueryMap(prefixPath, resultMap); } // Check cache first - if (!DataNodeSchemaCache.getInstance().getLastCache(resultMap)) { + if (!DataNodeSchemaCache.getInstance().getDeviceSchemaCache().getLastCache(resultMap)) { IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); TSLastDataQueryReq tsLastDataQueryReq = FastLastHandler.createTSLastDataQueryReq(clientSession, prefixPathList); @@ -169,14 +170,22 @@ public Response executeFastLastQueryStatement( List timeseries = new ArrayList<>(); List valueList = new ArrayList<>(); List dataTypeList = new ArrayList<>(); - for (Map.Entry> entry : resultMap.entrySet()) { - final String deviceWithPrefix = entry.getKey() + TsFileConstant.PATH_SEPARATOR; - for (Map.Entry measurementEntry : entry.getValue().entrySet()) { - final TimeValuePair tvPair = measurementEntry.getValue(); - valueList.add(tvPair.getValue().getStringValue()); - dataTypeList.add(tvPair.getValue().getDataType().name()); - targetDataSet.addTimestampsItem(tvPair.getTimestamp()); - timeseries.add(deviceWithPrefix + measurementEntry.getKey()); + for (Map.Entry>> entry : + resultMap.entrySet()) { + for (final Map.Entry> device2MeasurementLastEntry : + entry.getValue().entrySet()) { + final String deviceWithSeparator = + device2MeasurementLastEntry.getKey() + TsFileConstant.PATH_SEPARATOR; + for (Map.Entry measurementEntry : + device2MeasurementLastEntry.getValue().entrySet()) { + final TimeValuePair tvPair = measurementEntry.getValue(); + if (tvPair != DeviceLastCache.EMPTY_TIME_VALUE_PAIR) { + valueList.add(tvPair.getValue().getStringValue()); + dataTypeList.add(tvPair.getValue().getDataType().name()); + targetDataSet.addTimestampsItem(tvPair.getTimestamp()); + timeseries.add(deviceWithSeparator + measurementEntry.getKey()); + } + } } } if (!timeseries.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 64f5c0209fade..3e1d68ac3f32c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -70,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DeviceLastCache; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; @@ -817,7 +818,7 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath( "The \"executeFastLastDataQueryForOnePrefixPath\" dos not support wildcards.")); } - final Map> resultMap = new HashMap<>(); + final Map>> resultMap = new HashMap<>(); int sensorNum = 0; final String prefixString = prefixPath.toString(); @@ -830,7 +831,7 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath( } // 2.DATA_NODE_SCHEMA_CACHE.getLastCache() - if (!DataNodeSchemaCache.getInstance().getLastCache(resultMap)) { + if (!DataNodeSchemaCache.getInstance().getDeviceSchemaCache().getLastCache(resultMap)) { // 2.1 any sensor miss cache, construct last query sql, then return return executeLastDataQueryInternal(convert(req), SELECT_RESULT); } @@ -838,18 +839,26 @@ public TSExecuteStatementResp executeFastLastDataQueryForOnePrefixPath( // 2.2 all sensors hit cache, return response ~= 20ms final TsBlockBuilder builder = LastQueryUtil.createTsBlockBuilder(sensorNum); - for (final Map.Entry> result : resultMap.entrySet()) { - final String deviceWithPrefix = result.getKey() + TsFileConstant.PATH_SEPARATOR; - for (final Map.Entry measurementLastEntry : + for (final Map.Entry>> result : + resultMap.entrySet()) { + for (final Map.Entry> device2MeasurementLastEntry : result.getValue().entrySet()) { - final TimeValuePair tvPair = measurementLastEntry.getValue(); - LastQueryUtil.appendLastValue( - builder, - tvPair.getTimestamp(), - new Binary( - deviceWithPrefix + measurementLastEntry.getKey(), TSFileConfig.STRING_CHARSET), - tvPair.getValue().getStringValue(), - tvPair.getValue().getDataType().name()); + final String deviceWithSeparator = + device2MeasurementLastEntry.getKey() + TsFileConstant.PATH_SEPARATOR; + for (final Map.Entry measurementLastEntry : + device2MeasurementLastEntry.getValue().entrySet()) { + final TimeValuePair tvPair = measurementLastEntry.getValue(); + if (tvPair != DeviceLastCache.EMPTY_TIME_VALUE_PAIR) { + LastQueryUtil.appendLastValue( + builder, + tvPair.getTimestamp(), + new Binary( + deviceWithSeparator + measurementLastEntry.getKey(), + TSFileConfig.STRING_CHARSET), + tvPair.getValue().getStringValue(), + tvPair.getValue().getDataType().name()); + } + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index c7c51ed5f4b54..9d837b6f8fb09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -537,7 +537,7 @@ public TSStatus invalidatePartitionCache(final TInvalidateCacheReq req) { @Override public TSStatus invalidateLastCache(final String database) { - DataNodeSchemaCache.getInstance().invalidateLastCacheInDataRegion(database); + DataNodeSchemaCache.getInstance().invalidateDatabaseLastCache(database); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -546,7 +546,7 @@ public TSStatus invalidateSchemaCache(final TInvalidateCacheReq req) { DataNodeSchemaCache.getInstance().takeWriteLock(); try { // req.getFullPath() is a database path - DataNodeSchemaCache.getInstance().invalidate(req.getFullPath()); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidate(req.getFullPath()); ClusterTemplateManager.getInstance().invalid(req.getFullPath()); LOGGER.info("Schema cache of {} has been invalidated", req.getFullPath()); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); @@ -1974,12 +1974,16 @@ public TSStatus clearCacheImpl(final Set options) { || options.contains(CacheClearOptions.QUERY)) { storageEngine.clearCache(); } - if (options.contains(CacheClearOptions.TREE_SCHEMA)) { - schemaCache.invalidateAll(); + if (options.contains(CacheClearOptions.QUERY) + && options.contains(CacheClearOptions.TREE_SCHEMA)) { + schemaCache.getDeviceSchemaCache().invalidateAll(); return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } if (options.contains(CacheClearOptions.QUERY)) { - schemaCache.invalidateLastCache(new PartialPath("root.**")); + schemaCache.getDeviceSchemaCache().invalidateLastCache(); + } + if (options.contains(CacheClearOptions.TREE_SCHEMA)) { + schemaCache.getDeviceSchemaCache().invalidateTreeSchema(); } } catch (final Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/DeviceSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/DeviceSchemaInfo.java index 8bb8a8f4ab045..fc36f38941cdf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/DeviceSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/DeviceSchemaInfo.java @@ -30,16 +30,12 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; - public class DeviceSchemaInfo { - private PartialPath devicePath; - private boolean isAligned; - private List measurementSchemaInfoList; - private int templateId = NON_TEMPLATE; - - private DeviceSchemaInfo() {} + private final PartialPath devicePath; + private final boolean isAligned; + private final List measurementSchemaInfoList; + private final int templateId; public DeviceSchemaInfo( PartialPath devicePath, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java index 30d305bcec7ea..ce8d61625ee79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/AbstractUpdateLastCacheOperator.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; import org.apache.iotdb.db.queryengine.execution.operator.process.ProcessOperator; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DeviceLastCache; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; @@ -35,9 +36,11 @@ import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.schema.IMeasurementSchema; import javax.annotation.Nullable; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractUpdateLastCacheOperator implements ProcessOperator { @@ -117,18 +120,36 @@ protected void mayUpdateLastCache( } if (!deviceInMultiRegion) { - lastCache.updateLastCache( - getDatabaseName(), fullPath, new TimeValuePair(time, value), false, Long.MIN_VALUE); + lastCache.updateLastCacheIfExists( + getDatabaseName(), + fullPath.getDevicePath(), + new String[] {fullPath.getMeasurement()}, + new TimeValuePair[] { + Objects.nonNull(value) + ? new TimeValuePair(time, value) + : needUpdateNullEntry ? DeviceLastCache.EMPTY_TIME_VALUE_PAIR : null + }, + fullPath.isUnderAlignedEntity(), + new IMeasurementSchema[] {fullPath.getMeasurementSchema()}); return; } // update cache in DataNodeQueryContext if (seriesScanInfo.right == null || time > seriesScanInfo.right.getTimestamp()) { - seriesScanInfo.right = new TimeValuePair(time, value); + if (Objects.nonNull(value)) { + seriesScanInfo.right = new TimeValuePair(time, value); + } else { + seriesScanInfo.right = needUpdateNullEntry ? DeviceLastCache.EMPTY_TIME_VALUE_PAIR : null; + } } if (seriesScanInfo.left.decrementAndGet() == 0) { - lastCache.updateLastCache( - getDatabaseName(), fullPath, seriesScanInfo.right, false, Long.MIN_VALUE); + lastCache.updateLastCacheIfExists( + getDatabaseName(), + fullPath.getDevicePath(), + new String[] {fullPath.getMeasurement()}, + new TimeValuePair[] {seriesScanInfo.right}, + fullPath.isUnderAlignedEntity(), + new IMeasurementSchema[] {fullPath.getMeasurementSchema()}); } } finally { dataNodeQueryContext.unLock(deviceInMultiRegion); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java index 98525791f6dd5..f6578c7ef70b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/last/LastQueryUtil.java @@ -166,8 +166,8 @@ public static List createAggregators(TSDataType dataType, int valueC public static boolean needUpdateCache(Filter timeFilter) { // Update the cache only when, the filter is gt (greater than) or ge (greater than or equal to) - return CACHE_ENABLED && (timeFilter == null || timeFilter instanceof TimeGtEq) - || (timeFilter instanceof TimeGt); + return CACHE_ENABLED + && (timeFilter == null || timeFilter instanceof TimeGtEq || timeFilter instanceof TimeGt); } public static boolean needUpdateNullEntry(Filter timeFilter) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java index 1c8ac8cb0a09e..a015f6ebb502f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCache.java @@ -22,30 +22,32 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.service.metric.MetricService; -import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.schema.view.LogicalViewSchema; +import org.apache.iotdb.db.exception.metadata.view.InsertNonWritableViewException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; -import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; +import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.IntFunction; -import java.util.function.IntPredicate; - -import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; +import java.util.stream.Collectors; +import java.util.stream.IntStream; /** * This class takes the responsibility of metadata cache management of all DataRegions under @@ -53,32 +55,15 @@ */ public class DataNodeSchemaCache { - private static final Logger logger = LoggerFactory.getLogger(DataNodeSchemaCache.class); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private final ITemplateManager templateManager = ClusterTemplateManager.getInstance(); - private final DeviceUsingTemplateSchemaCache deviceUsingTemplateSchemaCache; - - private final TimeSeriesSchemaCache timeSeriesSchemaCache; + private final DeviceSchemaCache deviceSchemaCache = new DeviceSchemaCache(); // cache update or clean have higher priority than cache read private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); private DataNodeSchemaCache() { - deviceUsingTemplateSchemaCache = new DeviceUsingTemplateSchemaCache(templateManager); - timeSeriesSchemaCache = new TimeSeriesSchemaCache(); - - MetricService.getInstance().addMetricSet(new DataNodeSchemaCacheMetrics(this)); - } - - public long getHitCount() { - return deviceUsingTemplateSchemaCache.getHitCount() + timeSeriesSchemaCache.getHitCount(); - } - - public long getRequestCount() { - return deviceUsingTemplateSchemaCache.getRequestCount() - + timeSeriesSchemaCache.getRequestCount(); + // private constructor } public static DataNodeSchemaCache getInstance() { @@ -113,18 +98,27 @@ public void releaseWriteLock() { * @param measurements * @return timeseries partialPath and its SchemaEntity */ - public ClusterSchemaTree get(PartialPath devicePath, String[] measurements) { - return timeSeriesSchemaCache.get(devicePath, measurements); - } - - public ClusterSchemaTree get(PartialPath fullPath) { - ClusterSchemaTree clusterSchemaTree = - deviceUsingTemplateSchemaCache.get(fullPath.getDevicePath()); - if (clusterSchemaTree == null || clusterSchemaTree.isEmpty()) { - return timeSeriesSchemaCache.get(fullPath); - } else { - return clusterSchemaTree; + public ClusterSchemaTree get(final PartialPath devicePath, final String[] measurements) { + final ClusterSchemaTree tree = new ClusterSchemaTree(); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath); + if (!(schema instanceof DeviceNormalSchema)) { + return tree; } + final DeviceNormalSchema treeSchema = (DeviceNormalSchema) schema; + for (final String measurement : measurements) { + final SchemaCacheEntry entry = treeSchema.getSchemaCacheEntry(measurement); + if (Objects.nonNull(entry)) { + tree.appendSingleMeasurement( + devicePath.concatNode(measurement), + entry.getSchema(), + entry.getTagMap(), + null, + null, + treeSchema.isAligned()); + } + } + tree.setDatabases(Collections.singleton(treeSchema.getDatabase())); + return tree; } /** @@ -133,8 +127,17 @@ public ClusterSchemaTree get(PartialPath fullPath) { * @param devicePath full path of the device * @return empty if cache miss or the device path is not a template activated path */ - public ClusterSchemaTree getMatchedSchemaWithTemplate(PartialPath devicePath) { - return deviceUsingTemplateSchemaCache.getMatchedSchemaWithTemplate(devicePath); + public ClusterSchemaTree getMatchedTemplateSchema(final PartialPath devicePath) { + final ClusterSchemaTree tree = new ClusterSchemaTree(); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(devicePath); + if (!(schema instanceof DeviceTemplateSchema)) { + return tree; + } + final DeviceTemplateSchema treeSchema = (DeviceTemplateSchema) schema; + Template template = templateManager.getTemplate(treeSchema.getTemplateId()); + tree.appendTemplateDevice(devicePath, template.isDirectAligned(), template.getId(), template); + tree.setDatabases(Collections.singleton(treeSchema.getDatabase())); + return tree; } /** @@ -143,14 +146,47 @@ public ClusterSchemaTree getMatchedSchemaWithTemplate(PartialPath devicePath) { * @param fullPath full path * @return empty if cache miss */ - public ClusterSchemaTree getMatchedSchemaWithoutTemplate(PartialPath fullPath) { - return timeSeriesSchemaCache.get(fullPath); + public ClusterSchemaTree getMatchedNormalSchema(final PartialPath fullPath) { + final ClusterSchemaTree tree = new ClusterSchemaTree(); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevicePath()); + if (!(schema instanceof DeviceNormalSchema)) { + return tree; + } + final DeviceNormalSchema treeSchema = (DeviceNormalSchema) schema; + final SchemaCacheEntry entry = treeSchema.getSchemaCacheEntry(fullPath.getMeasurement()); + if (Objects.isNull(entry)) { + return tree; + } + tree.appendSingleMeasurement( + fullPath, entry.getSchema(), entry.getTagMap(), null, null, treeSchema.isAligned()); + tree.setDatabases(Collections.singleton(treeSchema.getDatabase())); + return tree; } - public List computeWithoutTemplate(ISchemaComputation schemaComputation) { - List result = timeSeriesSchemaCache.computeAndRecordLogicalView(schemaComputation); + public List computeWithoutTemplate(final ISchemaComputation schemaComputation) { + final List indexOfMissingMeasurements = new ArrayList<>(); + final String[] measurements = schemaComputation.getMeasurements(); + + final IDeviceSchema schema = + deviceSchemaCache.getDeviceSchema(schemaComputation.getDevicePath()); + if (!(schema instanceof DeviceNormalSchema)) { + return IntStream.range(0, schemaComputation.getMeasurements().length) + .boxed() + .collect(Collectors.toList()); + } + final DeviceNormalSchema treeSchema = (DeviceNormalSchema) schema; + + for (int i = 0; i < schemaComputation.getMeasurements().length; i++) { + final SchemaCacheEntry value = treeSchema.getSchemaCacheEntry(measurements[i]); + if (value == null) { + indexOfMissingMeasurements.add(i); + } else { + schemaComputation.computeMeasurement(i, value); + } + } + schemaComputation.computeDevice(treeSchema.isAligned()); schemaComputation.recordRangeOfLogicalViewSchemaListNow(); - return result; + return indexOfMissingMeasurements; } /** @@ -163,177 +199,267 @@ public List computeWithoutTemplate(ISchemaComputation schemaComputation * @return The indexes of missed views and full paths of their source paths will be returned. */ public Pair, List> computeSourceOfLogicalView( - ISchemaComputation schemaComputation) { + final ISchemaComputation schemaComputation) { if (!schemaComputation.hasLogicalViewNeedProcess()) { return new Pair<>(new ArrayList<>(), new ArrayList<>()); } - return timeSeriesSchemaCache.computeSourceOfLogicalView(schemaComputation); - } - public List computeWithTemplate(ISchemaComputation schemaComputation) { - return deviceUsingTemplateSchemaCache.compute(schemaComputation); + final List indexOfMissingMeasurements = new ArrayList<>(); + final Pair beginToEnd = + schemaComputation.getRangeOfLogicalViewSchemaListRecorded(); + final List logicalViewSchemaList = + schemaComputation.getLogicalViewSchemaList(); + final List indexListOfLogicalViewPaths = + schemaComputation.getIndexListOfLogicalViewPaths(); + + for (int i = beginToEnd.left; i < beginToEnd.right; i++) { + final LogicalViewSchema logicalViewSchema = logicalViewSchemaList.get(i); + final int realIndex = indexListOfLogicalViewPaths.get(i); + if (!logicalViewSchema.isWritable()) { + throw new RuntimeException( + new InsertNonWritableViewException( + schemaComputation + .getDevicePath() + .concatAsMeasurementPath(schemaComputation.getMeasurements()[realIndex]) + .getFullPath())); + } + + final PartialPath fullPath = logicalViewSchema.getSourcePathIfWritable(); + final IDeviceSchema schema = deviceSchemaCache.getDeviceSchema(fullPath.getDevicePath()); + if (!(schema instanceof DeviceNormalSchema)) { + indexOfMissingMeasurements.add(i); + continue; + } + + final DeviceNormalSchema treeSchema = (DeviceNormalSchema) schema; + final SchemaCacheEntry value = treeSchema.getSchemaCacheEntry(fullPath.getMeasurement()); + if (Objects.isNull(value)) { + indexOfMissingMeasurements.add(i); + continue; + } + + if (value.isLogicalView()) { + throw new RuntimeException( + new UnsupportedOperationException( + String.format( + "The source of view [%s] is also a view! Nested view is unsupported! " + + "Please check it.", + logicalViewSchema.getSourcePathIfWritable()))); + } + + schemaComputation.computeMeasurementOfView(realIndex, value, treeSchema.isAligned()); + } + + return new Pair<>( + indexOfMissingMeasurements, + indexOfMissingMeasurements.stream() + .map(index -> logicalViewSchemaList.get(index).getSourcePathStringIfWritable()) + .collect(Collectors.toList())); } - /** - * Store the fetched schema in either the schemaCache or templateSchemaCache, depending on its - * associated device. - */ - public void put(ClusterSchemaTree tree) { - PartialPath devicePath; - for (DeviceSchemaInfo deviceSchemaInfo : tree.getAllDevices()) { - devicePath = deviceSchemaInfo.getDevicePath(); - if (deviceSchemaInfo.getTemplateId() != NON_TEMPLATE) { - deviceUsingTemplateSchemaCache.put( - devicePath, tree.getBelongedDatabase(devicePath), deviceSchemaInfo.getTemplateId()); - } else { - for (MeasurementPath measurementPath : deviceSchemaInfo.getMeasurementSchemaPathList()) { - timeSeriesSchemaCache.putSingleMeasurementPath( - tree.getBelongedDatabase(devicePath), measurementPath); - } + public List computeWithTemplate(final ISchemaComputation computation) { + final List indexOfMissingMeasurements = new ArrayList<>(); + final String[] measurements = computation.getMeasurements(); + final IDeviceSchema deviceSchema = + deviceSchemaCache.getDeviceSchema(computation.getDevicePath()); + + if (!(deviceSchema instanceof DeviceTemplateSchema)) { + return IntStream.range(0, measurements.length).boxed().collect(Collectors.toList()); + } + + final DeviceTemplateSchema deviceTemplateSchema = (DeviceTemplateSchema) deviceSchema; + + final Template template = templateManager.getTemplate(deviceTemplateSchema.getTemplateId()); + computation.computeDevice(template.isDirectAligned()); + final Map templateSchema = template.getSchemaMap(); + for (int i = 0; i < measurements.length; i++) { + if (!templateSchema.containsKey(measurements[i])) { + indexOfMissingMeasurements.add(i); + continue; } + final IMeasurementSchema schema = templateSchema.get(measurements[i]); + computation.computeMeasurement(i, new WrappedSchemaInfo(schema)); } + return indexOfMissingMeasurements; } - public TimeValuePair getLastCache(PartialPath seriesPath) { - return timeSeriesSchemaCache.getLastCache(seriesPath); + /** + * Store the fetched schema in either the {@link DeviceNormalSchema} or {@link + * DeviceTemplateSchema}, depending on its associated device. + */ + public void put(final ClusterSchemaTree tree) { + tree.getAllDevices() + .forEach( + deviceSchemaInfo -> + deviceSchemaCache.putDeviceSchema( + tree.getBelongedDatabase(deviceSchemaInfo.getDevicePath()), deviceSchemaInfo)); } - public boolean getLastCache(final Map> inputMap) { - return timeSeriesSchemaCache.getLastCache(inputMap); + public TimeValuePair getLastCache(final PartialPath seriesPath) { + return deviceSchemaCache.getLastEntry(seriesPath.getDevicePath(), seriesPath.getMeasurement()); } - public void invalidateLastCache(PartialPath path) { + public void invalidateLastCache(final PartialPath path) { if (!CommonDescriptor.getInstance().getConfig().isLastCacheEnable()) { return; } - takeReadLock(); - try { - timeSeriesSchemaCache.invalidateLastCache(path); - } finally { - releaseReadLock(); - } + deviceSchemaCache.invalidateLastCache(path.getDevicePath(), path.getMeasurement()); } - public void invalidateLastCacheInDataRegion(String database) { + public void invalidateDatabaseLastCache(final String database) { if (!CommonDescriptor.getInstance().getConfig().isLastCacheEnable()) { return; } - takeReadLock(); - try { - timeSeriesSchemaCache.invalidateDataRegionLastCache(database); - } finally { - releaseReadLock(); - } - } - - /** get SchemaCacheEntry and update last cache */ - @TestOnly - public void updateLastCache( - PartialPath devicePath, - String measurement, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - timeSeriesSchemaCache.updateLastCache( - devicePath, measurement, timeValuePair, highPriorityUpdate, latestFlushedTime); + deviceSchemaCache.invalidateLastCache(database); } - public void updateLastCache( - String database, - PartialPath devicePath, - String[] measurements, - MeasurementSchema[] measurementSchemas, - boolean isAligned, - IntFunction timeValuePairProvider, - IntPredicate shouldUpdateProvider, - boolean highPriorityUpdate, - Long latestFlushedTime) { - takeReadLock(); - try { - timeSeriesSchemaCache.updateLastCache( - database, - devicePath, - measurements, - measurementSchemas, - isAligned, - timeValuePairProvider, - shouldUpdateProvider, - highPriorityUpdate, - latestFlushedTime); - } finally { - releaseReadLock(); - } + /** + * Update the {@link DeviceLastCache} in writing. If a measurement is with all {@code null}s, its + * {@link TimeValuePair}[] shall be {@code null}. For correctness, this will put the {@link + * DeviceCacheEntry} lazily and only update the existing {@link DeviceLastCache}s of measurements. + * + * @param database the device's database, WITH "root" + * @param deviceID {@link IDeviceID} + * @param measurements the fetched measurements + * @param timeValuePairs the {@link TimeValuePair}s with indexes corresponding to the measurements + */ + public void updateLastCacheIfExists( + final String database, + final PartialPath deviceID, + final String[] measurements, + final @Nonnull TimeValuePair[] timeValuePairs, + final boolean isAligned, + final IMeasurementSchema[] measurementSchemas) { + deviceSchemaCache.updateLastCache( + database, deviceID, measurements, timeValuePairs, isAligned, measurementSchemas, false); } - public void updateLastCacheWithoutLock( - String database, - PartialPath devicePath, - String[] measurements, - MeasurementSchema[] measurementSchemas, - boolean isAligned, - IntFunction timeValuePairProvider, - IntPredicate shouldUpdateProvider, - boolean highPriorityUpdate, - Long latestFlushedTime) { - timeSeriesSchemaCache.updateLastCache( + /** + * Update the {@link DeviceLastCache} on query. + * + *

Note: The query shall put the {@link DeviceLastCache} twice: + * + *

- First time call this before the query accesses data. It is just to allow the writing to + * update the cache, then avoid that the query put a stale value to cache and break the + * consistency. WARNING: The writing may temporarily put a stale value in cache if a stale value + * is written, but it won't affect the eventual consistency. + * + *

- Second time put the calculated {@link TimeValuePair}, and use {@link + * #updateLastCacheIfExists(String, PartialPath, String[], TimeValuePair[], boolean, + * IMeasurementSchema[])}. The input {@link TimeValuePair} shall never be or contain {@code null}, + * if the measurement is with all {@code null}s, its {@link TimeValuePair} shall be {@link + * DeviceLastCache#EMPTY_TIME_VALUE_PAIR}. This method is not supposed to update time column. + * + * @param database the device's database, WITH "root" + * @param measurementPath the fetched {@link MeasurementPath} + */ + public void declareLastCache(final String database, final MeasurementPath measurementPath) { + deviceSchemaCache.updateLastCache( database, - devicePath, - measurements, - measurementSchemas, - isAligned, - timeValuePairProvider, - shouldUpdateProvider, - highPriorityUpdate, - latestFlushedTime); + measurementPath.getDevicePath(), + new String[] {measurementPath.getMeasurement()}, + null, + measurementPath.isUnderAlignedEntity(), + new IMeasurementSchema[] {measurementPath.getMeasurementSchema()}, + true); } /** - * get or create SchemaCacheEntry and update last cache, only support non-aligned sensor or - * aligned sensor without only one sub sensor + * Update the {@link DeviceLastCache} on query. + * + *

If the query has ended abnormally, it shall call this to invalidate the entry it has pushed + * in the first time, to avoid the stale writing damaging the eventual consistency. + * + * @param database the device's database, WITH "root" + * @param measurementPath the fetched {@link MeasurementPath} */ - public void updateLastCache( - String storageGroup, - MeasurementPath measurementPath, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - takeReadLock(); - try { - timeSeriesSchemaCache.updateLastCache( - storageGroup, measurementPath, timeValuePair, highPriorityUpdate, latestFlushedTime); - } finally { - releaseReadLock(); - } + public void invalidateLastCache(final String database, final MeasurementPath measurementPath) { + deviceSchemaCache.updateLastCache( + database, + measurementPath.getDevicePath(), + new String[] {measurementPath.getMeasurement()}, + new TimeValuePair[] {null}, + measurementPath.isUnderAlignedEntity(), + new IMeasurementSchema[] {measurementPath.getMeasurementSchema()}, + true); } - public void invalidate(String database) { - deviceUsingTemplateSchemaCache.invalidateCache(database); - timeSeriesSchemaCache.invalidate(database); + public void invalidate(final List partialPathList) { + // Currently invalidate by device + partialPathList.forEach( + measurementPath -> { + final boolean isMultiLevelWildcardMeasurement = + PathPatternUtil.isMultiLevelMatchWildcard(measurementPath.getMeasurement()); + deviceSchemaCache.invalidateCache( + isMultiLevelWildcardMeasurement ? measurementPath : measurementPath.getDevicePath(), + isMultiLevelWildcardMeasurement); + }); } - public void invalidate(List partialPathList) { - boolean doPrecise = true; - for (PartialPath partialPath : partialPathList) { - if (partialPath.getDevicePath().hasWildcard()) { - doPrecise = false; - break; + public DeviceSchemaCache getDeviceSchemaCache() { + return deviceSchemaCache; + } + + public void cleanUp() { + deviceSchemaCache.invalidateAll(); + } + + private static class WrappedSchemaInfo implements IMeasurementSchemaInfo { + private final IMeasurementSchema schema; + + public WrappedSchemaInfo(final IMeasurementSchema schema) { + this.schema = schema; + } + + @Override + public String getName() { + return schema.getMeasurementId(); + } + + @Override + public IMeasurementSchema getSchema() { + if (isLogicalView()) { + return new LogicalViewSchema( + schema.getMeasurementId(), ((LogicalViewSchema) schema).getExpression()); + } else { + return this.getSchemaAsMeasurementSchema(); } } - if (doPrecise) { - deviceUsingTemplateSchemaCache.invalidateCache(partialPathList); - timeSeriesSchemaCache.invalidate(partialPathList); - } else { - invalidateAll(); + + @Override + public MeasurementSchema getSchemaAsMeasurementSchema() { + return new MeasurementSchema( + schema.getMeasurementId(), + schema.getType(), + schema.getEncodingType(), + schema.getCompressor()); } - } - public void invalidateAll() { - deviceUsingTemplateSchemaCache.invalidateCache(); - timeSeriesSchemaCache.invalidateAll(); - } + @Override + public LogicalViewSchema getSchemaAsLogicalViewSchema() { + throw new RuntimeException( + new UnsupportedOperationException( + "Function getSchemaAsLogicalViewSchema is not supported in DeviceUsingTemplateSchemaCache.")); + } - public void cleanUp() { - deviceUsingTemplateSchemaCache.invalidateCache(); - timeSeriesSchemaCache.invalidateAll(); + @Override + public Map getTagMap() { + return null; + } + + @Override + public Map getAttributeMap() { + return null; + } + + @Override + public String getAlias() { + return null; + } + + @Override + public boolean isLogicalView() { + return schema.isLogicalView(); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCacheMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCacheMetrics.java index 696d28c5118a9..bb567143408e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCacheMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeSchemaCacheMetrics.java @@ -29,19 +29,23 @@ import java.util.Objects; public class DataNodeSchemaCacheMetrics implements IMetricSet { - private DataNodeSchemaCache dataNodeSchemaCache; - public DataNodeSchemaCacheMetrics(DataNodeSchemaCache dataNodeSchemaCache) { - this.dataNodeSchemaCache = dataNodeSchemaCache; + private static final String SCHEMA_CACHE_TOTAL_USAGE = "schema_cache_total_usage"; + private static final String SCHEMA_CACHE_MEM_CAPACITY = "schema_cache_mem_capacity"; + + private final DeviceSchemaCache deviceSchemaCache; + + public DataNodeSchemaCacheMetrics(final DeviceSchemaCache dataNodeSchemaCache) { + this.deviceSchemaCache = dataNodeSchemaCache; } @Override - public void bindTo(AbstractMetricService metricService) { + public void bindTo(final AbstractMetricService metricService) { metricService.createAutoGauge( Metric.CACHE.toString(), MetricLevel.IMPORTANT, - dataNodeSchemaCache, - DataNodeSchemaCache::getHitCount, + deviceSchemaCache, + DeviceSchemaCache::getHitCount, Tag.NAME.toString(), "SchemaCache", Tag.TYPE.toString(), @@ -49,16 +53,35 @@ public void bindTo(AbstractMetricService metricService) { metricService.createAutoGauge( Metric.CACHE.toString(), MetricLevel.IMPORTANT, - dataNodeSchemaCache, - DataNodeSchemaCache::getRequestCount, + deviceSchemaCache, + DeviceSchemaCache::getRequestCount, Tag.NAME.toString(), "SchemaCache", Tag.TYPE.toString(), "all"); + metricService.createAutoGauge( + Metric.MEM.toString(), + MetricLevel.IMPORTANT, + deviceSchemaCache, + DeviceSchemaCache::getMemoryUsage, + Tag.NAME.toString(), + SCHEMA_CACHE_TOTAL_USAGE); + metricService.createAutoGauge( + Metric.MEM.toString(), + MetricLevel.IMPORTANT, + deviceSchemaCache, + DeviceSchemaCache::capacity, + Tag.NAME.toString(), + SCHEMA_CACHE_MEM_CAPACITY); + metricService.createAutoGauge( + Metric.CACHE_ENTRIES_NUM.toString(), + MetricLevel.IMPORTANT, + deviceSchemaCache, + DeviceSchemaCache::entriesCount); } @Override - public void unbindFrom(AbstractMetricService metricService) { + public void unbindFrom(final AbstractMetricService metricService) { metricService.remove( MetricType.AUTO_GAUGE, Metric.CACHE.toString(), @@ -73,22 +96,33 @@ public void unbindFrom(AbstractMetricService metricService) { "SchemaCache", Tag.TYPE.toString(), "all"); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.MEM.toString(), + Tag.NAME.toString(), + SCHEMA_CACHE_TOTAL_USAGE); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.MEM.toString(), + Tag.NAME.toString(), + SCHEMA_CACHE_MEM_CAPACITY); + metricService.remove(MetricType.AUTO_GAUGE, Metric.CACHE_ENTRIES_NUM.toString()); } @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - DataNodeSchemaCacheMetrics that = (DataNodeSchemaCacheMetrics) o; - return Objects.equals(dataNodeSchemaCache, that.dataNodeSchemaCache); + final DataNodeSchemaCacheMetrics that = (DataNodeSchemaCacheMetrics) o; + return Objects.equals(deviceSchemaCache, that.deviceSchemaCache); } @Override public int hashCode() { - return Objects.hash(dataNodeSchemaCache); + return Objects.hash(deviceSchemaCache); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java new file mode 100644 index 0000000000000..a12b9c7661b79 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; + +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +import javax.annotation.Nonnull; +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; + +@ThreadSafe +public class DeviceCacheEntry { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(DeviceCacheEntry.class) + + 2 * RamUsageEstimator.shallowSizeOfInstance(AtomicReference.class); + + // the cached attributeMap may not be the latest, but there won't be any correctness problems + // because when missing getting the key-value from this attributeMap, caller will try to get or + // create from remote + // there may exist key is not null, but value is null in this map, which means that the key's + // corresponding value is null, doesn't mean that the key doesn't exist + private final AtomicReference deviceSchema = new AtomicReference<>(); + private final AtomicReference lastCache = new AtomicReference<>(); + + int setDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { + // Safe here because tree schema is invalidated by the whole entry + if (deviceSchemaInfo.getTemplateId() == NON_TEMPLATE) { + final int result = + (deviceSchema.compareAndSet( + null, new DeviceNormalSchema(database, deviceSchemaInfo.isAligned())) + ? DeviceNormalSchema.INSTANCE_SIZE + : 0); + return deviceSchema.get() instanceof DeviceNormalSchema + ? result + + ((DeviceNormalSchema) deviceSchema.get()) + .update(deviceSchemaInfo.getMeasurementSchemaInfoList()) + : 0; + } else { + return deviceSchema.compareAndSet( + null, new DeviceTemplateSchema(database, deviceSchemaInfo.getTemplateId())) + ? DeviceTemplateSchema.INSTANCE_SIZE + : 0; + } + } + + int setMeasurementSchema( + final String database, + final boolean isAligned, + final String[] measurements, + final IMeasurementSchema[] schemas) { + if (schemas == null) { + return 0; + } + // Safe here because schema is invalidated by the whole entry + final int result = + (deviceSchema.compareAndSet(null, new DeviceNormalSchema(database, isAligned)) + ? DeviceNormalSchema.INSTANCE_SIZE + : 0); + return deviceSchema.get() instanceof DeviceNormalSchema + ? result + ((DeviceNormalSchema) deviceSchema.get()).update(measurements, schemas) + : 0; + } + + IDeviceSchema getDeviceSchema() { + return deviceSchema.get(); + } + + int invalidateSchema() { + final AtomicInteger size = new AtomicInteger(0); + deviceSchema.updateAndGet( + schema -> { + size.set(schema.estimateSize()); + return schema; + }); + return size.get(); + } + + /////////////////////////////// Last Cache /////////////////////////////// + + int initOrInvalidateLastCache(final String[] measurements, final boolean isInvalidate) { + int result = + lastCache.compareAndSet(null, new DeviceLastCache()) ? DeviceLastCache.INSTANCE_SIZE : 0; + final DeviceLastCache cache = lastCache.get(); + result += Objects.nonNull(cache) ? cache.initOrInvalidate(measurements, isInvalidate) : 0; + return Objects.nonNull(lastCache.get()) ? result : 0; + } + + int tryUpdateLastCache( + final String[] measurements, final TimeValuePair[] timeValuePairs, boolean invalidateNull) { + final DeviceLastCache cache = lastCache.get(); + final int result = + Objects.nonNull(cache) ? cache.tryUpdate(measurements, timeValuePairs, invalidateNull) : 0; + return Objects.nonNull(lastCache.get()) ? result : 0; + } + + int tryUpdateLastCache(final String[] measurements, final TimeValuePair[] timeValuePairs) { + return tryUpdateLastCache(measurements, timeValuePairs, false); + } + + int invalidateLastCache(final String measurement) { + final DeviceLastCache cache = lastCache.get(); + final int result = Objects.nonNull(cache) ? cache.invalidate(measurement) : 0; + return Objects.nonNull(lastCache.get()) ? result : 0; + } + + TimeValuePair getTimeValuePair(final String measurement) { + final DeviceLastCache cache = lastCache.get(); + return Objects.nonNull(cache) ? cache.getTimeValuePair(measurement) : null; + } + + boolean updateInputMap(final @Nonnull Map updateMap) { + for (final String measurement : updateMap.keySet()) { + final TimeValuePair result = getTimeValuePair(measurement); + if (result == null) { + return false; + } + updateMap.put(measurement, result); + } + return true; + } + + int invalidateLastCache() { + final AtomicInteger size = new AtomicInteger(0); + lastCache.updateAndGet( + cacheEntry -> { + if (Objects.nonNull(cacheEntry)) { + size.set(cacheEntry.estimateSize()); + } + return null; + }); + return size.get(); + } + + /////////////////////////////// Management /////////////////////////////// + + int estimateSize() { + final IDeviceSchema schema = deviceSchema.get(); + final DeviceLastCache cache = lastCache.get(); + return (int) + (INSTANCE_SIZE + + (Objects.nonNull(schema) ? schema.estimateSize() : 0) + + (Objects.nonNull(cache) ? cache.estimateSize() : 0)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java new file mode 100644 index 0000000000000..a2f7696b6dfb5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceLastCache.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +@ThreadSafe +public class DeviceLastCache { + static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(DeviceLastCache.class) + + (int) RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); + + public static final TsPrimitiveType EMPTY_PRIMITIVE_TYPE = + new TsPrimitiveType() { + @Override + public void setObject(Object o) { + // Do nothing + } + + @Override + public void reset() { + // Do nothing + } + + @Override + public int getSize() { + return 0; + } + + @Override + public Object getValue() { + return null; + } + + @Override + public String getStringValue() { + return null; + } + + @Override + public TSDataType getDataType() { + return null; + } + }; + + /** This means that the tv pair has been put, and the value is null */ + public static final TimeValuePair EMPTY_TIME_VALUE_PAIR = + new TimeValuePair(Long.MIN_VALUE, EMPTY_PRIMITIVE_TYPE); + + /** This means that the tv pair has been declared, and is ready for the next put. */ + private static final TimeValuePair PLACEHOLDER_TIME_VALUE_PAIR = + new TimeValuePair(Long.MIN_VALUE, EMPTY_PRIMITIVE_TYPE); + + // Time is seen as "" as a measurement + private final Map measurement2CachedLastMap = new ConcurrentHashMap<>(); + + int initOrInvalidate(final String[] measurements, final boolean isInvalidate) { + final AtomicInteger diff = new AtomicInteger(0); + + for (final String measurement : measurements) { + if (Objects.isNull(measurement)) { + continue; + } + final TimeValuePair newPair = isInvalidate ? null : PLACEHOLDER_TIME_VALUE_PAIR; + + measurement2CachedLastMap.compute( + measurement, + (measurementKey, tvPair) -> { + if (Objects.isNull(newPair)) { + diff.addAndGet( + -((int) RamUsageEstimator.sizeOf(measurement) + getTVPairEntrySize(tvPair))); + return null; + } + if (Objects.isNull(tvPair)) { + diff.addAndGet( + (int) RamUsageEstimator.sizeOf(measurement) + + (int) RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY); + return newPair; + } + return tvPair; + }); + } + return diff.get(); + } + + int tryUpdate( + final @Nonnull String[] measurements, final @Nonnull TimeValuePair[] timeValuePairs) { + return tryUpdate(measurements, timeValuePairs, false); + } + + int tryUpdate( + final @Nonnull String[] measurements, + final @Nonnull TimeValuePair[] timeValuePairs, + final boolean invalidateNull) { + final AtomicInteger diff = new AtomicInteger(0); + long lastTime = Long.MIN_VALUE; + + for (int i = 0; i < measurements.length; ++i) { + if (Objects.isNull(timeValuePairs[i])) { + if (invalidateNull) { + measurement2CachedLastMap.remove(measurements[i]); + } + continue; + } + + final int finalI = i; + if (lastTime < timeValuePairs[i].getTimestamp()) { + lastTime = timeValuePairs[i].getTimestamp(); + } + measurement2CachedLastMap.computeIfPresent( + measurements[i], + (measurement, tvPair) -> { + if (tvPair.getTimestamp() <= timeValuePairs[finalI].getTimestamp()) { + diff.addAndGet(getDiffSize(tvPair, timeValuePairs[finalI])); + return timeValuePairs[finalI]; + } + return tvPair; + }); + } + final long finalLastTime = lastTime; + measurement2CachedLastMap.computeIfPresent( + "", + (time, tvPair) -> + tvPair.getTimestamp() < finalLastTime + ? new TimeValuePair(finalLastTime, EMPTY_PRIMITIVE_TYPE) + : tvPair); + return diff.get(); + } + + @GuardedBy("DataRegionInsertLock#writeLock") + int invalidate(final String measurement) { + final AtomicInteger diff = new AtomicInteger(); + final AtomicLong time = new AtomicLong(); + measurement2CachedLastMap.computeIfPresent( + measurement, + (s, timeValuePair) -> { + diff.set((int) RamUsageEstimator.sizeOf(s) + getTVPairEntrySize(timeValuePair)); + time.set(timeValuePair.getTimestamp()); + return null; + }); + if (diff.get() == 0) { + return 0; + } + measurement2CachedLastMap.computeIfPresent( + "", + (s, timeValuePair) -> { + if (timeValuePair.getTimestamp() <= time.get()) { + diff.addAndGet(getTVPairEntrySize(timeValuePair)); + return null; + } + return timeValuePair; + }); + + return diff.get(); + } + + private int getTVPairEntrySize(final TimeValuePair tvPair) { + return (int) RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + ((Objects.isNull(tvPair) + || tvPair == PLACEHOLDER_TIME_VALUE_PAIR + || tvPair == EMPTY_TIME_VALUE_PAIR) + ? 0 + : tvPair.getSize()); + } + + @Nullable + TimeValuePair getTimeValuePair(final @Nonnull String measurement) { + final TimeValuePair result = measurement2CachedLastMap.get(measurement); + return result != PLACEHOLDER_TIME_VALUE_PAIR ? result : null; + } + + int estimateSize() { + return INSTANCE_SIZE + + (int) RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY * measurement2CachedLastMap.size() + + measurement2CachedLastMap.values().stream() + .mapToInt(TimeValuePair::getSize) + .reduce(0, Integer::sum); + } + + private static int getDiffSize( + final TimeValuePair oldTimeValuePair, final TimeValuePair newTimeValuePair) { + if (oldTimeValuePair == EMPTY_TIME_VALUE_PAIR + || oldTimeValuePair == PLACEHOLDER_TIME_VALUE_PAIR) { + return newTimeValuePair.getSize(); + } + final TsPrimitiveType oldValue = oldTimeValuePair.getValue(); + final TsPrimitiveType newValue = newTimeValuePair.getValue(); + if (oldValue == null) { + return newValue == null ? 0 : newValue.getSize(); + } else if (oldValue instanceof TsPrimitiveType.TsBinary) { + return (newValue == null ? 0 : newValue.getSize()) - oldValue.getSize(); + } else { + return newValue == null ? -oldValue.getSize() : 0; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceNormalSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceNormalSchema.java new file mode 100644 index 0000000000000..cfad9c9e45e05 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceNormalSchema.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; + +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class DeviceNormalSchema implements IDeviceSchema { + + static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(DeviceNormalSchema.class) + + (int) RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); + private final String database; + private final boolean isAligned; + + private final ConcurrentMap measurementMap = new ConcurrentHashMap<>(); + + public DeviceNormalSchema(final String database, final boolean isAligned) { + this.database = database; + this.isAligned = isAligned; + } + + public String getDatabase() { + return database; + } + + public boolean isAligned() { + return isAligned; + } + + public SchemaCacheEntry getSchemaCacheEntry(final String measurement) { + return measurementMap.get(measurement); + } + + public int update(final String[] measurements, final IMeasurementSchema[] schemas) { + int diff = 0; + if (schemas == null) { + return diff; + } + + final int length = measurements.length; + + for (int i = 0; i < length; ++i) { + // Skip this to avoid instance creation/gc for writing performance + if (measurements[i] == null || measurementMap.containsKey(measurements[i])) { + continue; + } + diff += putEntry(measurements[i], schemas[i], null); + } + return diff; + } + + public int update(final List schemaInfoList) { + return schemaInfoList.stream() + .mapToInt( + schemaInfo -> + putEntry(schemaInfo.getName(), schemaInfo.getSchema(), schemaInfo.getTagMap())) + .reduce(0, Integer::sum); + } + + private int putEntry( + final String measurement, final IMeasurementSchema schema, final Map tagMap) { + final SchemaCacheEntry putEntry = new SchemaCacheEntry(schema, tagMap); + final SchemaCacheEntry cachedEntry = measurementMap.put(measurement, putEntry); + return Objects.isNull(cachedEntry) + ? (int) (RamUsageEstimator.sizeOf(measurement) + SchemaCacheEntry.estimateSize(putEntry)) + : SchemaCacheEntry.estimateSize(putEntry) - SchemaCacheEntry.estimateSize(cachedEntry); + } + + @Override + public int estimateSize() { + // Do not need to calculate database because it is interned + return INSTANCE_SIZE + + measurementMap.entrySet().stream() + .mapToInt( + entry -> + Math.toIntExact( + RamUsageEstimator.sizeOf(entry.getKey()) + + SchemaCacheEntry.estimateSize(entry.getValue()))) + .reduce(0, Integer::sum); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java new file mode 100644 index 0000000000000..8d66cb3d619ea --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; + +import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Predicate; +import java.util.function.ToIntFunction; + +/** + * The {@link DeviceSchemaCache} caches some of the devices and their: measurement info / template + * info. The last value of one device is also cached here. + */ +@ThreadSafe +public class DeviceSchemaCache { + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final Logger logger = LoggerFactory.getLogger(DeviceSchemaCache.class); + + /** + * Leading_Segment, {@link IDeviceID}, Map{@literal <}Measurement, Schema{@literal + * >}/templateInfo{@literal >} + * + *

The segment is used to: + * + *

1. Keep abreast of the newer versions. + * + *

2. Optimize the speed in invalidation by databases for most scenarios. + */ + private final IDualKeyCache dualKeyCache; + + private final Map databasePool = new ConcurrentHashMap<>(); + + private final ReentrantLock lock = new ReentrantLock(false); + + DeviceSchemaCache() { + dualKeyCache = + new DualKeyCacheBuilder() + .cacheEvictionPolicy( + DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) + .memoryCapacity(config.getAllocateMemoryForSchemaCache()) + .firstKeySizeComputer(segment -> (int) RamUsageEstimator.sizeOf(segment)) + .secondKeySizeComputer(PartialPath::estimateSize) + .valueSizeComputer(DeviceCacheEntry::estimateSize) + .build(); + MetricService.getInstance().addMetricSet(new DataNodeSchemaCacheMetrics(this)); + } + + /////////////////////////////// Last Cache /////////////////////////////// + + /** + * Get the last {@link TimeValuePair} of a measurement, the measurement shall never be "time". + * + * @param device {@link IDeviceID} + * @param measurement the measurement to get + * @return {@code null} iff cache miss, {@link DeviceLastCache#EMPTY_TIME_VALUE_PAIR} iff cache + * hit but result is {@code null}, and the result value otherwise. + */ + public TimeValuePair getLastEntry(final PartialPath device, final String measurement) { + final DeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); + return Objects.nonNull(entry) ? entry.getTimeValuePair(measurement) : null; + } + + /** + * Invalidate the last cache of one device. + * + * @param device IDeviceID + */ + public void invalidateDeviceLastCache(final PartialPath device) { + dualKeyCache.update( + getLeadingSegment(device), device, null, entry -> -entry.invalidateLastCache(), false); + } + + public void putDeviceSchema(final String database, final DeviceSchemaInfo deviceSchemaInfo) { + final PartialPath devicePath = deviceSchemaInfo.getDevicePath(); + final String databaseToUse = databasePool.computeIfAbsent(database, k -> database); + + dualKeyCache.update( + getLeadingSegment(devicePath), + devicePath, + new DeviceCacheEntry(), + entry -> entry.setDeviceSchema(databaseToUse, deviceSchemaInfo), + true); + } + + public IDeviceSchema getDeviceSchema(final PartialPath device) { + final DeviceCacheEntry entry = dualKeyCache.get(getLeadingSegment(device), device); + return Objects.nonNull(entry) ? entry.getDeviceSchema() : null; + } + + void updateLastCache( + final String database, + final PartialPath device, + final String[] measurements, + final @Nullable TimeValuePair[] timeValuePairs, + final boolean isAligned, + final IMeasurementSchema[] measurementSchemas, + final boolean initOrInvalidate) { + final String database2Use = databasePool.computeIfAbsent(database, k -> database); + + dualKeyCache.update( + getLeadingSegment(device), + device, + new DeviceCacheEntry(), + initOrInvalidate + ? entry -> + entry.setMeasurementSchema( + database2Use, isAligned, measurements, measurementSchemas) + + entry.initOrInvalidateLastCache(measurements, Objects.nonNull(timeValuePairs)) + : entry -> + entry.setMeasurementSchema( + database2Use, isAligned, measurements, measurementSchemas) + + entry.tryUpdateLastCache(measurements, timeValuePairs), + Objects.isNull(timeValuePairs)); + } + + public boolean getLastCache( + final Map>> inputMap) { + return dualKeyCache.batchApply(inputMap, DeviceCacheEntry::updateInputMap); + } + + void invalidateLastCache(final PartialPath devicePath, final String measurement) { + final ToIntFunction updateFunction = + PathPatternUtil.hasWildcard(measurement) + ? entry -> -entry.invalidateLastCache() + : entry -> -entry.invalidateLastCache(measurement); + + if (!devicePath.hasWildcard()) { + dualKeyCache.update(getLeadingSegment(devicePath), devicePath, null, updateFunction, false); + } else { + // This may take quite a long time to perform, yet we assume that the "invalidateLastCache" is + // only called by deletions, which has a low frequency; and it has avoided that + // the un-related paths being cleared, like "root.*.b.c.**" affects + // "root.*.d.c.**", thereby lower the query performance. + dualKeyCache.update( + segment -> { + try { + return devicePath.matchPrefixPath(new PartialPath(segment)); + } catch (final IllegalPathException e) { + logger.warn( + "Illegal segmentID {} found in cache when invalidating by path {}, invalidate it anyway", + segment, + devicePath); + return true; + } + }, + cachedDeviceID -> cachedDeviceID.matchFullPath(devicePath), + updateFunction); + } + } + + void invalidateCache( + final @Nonnull PartialPath devicePath, final boolean isMultiLevelWildcardMeasurement) { + if (!devicePath.hasWildcard()) { + dualKeyCache.invalidate(getLeadingSegment(devicePath), devicePath); + } else { + // This may take quite a long time to perform, yet we assume that the "invalidateLastCache" is + // only called by deletions, which has a low frequency; and it has avoided that + // the un-related paths being cleared, like "root.*.b.c.**" affects + // "root.*.d.c.**", thereby lower the query performance. + dualKeyCache.invalidate( + segment -> { + try { + return devicePath.matchPrefixPath(new PartialPath(segment)); + } catch (final IllegalPathException e) { + logger.warn( + "Illegal segmentID {} found in cache when invalidating by path {}, invalidate it anyway", + segment, + devicePath); + return true; + } + }, + cachedDeviceID -> + isMultiLevelWildcardMeasurement + ? devicePath.matchPrefixPath(cachedDeviceID) + : devicePath.matchFullPath(cachedDeviceID)); + } + } + + /////////////////////////////// Management /////////////////////////////// + + long getHitCount() { + return dualKeyCache.stats().hitCount(); + } + + long getRequestCount() { + return dualKeyCache.stats().requestCount(); + } + + long getMemoryUsage() { + return dualKeyCache.stats().memoryUsage(); + } + + long capacity() { + return dualKeyCache.stats().capacity(); + } + + long entriesCount() { + return dualKeyCache.stats().entriesCount(); + } + + // Note: It might be very slow if the database is too long + void invalidateLastCache(final @Nonnull String database) { + Predicate secondKeyChecker; + try { + final PartialPath databasePath = new PartialPath(database); + secondKeyChecker = device -> device.matchPrefixPath(databasePath); + } catch (final Exception ignored) { + secondKeyChecker = device -> device.startsWith(database); + } + + lock.lock(); + try { + dualKeyCache.update( + segment -> segment.startsWith(database), + device -> true, + entry -> -entry.invalidateLastCache()); + dualKeyCache.update( + database::startsWith, secondKeyChecker, entry -> -entry.invalidateLastCache()); + } finally { + lock.unlock(); + } + } + + // Note: It might be very slow if the database is too long + public void invalidate(final @Nonnull String database) { + lock.lock(); + try { + dualKeyCache.invalidate(segment -> segment.startsWith(database), device -> true); + dualKeyCache.invalidate(database::startsWith, device -> device.startsWith(database)); + } finally { + lock.unlock(); + } + } + + public void invalidateLastCache() { + lock.lock(); + try { + dualKeyCache.update(segment -> true, device -> true, entry -> -entry.invalidateLastCache()); + } finally { + lock.unlock(); + } + } + + public void invalidateTreeSchema() { + lock.lock(); + try { + dualKeyCache.update(segment -> true, device -> true, entry -> -entry.invalidateSchema()); + } finally { + lock.unlock(); + } + } + + public void invalidateAll() { + lock.lock(); + try { + dualKeyCache.invalidateAll(); + } finally { + lock.unlock(); + } + } + + // Utils of leading segment + + public static String getLeadingSegment(final PartialPath device) { + final String segment; + int lastSeparatorPos = -1; + int separatorNum = 0; + + final String deviceStr = device.getFullPath(); + for (int i = 0; i < deviceStr.length(); i++) { + if (deviceStr.charAt(i) == TsFileConstant.PATH_SEPARATOR_CHAR) { + lastSeparatorPos = i; + separatorNum++; + if (separatorNum == 3) { + break; + } + } + } + if (lastSeparatorPos == -1) { + segment = deviceStr; + } else { + segment = deviceStr.substring(0, lastSeparatorPos); + } + + return segment; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheUpdating.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceTemplateSchema.java similarity index 55% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheUpdating.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceTemplateSchema.java index 7ebdd566c9eff..5a9b054bd7cad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheUpdating.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceTemplateSchema.java @@ -17,29 +17,33 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache; +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; -/** - * This interfaces defines the behaviour needed be implemented and executed during cache value - * traverse. - * - * @param The first key of target cache values - * @param The second key of one target cache value - * @param The cache value - */ -public interface IDualKeyCacheUpdating { +import org.apache.tsfile.utils.RamUsageEstimator; + +public class DeviceTemplateSchema implements IDeviceSchema { + static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(DeviceTemplateSchema.class); + + private final String database; + private final int templateId; + + DeviceTemplateSchema(final String database, final int templateId) { + this.database = database; + this.templateId = templateId; + } - /** Return the first key of target cache values. */ - FK getFirstKey(); + public int getTemplateId() { + return templateId; + } - /** Return the second key list of target cache values. */ - SK[] getSecondKeyList(); + public String getDatabase() { + return database; + } - /** - * Compute each target cache value. The index is the second key's position in second key list. The - * value here can be updated. - * - * @return increasing memory usage - */ - int updateValue(int index, V value); + @Override + public int estimateSize() { + // Do not need to calculate database because it is interned + return INSTANCE_SIZE; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java deleted file mode 100644 index 167242ff20cc3..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; - -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.schema.view.LogicalViewSchema; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; -import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; -import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; -import org.apache.iotdb.db.schemaengine.template.ITemplateManager; -import org.apache.iotdb.db.schemaengine.template.Template; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.Weigher; -import org.apache.tsfile.write.schema.IMeasurementSchema; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class DeviceUsingTemplateSchemaCache { - - private static final Logger logger = - LoggerFactory.getLogger(DeviceUsingTemplateSchemaCache.class); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - - private final Cache cache; - - private final ITemplateManager templateManager; - - DeviceUsingTemplateSchemaCache(ITemplateManager templateManager) { - this.templateManager = templateManager; - // TODO proprietary config parameter expected - cache = - Caffeine.newBuilder() - .maximumWeight(config.getAllocateMemoryForSchemaCache()) - .weigher( - (Weigher) - (key, val) -> (PartialPath.estimateSize(key) + 32)) - .recordStats() - .build(); - } - - public long getHitCount() { - return cache.stats().hitCount(); - } - - public long getRequestCount() { - return cache.stats().requestCount(); - } - - public ClusterSchemaTree get(PartialPath fullPath) { - DeviceCacheEntry deviceCacheEntry = cache.getIfPresent(fullPath.getDevicePath()); - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - if (deviceCacheEntry != null) { - Template template = templateManager.getTemplate(deviceCacheEntry.getTemplateId()); - if (template.getSchema(fullPath.getMeasurement()) != null) { - schemaTree.appendTemplateDevice( - fullPath.getDevicePath(), template.isDirectAligned(), template.getId(), template); - schemaTree.setDatabases(Collections.singleton(deviceCacheEntry.getDatabase())); - } - } - return schemaTree; - } - - public ClusterSchemaTree getMatchedSchemaWithTemplate(PartialPath devicePath) { - DeviceCacheEntry deviceCacheEntry = cache.getIfPresent(devicePath); - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - if (deviceCacheEntry != null) { - Template template = templateManager.getTemplate(deviceCacheEntry.getTemplateId()); - schemaTree.appendTemplateDevice( - devicePath, template.isDirectAligned(), template.getId(), template); - schemaTree.setDatabases(Collections.singleton(deviceCacheEntry.getDatabase())); - } - return schemaTree; - } - - /** - * CONFORM indicates that the provided devicePath had been cached as a template activated path, - * ensuring that the alignment of the device, as well as the name and schema of every measurement - * are consistent with the cache. - * - * @param computation - * @return true if conform to template cache, which means no need to fetch or create anymore - */ - public List compute(ISchemaComputation computation) { - List indexOfMissingMeasurements = new ArrayList<>(); - PartialPath devicePath = computation.getDevicePath(); - String[] measurements = computation.getMeasurements(); - DeviceCacheEntry deviceCacheEntry = cache.getIfPresent(devicePath); - if (deviceCacheEntry == null) { - for (int i = 0; i < measurements.length; i++) { - indexOfMissingMeasurements.add(i); - } - return indexOfMissingMeasurements; - } - - computation.computeDevice( - templateManager.getTemplate(deviceCacheEntry.getTemplateId()).isDirectAligned()); - Map templateSchema = - templateManager.getTemplate(deviceCacheEntry.getTemplateId()).getSchemaMap(); - for (int i = 0; i < measurements.length; i++) { - if (!templateSchema.containsKey(measurements[i])) { - indexOfMissingMeasurements.add(i); - continue; - } - IMeasurementSchema schema = templateSchema.get(measurements[i]); - computation.computeMeasurement( - i, - new IMeasurementSchemaInfo() { - @Override - public String getName() { - return schema.getMeasurementId(); - } - - @Override - public IMeasurementSchema getSchema() { - if (isLogicalView()) { - return new LogicalViewSchema( - schema.getMeasurementId(), ((LogicalViewSchema) schema).getExpression()); - } else { - return this.getSchemaAsMeasurementSchema(); - } - } - - @Override - public MeasurementSchema getSchemaAsMeasurementSchema() { - return new MeasurementSchema( - schema.getMeasurementId(), - schema.getType(), - schema.getEncodingType(), - schema.getCompressor()); - } - - @Override - public LogicalViewSchema getSchemaAsLogicalViewSchema() { - throw new RuntimeException( - new UnsupportedOperationException( - "Function getSchemaAsLogicalViewSchema is not supported in DeviceUsingTemplateSchemaCache.")); - } - - @Override - public Map getTagMap() { - return null; - } - - @Override - public Map getAttributeMap() { - return null; - } - - @Override - public String getAlias() { - return null; - } - - @Override - public boolean isLogicalView() { - return schema.isLogicalView(); - } - }); - } - return indexOfMissingMeasurements; - } - - public void put(PartialPath path, String database, Integer id) { - cache.put(path, new DeviceCacheEntry(database, id)); - } - - public void invalidateCache() { - cache.invalidateAll(); - } - - public void invalidateCache(String database) { - for (PartialPath key : cache.asMap().keySet()) { - if (key.startsWith(database)) { - cache.invalidate(key); - } - } - } - - public void invalidateCache(List partialPathList) { - for (PartialPath path : partialPathList) { - for (PartialPath key : cache.asMap().keySet()) { - if (key.startsWith(path.getDevice())) { - cache.invalidate(key); - } - } - } - } - - private static class DeviceCacheEntry { - - private final String database; - - private final int templateId; - - private DeviceCacheEntry(String database, int templateId) { - this.database = database.intern(); - this.templateId = templateId; - } - - public int getTemplateId() { - return templateId; - } - - public String getDatabase() { - return database; - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/IDeviceSchema.java similarity index 65% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/IDeviceSchema.java index c7782ef260584..473429ac0a660 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/ILastCacheValue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/IDeviceSchema.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -17,23 +17,8 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.value; - -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.TsPrimitiveType; - -// this interface declares the simplest storage operation of lastCacheValue -public interface ILastCacheValue { - - long getTimestamp(); - - TsPrimitiveType getValue(); - - void setTimestamp(long timestamp); - - void setValue(TsPrimitiveType value); - - TimeValuePair getTimeValuePair(); +package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; +public interface IDeviceSchema { int estimateSize(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java index d97d5ac30f2c7..d048692962281 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java @@ -21,50 +21,47 @@ import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.db.queryengine.common.schematree.IMeasurementSchemaInfo; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.ILastCacheContainer; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.LastCacheContainer; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import java.util.Map; public class SchemaCacheEntry implements IMeasurementSchemaInfo { - private final String storageGroup; + /** + * MeasurementSchema takes 75B + * + *

    + *
  • MeasurementSchema + *
      + *
    • Object header, 8B + *
    • String measurementId basic, 8 + 8 + 4 + 8 + 4 = 32B + *
    • type, encoding, compressor, 3 B + *
    • encodingConverter, 8 + 8 + 8 = 24B + *
    • props, 8B + *
    + *
+ */ + private static final int INSTANCE_SIZE = + (int) RamUsageEstimator.shallowSizeOfInstance(SchemaCacheEntry.class) + 75; private final IMeasurementSchema iMeasurementSchema; - private final Map tagMap; - private final boolean isAligned; - @SuppressWarnings("java:S3077") - private volatile ILastCacheContainer lastCacheContainer = null; + private final Map tagMap; public SchemaCacheEntry( - String storageGroup, - IMeasurementSchema iMeasurementSchema, - Map tagMap, - boolean isAligned) { - this.storageGroup = storageGroup.intern(); + final @Nonnull IMeasurementSchema iMeasurementSchema, + final @Nullable Map tagMap) { this.iMeasurementSchema = iMeasurementSchema; - this.isAligned = isAligned; this.tagMap = tagMap; } - public String getSchemaEntryId() { - return iMeasurementSchema.getMeasurementId(); - } - - public String getStorageGroup() { - return storageGroup; - } - - public IMeasurementSchema getIMeasurementSchema() { - return iMeasurementSchema; - } - @Override public Map getTagMap() { return tagMap; @@ -79,59 +76,10 @@ public TSDataType getTsDataType() { return iMeasurementSchema.getType(); } - public boolean isAligned() { - return isAligned; - } - - public ILastCacheContainer getLastCacheContainer() { - return lastCacheContainer; - } - - public int updateLastCache( - TimeValuePair timeValuePair, boolean highPriorityUpdate, Long latestFlushedTime) { - - if (lastCacheContainer == null) { - synchronized (this) { - if (lastCacheContainer == null) { - ILastCacheContainer tmp = new LastCacheContainer(); - int changeSize = tmp.estimateSize(); - changeSize += tmp.updateCachedLast(timeValuePair, highPriorityUpdate, latestFlushedTime); - lastCacheContainer = tmp; - return changeSize; - } - } - } - return lastCacheContainer.updateCachedLast( - timeValuePair, highPriorityUpdate, latestFlushedTime); - } - - /** - * Total basic 100B - * - *
    - *
  • SchemaCacheEntry Object header, 8B - *
  • isAligned, 1B - *
  • LastCacheContainer reference, 8B - *
  • MeasurementSchema - *
      - *
    • Reference, 8B - *
    • Object header, 8B - *
    • String measurementId basic, 8 + 8 + 4 + 8 + 4 = 32B - *
    • type, encoding, compressor, 3 B - *
    • encodingConverter, 8 + 8 + 8 = 24B - *
    • props, 8B - *
    - *
- */ - public static int estimateSize(SchemaCacheEntry schemaCacheEntry) { - // each char takes 2B in Java - int lastCacheContainerSize = - schemaCacheEntry.getLastCacheContainer() == null - ? 0 - : schemaCacheEntry.getLastCacheContainer().estimateSize(); - return 100 - + 2 * schemaCacheEntry.getIMeasurementSchema().getMeasurementId().length() - + lastCacheContainerSize; + public static int estimateSize(final SchemaCacheEntry schemaCacheEntry) { + return INSTANCE_SIZE + + 2 * schemaCacheEntry.getSchema().getMeasurementId().length() + + (int) RamUsageEstimator.sizeOfMap(schemaCacheEntry.getTagMap()); } @Override @@ -169,12 +117,4 @@ public String getAlias() { public boolean isLogicalView() { return this.iMeasurementSchema.isLogicalView(); } - - public int invalidateLastCache() { - if (this.lastCacheContainer == null || this.lastCacheContainer.getCachedLast() == null) { - return 0; - } - - return this.lastCacheContainer.invalidateLastCache(); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java deleted file mode 100644 index 51f22abd1e968..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema; - -import org.apache.iotdb.commons.path.MeasurementPath; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.schema.view.LogicalViewSchema; -import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.metadata.view.InsertNonWritableViewException; -import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheComputation; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheUpdating; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.DataNodeLastCacheManager; -import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaComputation; - -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.IntFunction; -import java.util.function.IntPredicate; - -public class TimeSeriesSchemaCache { - - private static final Logger logger = LoggerFactory.getLogger(DataNodeSchemaCache.class); - private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - - // - private final IDualKeyCache dualKeyCache; - - TimeSeriesSchemaCache() { - DualKeyCacheBuilder dualKeyCacheBuilder = - new DualKeyCacheBuilder<>(); - dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy( - DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy())) - .memoryCapacity(config.getAllocateMemoryForSchemaCache()) - .firstKeySizeComputer(PartialPath::estimateSize) - .secondKeySizeComputer(s -> 32 + 2 * s.length()) - .valueSizeComputer(SchemaCacheEntry::estimateSize) - .build(); - } - - public long getHitCount() { - return dualKeyCache.stats().hitCount(); - } - - public long getRequestCount() { - return dualKeyCache.stats().requestCount(); - } - - /** - * Get SchemaEntity info without auto create schema - * - * @param devicePath should not be measurementPath or AlignedPath - * @param measurements - * @return timeseries partialPath and its SchemaEntity - */ - public ClusterSchemaTree get(PartialPath devicePath, String[] measurements) { - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - Set storageGroupSet = new HashSet<>(); - - dualKeyCache.compute( - new IDualKeyCacheComputation() { - @Override - public PartialPath getFirstKey() { - return devicePath; - } - - @Override - public String[] getSecondKeyList() { - return measurements; - } - - @Override - public void computeValue(int index, SchemaCacheEntry value) { - if (value != null) { - schemaTree.appendSingleMeasurement( - devicePath.concatNode(value.getSchemaEntryId()), - value.getIMeasurementSchema(), - value.getTagMap(), - null, - null, - value.isAligned()); - storageGroupSet.add(value.getStorageGroup()); - } - } - }); - schemaTree.setDatabases(storageGroupSet); - return schemaTree; - } - - public ClusterSchemaTree get(PartialPath fullPath) { - SchemaCacheEntry schemaCacheEntry = - dualKeyCache.get(fullPath.getDevicePath(), fullPath.getMeasurement()); - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - if (schemaCacheEntry != null) { - schemaTree.appendSingleMeasurement( - fullPath, - schemaCacheEntry.getIMeasurementSchema(), - schemaCacheEntry.getTagMap(), - null, - null, - schemaCacheEntry.isAligned()); - schemaTree.setDatabases(Collections.singleton(schemaCacheEntry.getStorageGroup())); - } - return schemaTree; - } - - public List computeAndRecordLogicalView(ISchemaComputation schemaComputation) { - List indexOfMissingMeasurements = new ArrayList<>(); - final AtomicBoolean isFirstNonViewMeasurement = new AtomicBoolean(true); - dualKeyCache.compute( - new IDualKeyCacheComputation() { - @Override - public PartialPath getFirstKey() { - return schemaComputation.getDevicePath(); - } - - @Override - public String[] getSecondKeyList() { - return schemaComputation.getMeasurements(); - } - - @Override - public void computeValue(int index, SchemaCacheEntry value) { - if (value == null) { - indexOfMissingMeasurements.add(index); - } else { - if (isFirstNonViewMeasurement.get() && (!value.isLogicalView())) { - schemaComputation.computeDevice(value.isAligned()); - isFirstNonViewMeasurement.getAndSet(false); - } - schemaComputation.computeMeasurement(index, value); - } - } - }); - return indexOfMissingMeasurements; - } - - public Pair, List> computeSourceOfLogicalView( - ISchemaComputation schemaComputation) { - List indexOfMissingMeasurements = new ArrayList<>(); - List missedPathStringList = new ArrayList<>(); - Pair beginToEnd = schemaComputation.getRangeOfLogicalViewSchemaListRecorded(); - List logicalViewSchemaList = schemaComputation.getLogicalViewSchemaList(); - List indexListOfLogicalViewPaths = schemaComputation.getIndexListOfLogicalViewPaths(); - for (int i = beginToEnd.left; i < beginToEnd.right; i++) { - LogicalViewSchema logicalViewSchema = logicalViewSchemaList.get(i); - final int realIndex = indexListOfLogicalViewPaths.get(i); - final int recordMissingIndex = i; - if (!logicalViewSchema.isWritable()) { - PartialPath path = schemaComputation.getDevicePath(); - path = path.concatNode(schemaComputation.getMeasurements()[realIndex]); - throw new RuntimeException(new InsertNonWritableViewException(path.getFullPath())); - } - PartialPath fullPath = logicalViewSchema.getSourcePathIfWritable(); - dualKeyCache.compute( - new IDualKeyCacheComputation() { - @Override - public PartialPath getFirstKey() { - return fullPath.getDevicePath(); - } - - @Override - public String[] getSecondKeyList() { - return new String[] {fullPath.getMeasurement()}; - } - - @Override - public void computeValue(int index, SchemaCacheEntry value) { - if (value == null) { - indexOfMissingMeasurements.add(recordMissingIndex); - } else { - // Can not call function computeDevice here, because the value is source of one - // view, but schemaComputation is the device in this insert statement. The - // computation between them is miss matched. - if (value.isLogicalView()) { - // does not support views in views - throw new RuntimeException( - new UnsupportedOperationException( - String.format( - "The source of view [%s] is also a view! Nested view is unsupported! " - + "Please check it.", - fullPath))); - } - schemaComputation.computeMeasurementOfView(realIndex, value, value.isAligned()); - } - } - }); - } - for (int index : indexOfMissingMeasurements) { - missedPathStringList.add(logicalViewSchemaList.get(index).getSourcePathStringIfWritable()); - } - return new Pair<>(indexOfMissingMeasurements, missedPathStringList); - } - - public void putSingleMeasurementPath(String storageGroup, MeasurementPath measurementPath) { - SchemaCacheEntry schemaCacheEntry = - new SchemaCacheEntry( - storageGroup, - measurementPath.getMeasurementSchema(), - measurementPath.getTagMap(), - measurementPath.isUnderAlignedEntity()); - dualKeyCache.put( - measurementPath.getDevicePath(), measurementPath.getMeasurement(), schemaCacheEntry); - } - - public TimeValuePair getLastCache(PartialPath seriesPath) { - SchemaCacheEntry entry = - dualKeyCache.get(seriesPath.getDevicePath(), seriesPath.getMeasurement()); - if (null == entry) { - return null; - } - - return DataNodeLastCacheManager.getLastCache(entry); - } - - public boolean getLastCache(final Map> inputMap) { - return dualKeyCache.batchGet(inputMap, DataNodeLastCacheManager::getLastCache); - } - - /** get SchemaCacheEntry and update last cache */ - @TestOnly - public void updateLastCache( - PartialPath devicePath, - String measurement, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - SchemaCacheEntry entry = dualKeyCache.get(devicePath, measurement); - if (null == entry) { - return; - } - - DataNodeLastCacheManager.updateLastCache( - entry, timeValuePair, highPriorityUpdate, latestFlushedTime); - } - - /** get SchemaCacheEntry and update last cache by device */ - public void updateLastCache( - String database, - PartialPath devicePath, - String[] measurements, - MeasurementSchema[] measurementSchemas, - boolean isAligned, - IntFunction timeValuePairProvider, - IntPredicate shouldUpdateProvider, - boolean highPriorityUpdate, - Long latestFlushedTime) { - SchemaCacheEntry entry; - List missingMeasurements = new ArrayList<>(); - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public PartialPath getFirstKey() { - return devicePath; - } - - @Override - public String[] getSecondKeyList() { - return measurements; - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - if (!shouldUpdateProvider.test(index)) { - return 0; - } - if (value == null) { - missingMeasurements.add(index); - return 0; - } else { - return DataNodeLastCacheManager.updateLastCache( - value, timeValuePairProvider.apply(index), highPriorityUpdate, latestFlushedTime); - } - } - }); - - for (int index : missingMeasurements) { - entry = dualKeyCache.get(devicePath, measurements[index]); - if (entry == null) { - synchronized (dualKeyCache) { - entry = dualKeyCache.get(devicePath, measurements[index]); - if (null == entry && measurementSchemas != null) { - entry = new SchemaCacheEntry(database, measurementSchemas[index], null, isAligned); - dualKeyCache.put(devicePath, measurements[index], entry); - } - } - } - } - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public PartialPath getFirstKey() { - return devicePath; - } - - @Override - public String[] getSecondKeyList() { - return missingMeasurements.stream().map(i -> measurements[i]).toArray(String[]::new); - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - return DataNodeLastCacheManager.updateLastCache( - value, - timeValuePairProvider.apply(missingMeasurements.get(index)), - highPriorityUpdate, - latestFlushedTime); - } - }); - } - - /** - * get or create SchemaCacheEntry and update last cache, only support non-aligned sensor or - * aligned sensor without only one sub sensor - */ - public void updateLastCache( - String storageGroup, - MeasurementPath measurementPath, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - PartialPath seriesPath = measurementPath.transformToPartialPath(); - SchemaCacheEntry entry = - dualKeyCache.get(seriesPath.getDevicePath(), seriesPath.getMeasurement()); - if (null == entry) { - synchronized (dualKeyCache) { - entry = dualKeyCache.get(seriesPath.getDevicePath(), seriesPath.getMeasurement()); - if (null == entry) { - entry = - new SchemaCacheEntry( - storageGroup, - measurementPath.getMeasurementSchema(), - measurementPath.getTagMap(), - measurementPath.isUnderAlignedEntity()); - dualKeyCache.put(seriesPath.getDevicePath(), seriesPath.getMeasurement(), entry); - } - } - } - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public PartialPath getFirstKey() { - return measurementPath.getDevicePath(); - } - - @Override - public String[] getSecondKeyList() { - return new String[] {measurementPath.getMeasurement()}; - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - return DataNodeLastCacheManager.updateLastCache( - value, timeValuePair, highPriorityUpdate, latestFlushedTime); - } - }); - } - - public void invalidateAll() { - dualKeyCache.invalidateAll(); - } - - public void invalidate(String database) { - dualKeyCache.invalidate(database); - } - - public void invalidateLastCache(PartialPath path) { - if (!path.hasWildcard()) { - SchemaCacheEntry entry = dualKeyCache.get(path.getDevicePath(), path.getMeasurement()); - if (null == entry) { - return; - } - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public PartialPath getFirstKey() { - return path.getDevicePath(); - } - - @Override - public String[] getSecondKeyList() { - return new String[] {path.getMeasurement()}; - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - return -DataNodeLastCacheManager.invalidateLastCache(value); - } - }); - } else { - dualKeyCache.invalidateLastCache(path); - } - } - - public void invalidateDataRegionLastCache(String database) { - dualKeyCache.invalidateDataRegionLastCache(database); - } - - public void invalidate(List partialPathList) { - dualKeyCache.invalidate(partialPathList); - } - - public void cleanUp() { - dualKeyCache.cleanUp(); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCache.java index 397ca18e89fee..1a2d788c9f792 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCache.java @@ -19,14 +19,12 @@ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.utils.TestOnly; - import javax.annotation.concurrent.GuardedBy; -import java.util.List; import java.util.Map; -import java.util.function.Function; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import java.util.function.ToIntFunction; /** * This interfaces defines the behaviour of a dual key cache. A dual key cache supports manage cache @@ -39,33 +37,51 @@ public interface IDualKeyCache { /** Get the cache value with given first key and second key. */ - V get(FK firstKey, SK secondKey); + V get(final FK firstKey, final SK secondKey); - boolean batchGet(final Map> inputMap, final Function mappingFunction); + boolean batchApply( + final Map> inputMap, final BiFunction mappingFunction); /** - * Traverse target cache values via given first key and second keys provided in computation and - * execute the defined computation logic. The computation is read only. + * Update the existing value. The updater shall return the difference caused by the update, + * because we do not want to call "valueSizeComputer" twice, which may include abundant useless + * calculations. + * + *

Warning: This method is without any locks for performance concerns. The caller shall ensure + * the concurrency safety for the value update. + * + * @param createIfNotExists put the value to cache iff it does not exist, */ - void compute(IDualKeyCacheComputation computation); + void update( + final FK firstKey, + final SK secondKey, + final V value, + final ToIntFunction updater, + final boolean createIfNotExists); /** - * Traverse target cache values via given first key and second keys provided in computation and - * execute the defined computation logic. Value can be updated in this computation. + * Update all the existing value with {@link SK} and a the {@link SK}s matching the given + * predicate. The updater shall return the difference caused by the update, because we do not want + * to call "valueSizeComputer" twice, which may include abundant useless calculations. + * + *

Warning: This method is without any locks for performance concerns. The caller shall ensure + * the concurrency safety for the value update. */ - void update(IDualKeyCacheUpdating updating); - - /** put the cache value into cache */ - void put(FK firstKey, SK secondKey, V value); + void update( + final FK firstKey, final Predicate secondKeyChecker, final ToIntFunction updater); /** - * Invalidate last cache in datanode schema cache. Do not invalidate time series cache. + * Update all the existing value with {@link SK} and a the {@link SK}s matching the given + * predicate. The updater shall return the difference caused by the update, because we do not want + * to call "valueSizeComputer" twice, which may include abundant useless calculations. * - * @param partialPathList + *

Warning: This method is without any locks for performance concerns. The caller shall ensure + * the concurrency safety for the value update. */ - void invalidateLastCache(PartialPath partialPath); - - void invalidateDataRegionLastCache(String database); + void update( + final Predicate firstKeyChecker, + final Predicate secondKeyChecker, + final ToIntFunction updater); /** * Invalidate all cache values in the cache and clear related cache keys. The cache status and @@ -74,29 +90,21 @@ public interface IDualKeyCache { @GuardedBy("DataNodeSchemaCache#writeLock") void invalidateAll(); - /** - * Invalidate cache values in the cache and clear related cache keys. The cache status and - * statistics won't be clear and they can still be accessed via cache.stats(). - */ - @GuardedBy("DataNodeSchemaCache#writeLock") - void invalidate(String database); + /** Return all the current cache status and statistics. */ + IDualKeyCacheStats stats(); - /** - * Invalidate cache values in the cache and clear related cache keys. The cache status and - * statistics won't be clear and they can still be accessed via cache.stats(). - */ + /** remove all entries for firstKey */ @GuardedBy("DataNodeSchemaCache#writeLock") - void invalidate(List partialPathList); + void invalidate(final FK firstKey); - /** - * Clean up all data and info of this cache, including cache keys, cache values and cache stats. - */ + /** remove matched entry */ @GuardedBy("DataNodeSchemaCache#writeLock") - void cleanUp(); + void invalidate(final FK firstKey, final SK secondKey); - /** Return all the current cache status and statistics. */ - IDualKeyCacheStats stats(); + @GuardedBy("DataNodeSchemaCache#writeLock") + void invalidate(final FK firstKey, final Predicate secondKeyChecker); - @TestOnly - void evictOneEntry(); + /** remove all entries matching the firstKey and the secondKey */ + @GuardedBy("DataNodeSchemaCache#writeLock") + void invalidate(final Predicate firstKeyChecker, final Predicate secondKeyChecker); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java index 9ed842cc602d5..9247e20a9c48a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java @@ -39,4 +39,10 @@ public interface IDualKeyCacheStats { /** Return current memory usage of dual key cache. */ long memoryUsage(); + + /** Return capacity of dual key cache. */ + long capacity(); + + /** Return entries num of dual key cache */ + long entriesCount(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheEntryGroupImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheEntryGroupImpl.java index b15f6d153fca0..54c53a15d5038 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheEntryGroupImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheEntryGroupImpl.java @@ -19,21 +19,36 @@ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl; +import org.apache.tsfile.utils.RamUsageEstimator; + import java.util.Iterator; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; +import java.util.function.Function; public class CacheEntryGroupImpl> implements ICacheEntryGroup { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(CacheEntryGroupImpl.class) + + RamUsageEstimator.shallowSizeOfInstance(AtomicLong.class) + + RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class) + // Calculate the outer entry of the "firstKeyMap" here + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY; + private final FK firstKey; private final Map cacheEntryMap = new ConcurrentHashMap<>(); + private final ICacheSizeComputer sizeComputer; + private final AtomicLong memory; - CacheEntryGroupImpl(FK firstKey) { + CacheEntryGroupImpl(final FK firstKey, final ICacheSizeComputer sizeComputer) { this.firstKey = firstKey; + this.sizeComputer = sizeComputer; + this.memory = new AtomicLong(INSTANCE_SIZE + sizeComputer.computeFirstKeySize(firstKey)); } @Override @@ -42,7 +57,7 @@ public FK getFirstKey() { } @Override - public T getCacheEntry(SK secondKey) { + public T getCacheEntry(final SK secondKey) { return secondKey == null ? null : cacheEntryMap.get(secondKey); } @@ -52,13 +67,23 @@ public Iterator> getAllCacheEntries() { } @Override - public T computeCacheEntry(SK secondKey, BiFunction computation) { - return cacheEntryMap.compute(secondKey, computation); + public T computeCacheEntry( + final SK secondKey, final Function> computation) { + return cacheEntryMap.compute(secondKey, computation.apply(memory)); } @Override - public T removeCacheEntry(SK secondKey) { - return cacheEntryMap.remove(secondKey); + public long removeCacheEntry(final SK secondKey) { + final T result = cacheEntryMap.remove(secondKey); + if (Objects.nonNull(result)) { + final long delta = + sizeComputer.computeSecondKeySize(result.getSecondKey()) + + sizeComputer.computeValueSize(result.getValue()) + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY; + memory.addAndGet(-delta); + return delta; + } + return 0; } @Override @@ -67,10 +92,24 @@ public boolean isEmpty() { } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CacheEntryGroupImpl that = (CacheEntryGroupImpl) o; + public long getMemory() { + return memory.get(); + } + + @Override + public int getEntriesCount() { + return cacheEntryMap.size(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final CacheEntryGroupImpl that = (CacheEntryGroupImpl) o; return Objects.equals(firstKey, that.firstKey); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java index 817092d3eb909..0628d35db164e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheStats; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; class CacheStats implements IDualKeyCacheStats { @@ -30,25 +31,23 @@ class CacheStats implements IDualKeyCacheStats { private final long memoryThreshold; - private final AtomicLong memoryUsage = new AtomicLong(0); + private final Supplier memoryComputation; + private final Supplier entriesComputation; private final AtomicLong requestCount = new AtomicLong(0); private final AtomicLong hitCount = new AtomicLong(0); - CacheStats(long memoryCapacity) { + CacheStats( + long memoryCapacity, + final Supplier memoryComputation, + final Supplier entriesComputation) { this.memoryThreshold = (long) (memoryCapacity * MEMORY_THRESHOLD_RATIO); + this.memoryComputation = memoryComputation; + this.entriesComputation = entriesComputation; } - void increaseMemoryUsage(int size) { - memoryUsage.getAndAdd(size); - } - - void decreaseMemoryUsage(int size) { - memoryUsage.getAndAdd(-size); - } - - boolean isExceedMemoryCapacity() { - return memoryUsage.get() > memoryThreshold; + long getExceedMemory() { + return memoryUsage() - memoryThreshold; } void recordHit(int num) { @@ -93,16 +92,16 @@ public double hitRate() { @Override public long memoryUsage() { - return memoryUsage.get(); + return memoryComputation.get(); } - void reset() { - resetMemoryUsage(); - hitCount.set(0); - requestCount.set(0); + @Override + public long capacity() { + return memoryThreshold; } - void resetMemoryUsage() { - memoryUsage.set(0); + @Override + public long entriesCount() { + return entriesComputation.get(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java index 5a4024bb31d28..5e614fa5fd351 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java @@ -19,25 +19,23 @@ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathPatternUtil; -import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.SchemaCacheEntry; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheComputation; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheStats; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheUpdating; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.DataNodeLastCacheManager; + +import org.apache.tsfile.utils.RamUsageEstimator; + +import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; -import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.ToIntFunction; class DualKeyCacheImpl> implements IDualKeyCache { @@ -52,22 +50,22 @@ class DualKeyCacheImpl> private final CacheStats cacheStats; DualKeyCacheImpl( - ICacheEntryManager cacheEntryManager, - ICacheSizeComputer sizeComputer, - long memoryCapacity) { + final ICacheEntryManager cacheEntryManager, + final ICacheSizeComputer sizeComputer, + final long memoryCapacity) { this.cacheEntryManager = cacheEntryManager; this.sizeComputer = sizeComputer; - this.cacheStats = new CacheStats(memoryCapacity); + this.cacheStats = new CacheStats(memoryCapacity, this::getMemory, this::getEntriesCount); } @Override - public V get(FK firstKey, SK secondKey) { - ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); + public V get(final FK firstKey, final SK secondKey) { + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); if (cacheEntryGroup == null) { cacheStats.recordMiss(1); return null; } else { - T cacheEntry = cacheEntryGroup.getCacheEntry(secondKey); + final T cacheEntry = cacheEntryGroup.getCacheEntry(secondKey); if (cacheEntry == null) { cacheStats.recordMiss(1); return null; @@ -80,8 +78,8 @@ public V get(FK firstKey, SK secondKey) { } @Override - public boolean batchGet( - final Map> inputMap, final Function mappingFunction) { + public boolean batchApply( + final Map> inputMap, final BiFunction mappingFunction) { for (final Map.Entry> fkMapEntry : inputMap.entrySet()) { final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(fkMapEntry.getKey()); if (cacheEntryGroup == null) { @@ -92,317 +90,260 @@ public boolean batchGet( if (cacheEntry == null) { return false; } - skrEntry.setValue(mappingFunction.apply(cacheEntry.getValue())); + if (!mappingFunction.apply(cacheEntry.getValue(), skrEntry.getValue())) { + return false; + } } } return true; } @Override - public void compute(IDualKeyCacheComputation computation) { - FK firstKey = computation.getFirstKey(); + public void update( + final FK firstKey, + final @Nonnull SK secondKey, + final V value, + final ToIntFunction updater, + final boolean createIfNotExists) { + ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); - SK[] secondKeyList = computation.getSecondKeyList(); - if (cacheEntryGroup == null) { - for (int i = 0; i < secondKeyList.length; i++) { - computation.computeValue(i, null); - } - cacheStats.recordMiss(secondKeyList.length); - } else { - T cacheEntry; - int hitCount = 0; - for (int i = 0; i < secondKeyList.length; i++) { - cacheEntry = cacheEntryGroup.getCacheEntry(secondKeyList[i]); - if (cacheEntry == null) { - computation.computeValue(i, null); - } else { - computation.computeValue(i, cacheEntry.getValue()); - cacheEntryManager.access(cacheEntry); - hitCount++; - } + if (Objects.isNull(cacheEntryGroup)) { + if (createIfNotExists) { + cacheEntryGroup = new CacheEntryGroupImpl<>(firstKey, sizeComputer); + firstKeyMap.put(firstKey, cacheEntryGroup); + } else { + return; } - cacheStats.recordHit(hitCount); - cacheStats.recordMiss(secondKeyList.length - hitCount); } - } - @Override - public void update(IDualKeyCacheUpdating updating) { - FK firstKey = updating.getFirstKey(); - ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); - SK[] secondKeyList = updating.getSecondKeyList(); - if (cacheEntryGroup == null) { - for (int i = 0; i < secondKeyList.length; i++) { - updating.updateValue(i, null); - } - cacheStats.recordMiss(secondKeyList.length); - } else { - T cacheEntry; - int hitCount = 0; - for (int i = 0; i < secondKeyList.length; i++) { - cacheEntry = cacheEntryGroup.getCacheEntry(secondKeyList[i]); - if (cacheEntry == null) { - updating.updateValue(i, null); - } else { - int changeSize = 0; - synchronized (cacheEntry) { - if (cacheEntry.getBelongedGroup() != null) { - // Only update the value when the cache entry is not evicted. - // If the cache entry is evicted, getBelongedGroup is null. - // Synchronized is to guarantee the cache entry is not evicted during the update. - changeSize = updating.updateValue(i, cacheEntry.getValue()); - cacheEntryManager.access(cacheEntry); - if (changeSize != 0) { - cacheStats.increaseMemoryUsage(changeSize); + final ICacheEntryGroup finalCacheEntryGroup = cacheEntryGroup; + cacheEntryGroup.computeCacheEntry( + secondKey, + memory -> + (sk, cacheEntry) -> { + if (Objects.isNull(cacheEntry)) { + if (!createIfNotExists) { + return null; + } + cacheEntry = + cacheEntryManager.createCacheEntry(secondKey, value, finalCacheEntryGroup); + cacheEntryManager.put(cacheEntry); + memory.getAndAdd( + sizeComputer.computeSecondKeySize(sk) + + sizeComputer.computeValueSize(cacheEntry.getValue()) + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY); } - } - } - if (changeSize != 0 && cacheStats.isExceedMemoryCapacity()) { - executeCacheEviction(changeSize); - } - hitCount++; - } - } - cacheStats.recordHit(hitCount); - cacheStats.recordMiss(secondKeyList.length - hitCount); - } + memory.getAndAdd(updater.applyAsInt(cacheEntry.getValue())); + return cacheEntry; + }); + + mayEvict(); } @Override - public void put(FK firstKey, SK secondKey, V value) { - int usedMemorySize = putToCache(firstKey, secondKey, value); - cacheStats.increaseMemoryUsage(usedMemorySize); - if (cacheStats.isExceedMemoryCapacity()) { - executeCacheEviction(usedMemorySize); + public void update( + final FK firstKey, final Predicate secondKeyChecker, final ToIntFunction updater) { + final ICacheEntryGroup entryGroup = firstKeyMap.get(firstKey); + if (Objects.nonNull(entryGroup)) { + entryGroup + .getAllCacheEntries() + .forEachRemaining( + entry -> { + if (!secondKeyChecker.test(entry.getKey())) { + return; + } + entryGroup.computeCacheEntry( + entry.getKey(), + memory -> + (secondKey, cacheEntry) -> { + if (Objects.nonNull(cacheEntry)) { + memory.getAndAdd(updater.applyAsInt(cacheEntry.getValue())); + } + return cacheEntry; + }); + }); } + mayEvict(); } - private int putToCache(FK firstKey, SK secondKey, V value) { - AtomicInteger usedMemorySize = new AtomicInteger(0); - firstKeyMap.compute( - firstKey, - (k, cacheEntryGroup) -> { - if (cacheEntryGroup == null) { - cacheEntryGroup = new CacheEntryGroupImpl<>(firstKey); - usedMemorySize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); - } - ICacheEntryGroup finalCacheEntryGroup = cacheEntryGroup; - cacheEntryGroup.computeCacheEntry( - secondKey, - (sk, cacheEntry) -> { - if (cacheEntry == null) { - cacheEntry = - cacheEntryManager.createCacheEntry(secondKey, value, finalCacheEntryGroup); - cacheEntryManager.put(cacheEntry); - usedMemorySize.getAndAdd(sizeComputer.computeSecondKeySize(sk)); - } else { - V existingValue = cacheEntry.getValue(); - if (existingValue != value && !existingValue.equals(value)) { - cacheEntry.replaceValue(value); - usedMemorySize.getAndAdd(-sizeComputer.computeValueSize(existingValue)); + @Override + public void update( + final Predicate firstKeyChecker, + final Predicate secondKeyChecker, + final ToIntFunction updater) { + for (final FK firstKey : firstKeyMap.getAllKeys()) { + if (!firstKeyChecker.test(firstKey)) { + continue; + } + final ICacheEntryGroup entryGroup = firstKeyMap.get(firstKey); + if (Objects.nonNull(entryGroup)) { + entryGroup + .getAllCacheEntries() + .forEachRemaining( + entry -> { + if (!secondKeyChecker.test(entry.getKey())) { + return; } - // update the cache status - cacheEntryManager.access(cacheEntry); - } - usedMemorySize.getAndAdd(sizeComputer.computeValueSize(value)); - return cacheEntry; - }); - return cacheEntryGroup; - }); - return usedMemorySize.get(); + entryGroup.computeCacheEntry( + entry.getKey(), + memory -> + (secondKey, cacheEntry) -> { + memory.getAndAdd(updater.applyAsInt(cacheEntry.getValue())); + return cacheEntry; + }); + }); + } + mayEvict(); + } } - /** - * Each thread putting new cache value only needs to evict cache values, total memory equals that - * the new cache value occupied. - */ - private void executeCacheEviction(int targetSize) { - int evictedSize; - while (targetSize > 0 && cacheStats.memoryUsage() > 0) { - evictedSize = evictOneCacheEntry(); - cacheStats.decreaseMemoryUsage(evictedSize); - targetSize -= evictedSize; + private void mayEvict() { + long exceedMemory; + while ((exceedMemory = cacheStats.getExceedMemory()) > 0) { + // Not compute each time to save time when FK is too many + // The hard-coded size is 100 + do { + exceedMemory -= evictOneCacheEntry(); + } while (exceedMemory > 0 && firstKeyMap.size() > 100); } } - private int evictOneCacheEntry() { - - ICacheEntry evictCacheEntry = cacheEntryManager.evict(); + // The returned delta may have some error, but it's OK + // Because the delta is only for loop round estimation + private long evictOneCacheEntry() { + final ICacheEntry evictCacheEntry = cacheEntryManager.evict(); if (evictCacheEntry == null) { return 0; } - synchronized (evictCacheEntry) { - AtomicInteger evictedSize = new AtomicInteger(0); - evictedSize.getAndAdd(sizeComputer.computeValueSize(evictCacheEntry.getValue())); - - ICacheEntryGroup belongedGroup = evictCacheEntry.getBelongedGroup(); - evictCacheEntry.setBelongedGroup(null); - belongedGroup.removeCacheEntry(evictCacheEntry.getSecondKey()); - evictedSize.getAndAdd(sizeComputer.computeSecondKeySize(evictCacheEntry.getSecondKey())); - - if (belongedGroup.isEmpty()) { - firstKeyMap.compute( - belongedGroup.getFirstKey(), - (firstKey, cacheEntryGroup) -> { - if (cacheEntryGroup == null) { - // has been removed by other threads - return null; - } - if (cacheEntryGroup.isEmpty()) { - evictedSize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); - return null; - } - // some other thread has put value to it - return cacheEntryGroup; - }); + final ICacheEntryGroup belongedGroup = evictCacheEntry.getBelongedGroup(); + evictCacheEntry.setBelongedGroup(null); + + long memory = belongedGroup.removeCacheEntry(evictCacheEntry.getSecondKey()); + + final ICacheEntryGroup cacheEntryGroup = + firstKeyMap.get(belongedGroup.getFirstKey()); + if (Objects.nonNull(cacheEntryGroup) && cacheEntryGroup.isEmpty()) { + // The removal is non-atomic, but it's ok because it's just a cache and does not affect the + // consistency if you evicts some entries being added + if (Objects.nonNull(firstKeyMap.remove(belongedGroup.getFirstKey()))) { + memory += + sizeComputer.computeFirstKeySize(belongedGroup.getFirstKey()) + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY; } - return evictedSize.get(); } + return memory; } @Override - public void invalidateLastCache(PartialPath path) { - String measurement = path.getMeasurement(); - PartialPath devicePath = path.getDevicePath(); - Function deviceFilter = null; - Function measurementFilter = null; - - if (PathPatternUtil.hasWildcard(devicePath.getFullPath())) { - deviceFilter = d -> devicePath.matchFullPath((PartialPath) d); - } - if (PathPatternUtil.isMultiLevelMatchWildcard(measurement)) { - measurementFilter = m -> true; - } - if (deviceFilter == null) { - deviceFilter = d -> d.equals(devicePath); - } - - if (measurementFilter == null) { - measurementFilter = m -> PathPatternUtil.isNodeMatch(measurement, m.toString()); - } + public void invalidateAll() { + firstKeyMap.clear(); + cacheEntryManager.cleanUp(); + } - for (FK device : firstKeyMap.getAllKeys()) { - if (Boolean.TRUE.equals(deviceFilter.apply(device))) { - ICacheEntryGroup entryGroup = firstKeyMap.get(device); - for (Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { - Map.Entry entry = it.next(); - if (Boolean.TRUE.equals(measurementFilter.apply(entry.getKey()))) { - T cacheEntry = entry.getValue(); - synchronized (cacheEntry) { - SchemaCacheEntry schemaCacheEntry = (SchemaCacheEntry) cacheEntry.getValue(); - cacheStats.decreaseMemoryUsage( - DataNodeLastCacheManager.invalidateLastCache(schemaCacheEntry)); - } - } - } - } - } + @Override + public IDualKeyCacheStats stats() { + return cacheStats; } @Override - public void invalidateDataRegionLastCache(String database) { - for (FK device : firstKeyMap.getAllKeys()) { - if (device.toString().startsWith(database)) { - ICacheEntryGroup entryGroup = firstKeyMap.get(device); - for (Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { - Map.Entry entry = it.next(); - T cacheEntry = entry.getValue(); - synchronized (cacheEntry) { - SchemaCacheEntry schemaCacheEntry = (SchemaCacheEntry) cacheEntry.getValue(); - cacheStats.decreaseMemoryUsage( - DataNodeLastCacheManager.invalidateLastCache(schemaCacheEntry)); - } - } + public void invalidate(final FK firstKey) { + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.remove(firstKey); + if (cacheEntryGroup != null) { + for (final Iterator> it = cacheEntryGroup.getAllCacheEntries(); + it.hasNext(); ) { + cacheEntryManager.invalidate(it.next().getValue()); } } } @Override - public void invalidateAll() { - executeInvalidateAll(); + public void invalidate(final FK firstKey, final SK secondKey) { + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); + if (Objects.isNull(cacheEntryGroup)) { + return; + } + + final T entry = cacheEntryGroup.getCacheEntry(secondKey); + if (Objects.nonNull(entry) && cacheEntryManager.invalidate(entry)) { + cacheEntryGroup.removeCacheEntry(entry.getSecondKey()); + } + + if (cacheEntryGroup.isEmpty()) { + firstKeyMap.remove(firstKey); + } } @Override - public void invalidate(String database) { - int estimateSize = 0; - for (FK device : firstKeyMap.getAllKeys()) { - if (device.toString().startsWith(database)) { - estimateSize += sizeComputer.computeFirstKeySize(device); - ICacheEntryGroup entryGroup = firstKeyMap.get(device); - for (Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { - Map.Entry entry = it.next(); - estimateSize += sizeComputer.computeSecondKeySize(entry.getKey()); - estimateSize += sizeComputer.computeValueSize(entry.getValue().getValue()); - cacheEntryManager.invalid(entry.getValue()); - } - firstKeyMap.remove(device); + public void invalidate(final FK firstKey, final Predicate secondKeyChecker) { + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); + if (Objects.isNull(cacheEntryGroup)) { + return; + } + for (final Iterator> it = cacheEntryGroup.getAllCacheEntries(); + it.hasNext(); ) { + final Map.Entry entry = it.next(); + if (secondKeyChecker.test(entry.getKey()) && cacheEntryManager.invalidate(entry.getValue())) { + cacheEntryGroup.removeCacheEntry(entry.getKey()); } } - cacheStats.decreaseMemoryUsage(estimateSize); + + if (cacheEntryGroup.isEmpty()) { + firstKeyMap.remove(firstKey); + } } @Override - public void invalidate(List partialPathList) { - int estimateSize = 0; - for (PartialPath path : partialPathList) { - String measurement = path.getMeasurement(); - Function deviceFilter; - Function measurementFilter; - if (PathPatternUtil.isMultiLevelMatchWildcard(measurement)) { - deviceFilter = d -> d.toString().startsWith(path.getDevicePath().getFullPath()); - measurementFilter = m -> true; - } else { - deviceFilter = d -> d.toString().equals(path.getDevicePath().getFullPath()); - measurementFilter = m -> PathPatternUtil.isNodeMatch(measurement, m.toString()); + public void invalidate( + final Predicate firstKeyChecker, final Predicate secondKeyChecker) { + for (final FK firstKey : firstKeyMap.getAllKeys()) { + if (!firstKeyChecker.test(firstKey)) { + continue; } - for (FK device : firstKeyMap.getAllKeys()) { - if (Boolean.TRUE.equals(deviceFilter.apply(device))) { - boolean allSKInvalid = true; - ICacheEntryGroup entryGroup = firstKeyMap.get(device); - for (Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { - Map.Entry entry = it.next(); - if (Boolean.TRUE.equals(measurementFilter.apply(entry.getKey()))) { - estimateSize += sizeComputer.computeSecondKeySize(entry.getKey()); - estimateSize += sizeComputer.computeValueSize(entry.getValue().getValue()); - cacheEntryManager.invalid(entry.getValue()); - it.remove(); - } else { - allSKInvalid = false; - } - } - if (allSKInvalid) { - firstKeyMap.remove(device); - estimateSize += sizeComputer.computeFirstKeySize(device); - } - } + + final ICacheEntryGroup cacheEntryGroup = firstKeyMap.get(firstKey); + if (Objects.isNull(cacheEntryGroup)) { + return; } - } - cacheStats.decreaseMemoryUsage(estimateSize); - } + for (final Iterator> it = cacheEntryGroup.getAllCacheEntries(); + it.hasNext(); ) { + final Map.Entry entry = it.next(); - private void executeInvalidateAll() { - firstKeyMap.clear(); - cacheEntryManager.cleanUp(); - cacheStats.resetMemoryUsage(); - } + if (secondKeyChecker.test(entry.getKey()) + && cacheEntryManager.invalidate(entry.getValue())) { + cacheEntryGroup.removeCacheEntry(entry.getKey()); + } + } - @Override - public void cleanUp() { - executeInvalidateAll(); - cacheStats.reset(); + if (cacheEntryGroup.isEmpty()) { + firstKeyMap.remove(firstKey); + } + } } - @Override - public IDualKeyCacheStats stats() { - return cacheStats; + private long getMemory() { + long memory = 0; + for (final Map> map : firstKeyMap.maps) { + if (Objects.nonNull(map)) { + for (final ICacheEntryGroup group : map.values()) { + memory += group.getMemory(); + } + } + } + return memory; } - @Override - @TestOnly - public void evictOneEntry() { - cacheStats.decreaseMemoryUsage(evictOneCacheEntry()); + private long getEntriesCount() { + long count = 0; + for (final Map> map : firstKeyMap.maps) { + if (Objects.nonNull(map)) { + for (final ICacheEntryGroup group : map.values()) { + count += group.getEntriesCount(); + } + } + } + return count; } /** @@ -412,29 +353,28 @@ public void evictOneEntry() { private static class SegmentedConcurrentHashMap { private static final int SLOT_NUM = 31; + private final Map[] maps = new ConcurrentHashMap[SLOT_NUM]; - V get(K key) { + V get(final K key) { return getBelongedMap(key).get(key); } - V remove(K key) { + V remove(final K key) { return getBelongedMap(key).remove(key); } - V compute(K key, BiFunction remappingFunction) { - return getBelongedMap(key).compute(key, remappingFunction); + V put(final K key, final V value) { + return getBelongedMap(key).put(key, value); } void clear() { synchronized (maps) { - for (int i = 0; i < SLOT_NUM; i++) { - maps[i] = null; - } + Arrays.fill(maps, null); } } - Map getBelongedMap(K key) { + Map getBelongedMap(final K key) { int slotIndex = key.hashCode() % SLOT_NUM; slotIndex = slotIndex < 0 ? slotIndex + SLOT_NUM : slotIndex; Map map = maps[slotIndex]; @@ -450,8 +390,9 @@ Map getBelongedMap(K key) { return map; } + // Copied list, deletion-safe List getAllKeys() { - List res = new ArrayList<>(); + final List res = new ArrayList<>(); Arrays.stream(maps) .iterator() .forEachRemaining( @@ -462,5 +403,15 @@ List getAllKeys() { }); return res; } + + int size() { + int size = 0; + for (final Map map : maps) { + if (Objects.nonNull(map)) { + size += map.size(); + } + } + return size; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/FIFOCacheEntryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/FIFOCacheEntryManager.java index 7d8a07acf4c86..c8dc64e01f363 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/FIFOCacheEntryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/FIFOCacheEntryManager.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; public class FIFOCacheEntryManager @@ -35,26 +36,33 @@ public class FIFOCacheEntryManager @Override public FIFOCacheEntry createCacheEntry( - SK secondKey, V value, ICacheEntryGroup> cacheEntryGroup) { + final SK secondKey, + final V value, + final ICacheEntryGroup> cacheEntryGroup) { return new FIFOCacheEntry<>(secondKey, value, cacheEntryGroup); } @Override - public void access(FIFOCacheEntry cacheEntry) { + public void access(final FIFOCacheEntry cacheEntry) { // do nothing } @Override - public void put(FIFOCacheEntry cacheEntry) { + public void put(final FIFOCacheEntry cacheEntry) { getNextList(cachePutRoundRobinIndex).add(cacheEntry); } @Override - public void invalid(FIFOCacheEntry cacheEntry) { + public boolean invalidate(final FIFOCacheEntry cacheEntry) { + if (cacheEntry.isInvalidated.getAndSet(true)) { + return false; + } + cacheEntry.next.pre = cacheEntry.pre; cacheEntry.pre.next = cacheEntry.next; cacheEntry.next = null; cacheEntry.pre = null; + return true; } @Override @@ -87,7 +95,7 @@ public void cleanUp() { } } - private FIFOLinkedList getNextList(AtomicInteger roundRobinIndex) { + private FIFOLinkedList getNextList(final AtomicInteger roundRobinIndex) { int listIndex = getNextIndex(roundRobinIndex); FIFOLinkedList fifoLinkedList = fifoLinkedLists[listIndex]; if (fifoLinkedList == null) { @@ -102,7 +110,7 @@ private FIFOLinkedList getNextList(AtomicInteger roundRobinIndex) { return fifoLinkedList; } - private int getNextIndex(AtomicInteger roundRobinIndex) { + private int getNextIndex(final AtomicInteger roundRobinIndex) { return roundRobinIndex.getAndUpdate( currentValue -> { currentValue = currentValue + 1; @@ -122,7 +130,10 @@ static class FIFOCacheEntry implements ICacheEntry { private FIFOCacheEntry pre = null; private FIFOCacheEntry next = null; - private FIFOCacheEntry(SK secondKey, V value, ICacheEntryGroup cacheEntryGroup) { + private final AtomicBoolean isInvalidated = new AtomicBoolean(false); + + private FIFOCacheEntry( + final SK secondKey, final V value, final ICacheEntryGroup cacheEntryGroup) { this.secondKey = secondKey; this.value = value; this.cacheEntryGroup = cacheEntryGroup; @@ -144,20 +155,24 @@ public ICacheEntryGroup getBelongedGroup() { } @Override - public void setBelongedGroup(ICacheEntryGroup belongedGroup) { + public void setBelongedGroup(final ICacheEntryGroup belongedGroup) { this.cacheEntryGroup = belongedGroup; } @Override - public void replaceValue(V newValue) { + public void replaceValue(final V newValue) { this.value = newValue; } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FIFOCacheEntry that = (FIFOCacheEntry) o; + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final FIFOCacheEntry that = (FIFOCacheEntry) o; return Objects.equals(secondKey, that.secondKey) && Objects.equals(cacheEntryGroup, that.cacheEntryGroup); } @@ -171,28 +186,40 @@ public int hashCode() { private static class FIFOLinkedList { // head.next is the newest - private final FIFOCacheEntry head; - private final FIFOCacheEntry tail; + private final FIFOCacheEntry head; + private final FIFOCacheEntry tail; public FIFOLinkedList() { - head = new FIFOCacheEntry(null, null, null); - tail = new FIFOCacheEntry(null, null, null); + head = new FIFOCacheEntry<>(null, null, null); + tail = new FIFOCacheEntry<>(null, null, null); head.next = tail; tail.pre = head; } - synchronized void add(FIFOCacheEntry cacheEntry) { - cacheEntry.next = head.next; + synchronized void add(final FIFOCacheEntry cacheEntry) { + FIFOCacheEntry nextEntry; + + do { + nextEntry = head.next; + } while (nextEntry.isInvalidated.get()); + + cacheEntry.next = nextEntry; cacheEntry.pre = head; - head.next.pre = cacheEntry; + nextEntry.pre = cacheEntry; head.next = cacheEntry; } - synchronized FIFOCacheEntry evict() { - if (tail.pre == head) { - return null; - } - FIFOCacheEntry cacheEntry = tail.pre; + synchronized FIFOCacheEntry evict() { + FIFOCacheEntry cacheEntry; + + do { + cacheEntry = tail.pre; + if (cacheEntry == head) { + return null; + } + + } while (cacheEntry.isInvalidated.compareAndSet(false, true)); + cacheEntry.pre.next = cacheEntry.next; cacheEntry.next.pre = cacheEntry.pre; cacheEntry.next = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryGroup.java index 89f562e31329e..d1f8ab9923e22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryGroup.java @@ -21,7 +21,9 @@ import java.util.Iterator; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; +import java.util.function.Function; /** * This interface defines the behaviour of a cache entry group, which is mainly accessed via first @@ -36,13 +38,19 @@ interface ICacheEntryGroup> { FK getFirstKey(); - T getCacheEntry(SK secondKey); + T getCacheEntry(final SK secondKey); Iterator> getAllCacheEntries(); - T computeCacheEntry(SK secondKey, BiFunction computation); + T computeCacheEntry( + final SK secondKey, final Function> computation); - T removeCacheEntry(SK secondKey); + long removeCacheEntry(final SK secondKey); boolean isEmpty(); + + // Metric + long getMemory(); + + int getEntriesCount(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java index a43e612b0e6d4..0117f836ba926 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/ICacheEntryManager.java @@ -30,14 +30,19 @@ */ interface ICacheEntryManager> { - T createCacheEntry(SK secondKey, V value, ICacheEntryGroup cacheEntryGroup); + T createCacheEntry( + final SK secondKey, final V value, final ICacheEntryGroup cacheEntryGroup); - void access(T cacheEntry); + void access(final T cacheEntry); - void put(T cacheEntry); + void put(final T cacheEntry); - void invalid(T cacheEntry); + // A cacheEntry is removed iff the caller has called "invalidate" and it returns "true" + // Shall never remove a cacheEntry directly or when the "invalidate" returns false + boolean invalidate(final T cacheEntry); + // The "evict" is allowed to be concurrently called + // Inner implementation guarantees that an entry won't be concurrently evicted T evict(); void cleanUp(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/LRUCacheEntryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/LRUCacheEntryManager.java index 726c4489ce2a3..52d2160d6f763 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/LRUCacheEntryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/LRUCacheEntryManager.java @@ -21,6 +21,7 @@ import java.util.Objects; import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; /** * This class implements the cache entry manager with LRU policy. @@ -40,32 +41,39 @@ class LRUCacheEntryManager @Override public LRUCacheEntry createCacheEntry( - SK secondKey, V value, ICacheEntryGroup> cacheEntryGroup) { + final SK secondKey, + final V value, + final ICacheEntryGroup> cacheEntryGroup) { return new LRUCacheEntry<>(secondKey, value, cacheEntryGroup); } @Override - public void access(LRUCacheEntry cacheEntry) { + public void access(final LRUCacheEntry cacheEntry) { getBelongedList(cacheEntry).moveToHead(cacheEntry); } @Override - public void put(LRUCacheEntry cacheEntry) { + public void put(final LRUCacheEntry cacheEntry) { getBelongedList(cacheEntry).add(cacheEntry); } @Override - public void invalid(LRUCacheEntry cacheEntry) { + public boolean invalidate(final LRUCacheEntry cacheEntry) { + if (cacheEntry.isInvalidated.getAndSet(true)) { + return false; + } + cacheEntry.next.pre = cacheEntry.pre; cacheEntry.pre.next = cacheEntry.next; cacheEntry.next = null; cacheEntry.pre = null; + return true; } @Override public LRUCacheEntry evict() { int startIndex = idxGenerator.nextInt(SLOT_NUM); - LRULinkedList lruLinkedList; + LRULinkedList lruLinkedList; LRUCacheEntry cacheEntry; for (int i = 0; i < SLOT_NUM; i++) { if (startIndex == SLOT_NUM) { @@ -119,6 +127,7 @@ static class LRUCacheEntry implements ICacheEntry { private LRUCacheEntry pre; private LRUCacheEntry next; + private final AtomicBoolean isInvalidated = new AtomicBoolean(false); private LRUCacheEntry(SK secondKey, V value, ICacheEntryGroup cacheEntryGroup) { this.secondKey = secondKey; @@ -166,31 +175,43 @@ public int hashCode() { } } - private static class LRULinkedList { + private static class LRULinkedList { // head.next is the most recently used entry - private final LRUCacheEntry head; - private final LRUCacheEntry tail; + private final LRUCacheEntry head; + private final LRUCacheEntry tail; public LRULinkedList() { - head = new LRUCacheEntry(null, null, null); - tail = new LRUCacheEntry(null, null, null); + head = new LRUCacheEntry<>(null, null, null); + tail = new LRUCacheEntry<>(null, null, null); head.next = tail; tail.pre = head; } - synchronized void add(LRUCacheEntry cacheEntry) { + synchronized void add(final LRUCacheEntry cacheEntry) { + LRUCacheEntry nextEntry; + + do { + nextEntry = head.next; + } while (nextEntry.isInvalidated.get()); + cacheEntry.next = head.next; cacheEntry.pre = head; head.next.pre = cacheEntry; head.next = cacheEntry; } - synchronized LRUCacheEntry evict() { - if (tail.pre == head) { - return null; - } - LRUCacheEntry cacheEntry = tail.pre; + synchronized LRUCacheEntry evict() { + LRUCacheEntry cacheEntry; + + do { + cacheEntry = tail.pre; + if (cacheEntry == head) { + return null; + } + + } while (cacheEntry.isInvalidated.compareAndSet(false, true)); + cacheEntry.pre.next = cacheEntry.next; cacheEntry.next.pre = cacheEntry.pre; cacheEntry.next = null; @@ -198,8 +219,8 @@ synchronized LRUCacheEntry evict() { return cacheEntry; } - synchronized void moveToHead(LRUCacheEntry cacheEntry) { - if (cacheEntry.next == null || cacheEntry.pre == null) { + synchronized void moveToHead(final LRUCacheEntry cacheEntry) { + if (cacheEntry.isInvalidated.get()) { // this cache entry has been evicted return; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/DataNodeLastCacheManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/DataNodeLastCacheManager.java deleted file mode 100644 index da31de2235546..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/DataNodeLastCacheManager.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache; - -import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.SchemaCacheEntry; - -import org.apache.tsfile.read.TimeValuePair; - -public class DataNodeLastCacheManager { - - private static final boolean CACHE_ENABLED = - CommonDescriptor.getInstance().getConfig().isLastCacheEnable(); - - /** - * get the last cache value from time series - * - * @param entry schema cache entry in DataNodeSchemaCache - * @return the last cache value - */ - public static TimeValuePair getLastCache(SchemaCacheEntry entry) { - if (!CACHE_ENABLED || null == entry) { - return null; - } - ILastCacheContainer lastCacheContainer = entry.getLastCacheContainer(); - return lastCacheContainer == null ? null : lastCacheContainer.getCachedLast(); - } - - /** - * update the last cache value of time series - * - * @param entry schema cache entry in DataNodeSchemaCache - * @param timeValuePair the latest point value - * @param highPriorityUpdate the last value from insertPlan is high priority - * @param latestFlushedTime latest flushed time - * @return increasing of memory usage - */ - public static int updateLastCache( - SchemaCacheEntry entry, - TimeValuePair timeValuePair, - boolean highPriorityUpdate, - Long latestFlushedTime) { - if (!CACHE_ENABLED || null == entry) { - return 0; - } - return entry.updateLastCache(timeValuePair, highPriorityUpdate, latestFlushedTime); - } - - public static int invalidateLastCache(SchemaCacheEntry entry) { - if (!CACHE_ENABLED || null == entry) { - return 0; - } - return entry.invalidateLastCache(); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java deleted file mode 100644 index edd0c202b9b06..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/ILastCacheContainer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache; - -import org.apache.tsfile.read.TimeValuePair; - -/** this interface declares the operations of LastCache data */ -public interface ILastCacheContainer { - - // get lastCache of monad timeseries - TimeValuePair getCachedLast(); - - /** - * update last point cache and enable last cache. - * - * @param timeValuePair last point - * @param highPriorityUpdate whether it's a high priority update - * @param latestFlushedTime latest flushed time - * @return increasing of memory usage - */ - int updateCachedLast( - TimeValuePair timeValuePair, boolean highPriorityUpdate, Long latestFlushedTime); - - /** Invalidate Last cache. */ - int invalidateLastCache(); - - int estimateSize(); -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java deleted file mode 100644 index 8730990333541..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/LastCacheContainer.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache; - -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.value.ILastCacheValue; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.value.LastCacheValue; - -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.TsPrimitiveType; - -/** - * This class possesses the ILastCacheValue and implements the basic last cache operations. - * - *

The ILastCacheValue may be extended to ILastCacheValue List in future to support batched last - * value cache. - */ -public class LastCacheContainer implements ILastCacheContainer { - - ILastCacheValue lastCacheValue; - - @Override - public TimeValuePair getCachedLast() { - return lastCacheValue == null ? null : lastCacheValue.getTimeValuePair(); - } - - @Override - public synchronized int updateCachedLast( - TimeValuePair timeValuePair, boolean highPriorityUpdate, Long latestFlushedTime) { - if (highPriorityUpdate) { // for write, we won't cache null value - if (timeValuePair == null || timeValuePair.getValue() == null) { - return 0; - } - } else { // for read, we need to cache null value, because it means that there is no data for - // this time series - if (timeValuePair == null) { - return 0; - } - } - - if (lastCacheValue == null) { - // If no cached last, (1) a last query (2) an unseq insertion or (3) a seq insertion will - // update cache. - if (!highPriorityUpdate || latestFlushedTime <= timeValuePair.getTimestamp()) { - lastCacheValue = new LastCacheValue(timeValuePair.getTimestamp(), timeValuePair.getValue()); - return lastCacheValue.estimateSize(); - } - } else if (timeValuePair.getTimestamp() > lastCacheValue.getTimestamp() - || (timeValuePair.getTimestamp() == lastCacheValue.getTimestamp() && highPriorityUpdate)) { - TsPrimitiveType oldValue = lastCacheValue.getValue(); - lastCacheValue.setTimestamp(timeValuePair.getTimestamp()); - lastCacheValue.setValue(timeValuePair.getValue()); - return getDiffSize(oldValue, timeValuePair.getValue()); - } - return 0; - } - - @Override - public int invalidateLastCache() { - int size = lastCacheValue.estimateSize(); - lastCacheValue = null; - return size; - } - - private int getDiffSize(TsPrimitiveType oldValue, TsPrimitiveType newValue) { - if (oldValue == null) { - return newValue == null ? 0 : newValue.getSize(); - } else if (oldValue instanceof TsPrimitiveType.TsBinary) { - return (newValue == null ? 0 : newValue.getSize()) - oldValue.getSize(); - } else { - return newValue == null ? -oldValue.getSize() : 0; - } - } - - /** - * Total basic 100B - * - *

    - *
  • LastCacheContainer Object header, 8B - *
  • ILastCacheValue reference, 8B - *
- */ - @Override - public int estimateSize() { - return 16 + (lastCacheValue == null ? 0 : lastCacheValue.estimateSize()); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java deleted file mode 100644 index c7b48da4c0958..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/lastcache/value/LastCacheValue.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.value; - -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.TsPrimitiveType; - -public class LastCacheValue implements ILastCacheValue { - - private long timestamp; - - private TsPrimitiveType value; - - public LastCacheValue(long timestamp, TsPrimitiveType value) { - this.timestamp = timestamp; - this.value = value; - } - - @Override - public long getTimestamp() { - return timestamp; - } - - @Override - public TsPrimitiveType getValue() { - return value; - } - - @Override - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - @Override - public void setValue(TsPrimitiveType value) { - this.value = value; - } - - @Override - public TimeValuePair getTimeValuePair() { - return new TimeValuePair(timestamp, value); - } - - /** - * Total basic 24B - * - *
    - *
  • LastCacheContainer Object header, 8B - *
  • long timestamp, 8B - *
  • TsPrimitiveType reference, 8B - *
- */ - @Override - public int estimateSize() { - return 24 + (value == null ? 0 : value.getSize()); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index a02393d390699..e621c2b445292 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -767,7 +767,7 @@ private void executeSetDatabaseStatement(Statement statement) if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode() && result.status.code != TSStatusCode.DATABASE_ALREADY_EXISTS - .getStatusCode() // In tree model, if the user creates a conflict database + .getStatusCode() // If the user creates a conflict database // concurrently, for instance, the // database created by user is root.db.ss.a, the auto-creation failed database is root.db, // we wait till "getOrCreatePartition" to judge if the time series (like root.db.ss.a.e / diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java index 13a9f5fc97a6d..c9914e0d7e9af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java @@ -119,7 +119,7 @@ public ClusterSchemaTree fetchSchema( Set storageGroupSet = new HashSet<>(); if (!explicitDevicePatternList.isEmpty()) { for (PartialPath explicitDevicePattern : explicitDevicePatternList) { - cachedSchema = schemaCache.getMatchedSchemaWithTemplate(explicitDevicePattern); + cachedSchema = schemaCache.getMatchedTemplateSchema(explicitDevicePattern); if (cachedSchema.isEmpty()) { isAllCached = false; break; @@ -132,7 +132,11 @@ public ClusterSchemaTree fetchSchema( if (isAllCached && !explicitPathList.isEmpty()) { for (PartialPath fullPath : explicitPathList) { - cachedSchema = schemaCache.getMatchedSchemaWithoutTemplate(fullPath); + // no path length <= 2 + if (fullPath.getNodeLength() <= 2) { + continue; + } + cachedSchema = schemaCache.getMatchedNormalSchema(fullPath); if (cachedSchema.isEmpty()) { isAllCached = false; break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 2e6a6be8d1adf..2f1b9671e31f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -2774,6 +2774,16 @@ private UpdateLastCacheOperator createUpdateLastCacheOperator( node.getPlanNodeId(), UpdateLastCacheOperator.class.getSimpleName()); + final boolean isNeedUpdateLastCache = context.isNeedUpdateLastCache(); + if (isNeedUpdateLastCache) { + DataNodeSchemaCache.getInstance() + .declareLastCache( + ((DataDriverContext) operatorContext.getDriverContext()) + .getDataRegion() + .getDatabaseName(), + fullPath); + } + return Objects.isNull(node.getOutputViewPath()) ? new UpdateLastCacheOperator( operatorContext, @@ -2811,6 +2821,26 @@ private AlignedUpdateLastCacheOperator createAlignedUpdateLastCacheOperator( planNodeId, AlignedUpdateLastCacheOperator.class.getSimpleName()); + final boolean isNeedUpdateLastCache = context.isNeedUpdateLastCache(); + if (isNeedUpdateLastCache) { + final int size = unCachedPath.getColumnNum(); + final PartialPath devicePath = unCachedPath.getDevicePath(); + final String databaseName = + ((DataDriverContext) operatorContext.getDriverContext()) + .getDataRegion() + .getDatabaseName(); + + for (int i = 0; i < size; ++i) { + DataNodeSchemaCache.getInstance() + .declareLastCache( + databaseName, + new MeasurementPath( + devicePath.concatNode(unCachedPath.getMeasurementList().get(i)), + unCachedPath.getSchemaList().get(i), + true)); + } + } + return Objects.isNull(outputViewPath) ? new AlignedUpdateLastCacheOperator( operatorContext, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index f013dd6755695..c664bbb9ddfaa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -295,6 +295,21 @@ public boolean allMeasurementFailed() { return true; } + public String[] getRawMeasurements() { + String[] measurements = getMeasurements(); + MeasurementSchema[] measurementSchemas = getMeasurementSchemas(); + String[] rawMeasurements = new String[measurements.length]; + for (int i = 0; i < measurements.length; i++) { + if (measurementSchemas[i] != null) { + // get raw measurement rather than alias + rawMeasurements[i] = measurementSchemas[i].getMeasurementId(); + } else { + rawMeasurements[i] = measurements[i]; + } + } + return rawMeasurements; + } + // endregion // region progress index diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index 29a163b2ec504..6518385c48a86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -854,10 +855,28 @@ public R accept(PlanVisitor visitor, C context) { } public TimeValuePair composeTimeValuePair(int columnIndex) { - if (columnIndex >= values.length) { + if (columnIndex >= values.length || Objects.isNull(dataTypes[columnIndex])) { return null; } Object value = values[columnIndex]; - return new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)); + return Objects.nonNull(value) + ? new TimeValuePair(time, TsPrimitiveType.getByType(dataTypes[columnIndex], value)) + : null; + } + + public void updateLastCache(String databaseName) { + String[] rawMeasurements = getRawMeasurements(); + TimeValuePair[] timeValuePairs = new TimeValuePair[rawMeasurements.length]; + for (int i = 0; i < rawMeasurements.length; i++) { + timeValuePairs[i] = composeTimeValuePair(i); + } + DataNodeSchemaCache.getInstance() + .updateLastCacheIfExists( + databaseName, + devicePath, + rawMeasurements, + timeValuePairs, + isAligned, + measurementSchemas); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index 790d2b7987b03..eb5015a93f44f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -1131,4 +1132,20 @@ public TimeValuePair composeLastTimeValuePair(int measurementIndex) { } return new TimeValuePair(times[lastIdx], value); } + + public void updateLastCache(final String databaseName) { + final String[] rawMeasurements = getRawMeasurements(); + final TimeValuePair[] timeValuePairs = new TimeValuePair[rawMeasurements.length]; + for (int i = 0; i < rawMeasurements.length; i++) { + timeValuePairs[i] = composeLastTimeValuePair(i); + } + DataNodeSchemaCache.getInstance() + .updateLastCacheIfExists( + databaseName, + devicePath, + rawMeasurements, + timeValuePairs, + isAligned, + measurementSchemas); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 6b050438062b4..12630a147c0d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -314,7 +314,8 @@ long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) throws MetadataException; int fillLastQueryMap( - final PartialPath pattern, final Map> mapToFill) + final PartialPath pattern, + final Map>> mapToFill) throws MetadataException; // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 95feb33e52a91..06a00c64e0330 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1325,7 +1325,8 @@ public long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) @Override public int fillLastQueryMap( - final PartialPath pattern, final Map> mapToFill) + final PartialPath pattern, + final Map>> mapToFill) throws MetadataException { return mtree.fillLastQueryMap(pattern, mapToFill); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 6ae88e4946823..7c05e864b81f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1430,7 +1430,8 @@ public long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) @Override public int fillLastQueryMap( - PartialPath pattern, Map> mapToFill) { + final PartialPath pattern, + final Map>> mapToFill) { throw new UnsupportedOperationException("Not implemented"); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 9f6316583d58a..60e7f99f74f4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.exception.metadata.template.TemplateIsInUseException; import org.apache.iotdb.db.exception.quota.ExceedQuotaException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DeviceSchemaCache; import org.apache.iotdb.db.schemaengine.metric.SchemaRegionMemMetric; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; @@ -1076,7 +1077,8 @@ public IDeviceSchemaInfo next() { } public int fillLastQueryMap( - final PartialPath prefixPath, final Map> mapToFill) + final PartialPath prefixPath, + final Map>> mapToFill) throws MetadataException { final int[] sensorNum = {0}; try (final EntityUpdater updater = @@ -1091,7 +1093,10 @@ protected void updateEntity(final IDeviceMNode node) { measurementMap.put(child.getName(), null); } } - mapToFill.put(node.getPartialPath(), measurementMap); + final PartialPath path = node.getPartialPath(); + mapToFill + .computeIfAbsent(DeviceSchemaCache.getLeadingSegment(path), o -> new HashMap<>()) + .put(path, measurementMap); sensorNum[0] += measurementMap.size(); } }) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index c1518fe5dadbc..a16bb396b5c88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -138,7 +138,6 @@ import org.apache.tsfile.read.reader.TsFileLastReader; import org.apache.tsfile.utils.FSUtils; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.writer.RestorableTsFileIOWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1282,29 +1281,7 @@ private boolean insertTabletToTsFileProcessor( } private void tryToUpdateInsertTabletLastCache(InsertTabletNode node) { - long latestFlushedTime = lastFlushTimeMap.getGlobalFlushedTime(node.getDeviceID()); - String[] measurements = node.getMeasurements(); - MeasurementSchema[] measurementSchemas = node.getMeasurementSchemas(); - String[] rawMeasurements = new String[measurements.length]; - for (int i = 0; i < measurements.length; i++) { - if (measurementSchemas[i] != null) { - // get raw measurement rather than alias - rawMeasurements[i] = measurementSchemas[i].getMeasurementId(); - } else { - rawMeasurements[i] = measurements[i]; - } - } - DataNodeSchemaCache.getInstance() - .updateLastCache( - getDatabaseName(), - node.getDevicePath(), - rawMeasurements, - node.getMeasurementSchemas(), - node.isAligned(), - node::composeLastTimeValuePair, - index -> node.getColumns()[index] != null, - true, - latestFlushedTime); + node.updateLastCache(getDatabaseName()); } private TsFileProcessor insertToTsFileProcessor( @@ -1324,29 +1301,7 @@ private TsFileProcessor insertToTsFileProcessor( } private void tryToUpdateInsertRowLastCache(InsertRowNode node) { - long latestFlushedTime = lastFlushTimeMap.getGlobalFlushedTime(node.getDeviceID()); - String[] measurements = node.getMeasurements(); - MeasurementSchema[] measurementSchemas = node.getMeasurementSchemas(); - String[] rawMeasurements = new String[measurements.length]; - for (int i = 0; i < measurements.length; i++) { - if (measurementSchemas[i] != null) { - // get raw measurement rather than alias - rawMeasurements[i] = measurementSchemas[i].getMeasurementId(); - } else { - rawMeasurements[i] = measurements[i]; - } - } - DataNodeSchemaCache.getInstance() - .updateLastCache( - getDatabaseName(), - node.getDevicePath(), - rawMeasurements, - node.getMeasurementSchemas(), - node.isAligned(), - node::composeTimeValuePair, - index -> node.getValues()[index] != null, - true, - latestFlushedTime); + node.updateLastCache(databaseName); } private List insertToTsFileProcessors( @@ -1416,35 +1371,8 @@ private List insertToTsFileProcessors( } private void tryToUpdateInsertRowsLastCache(List nodeList) { - DataNodeSchemaCache.getInstance().takeReadLock(); - try { - for (InsertRowNode node : nodeList) { - long latestFlushedTime = lastFlushTimeMap.getGlobalFlushedTime(node.getDeviceID()); - String[] measurements = node.getMeasurements(); - MeasurementSchema[] measurementSchemas = node.getMeasurementSchemas(); - String[] rawMeasurements = new String[measurements.length]; - for (int i = 0; i < measurements.length; i++) { - if (measurementSchemas[i] != null) { - // get raw measurement rather than alias - rawMeasurements[i] = measurementSchemas[i].getMeasurementId(); - } else { - rawMeasurements[i] = measurements[i]; - } - } - DataNodeSchemaCache.getInstance() - .updateLastCacheWithoutLock( - getDatabaseName(), - node.getDevicePath(), - rawMeasurements, - node.getMeasurementSchemas(), - node.isAligned(), - node::composeTimeValuePair, - index -> node.getValues()[index] != null, - true, - latestFlushedTime); - } - } finally { - DataNodeSchemaCache.getInstance().releaseReadLock(); + for (InsertRowNode node : nodeList) { + node.updateLastCache(databaseName); } } @@ -2487,7 +2415,7 @@ public void deleteDataDirectly( if (deleted) { return; } - DataNodeSchemaCache.getInstance().invalidateLastCacheInDataRegion(getDatabaseName()); + DataNodeSchemaCache.getInstance().invalidateDatabaseLastCache(getDatabaseName()); // write log to impacted working TsFileProcessors List walListeners = logDeletionInWAL(startTime, endTime, searchIndex, pathToDelete); @@ -3255,7 +3183,7 @@ private void onTsFileLoaded( // The inner cache is shared by TreeDeviceSchemaCacheManager and // TableDeviceSchemaCacheManager, // so cleaning either of them is enough - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); break; case CLEAN_DEVICE: ITimeIndex timeIndex = newTsFileResource.getTimeIndex(); @@ -3264,23 +3192,24 @@ private void onTsFileLoaded( for (IDeviceID deviceID : deviceTimeIndex.getDevices()) { try { DataNodeSchemaCache.getInstance() - .invalidateLastCache(new PartialPath(deviceID, "**")); + .getDeviceSchemaCache() + .invalidateDeviceLastCache(new PartialPath(deviceID)); } catch (IllegalPathException e) { logger.error( "Failed to construct path for invalidating last cache of {}", deviceID, e); - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); return; } } } else { - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); } break; default: logger.warn( "Unrecognized LastCacheLoadStrategy: {}, fall back to CLEAN_ALL", IoTDBDescriptor.getInstance().getConfig().getLastCacheLoadStrategy()); - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); break; } } @@ -3300,19 +3229,16 @@ private void updateLastCache(TsFileResource newTsFileResource, TsFileLastReader try { // we do not update schema here, so aligned is not relevant DataNodeSchemaCache.getInstance() - .updateLastCache( + .updateLastCacheIfExists( databaseName, new PartialPath(deviceID), measurements, - null, + timeValuePairs, false, - value -> timeValuePairs[value], - i -> true, - true, - 0L); + null); } catch (IllegalPathException e) { logger.error("Failed to construct path for invalidating last cache of {}", deviceID, e); - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); return; } } @@ -3330,24 +3256,21 @@ private void updateLastCache(TsFileResource newTsFileResource, TsFileLastReader try { // we do not update schema here, so aligned is not relevant DataNodeSchemaCache.getInstance() - .updateLastCache( + .updateLastCacheIfExists( databaseName, new PartialPath(deviceID), measurements, - null, + timeValuePairs, false, - value -> timeValuePairs[value], - i -> true, - true, - 0L); + null); } catch (IllegalPathException e) { logger.error("Failed to construct path for invalidating last cache of {}", deviceID, e); - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); return; } } } else { - DataNodeSchemaCache.getInstance().invalidateAll(); + DataNodeSchemaCache.getInstance().getDeviceSchemaCache().invalidateAll(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java index 81a1f0639beba..883de7958dca1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java @@ -68,7 +68,7 @@ public Optional convertForTreeModel(final LoadTsFileStatement loadTsFi return loadTsFileTreeStatement.accept(treeStatementDataTypeConvertExecutionVisitor, null); } catch (Exception e) { LOGGER.warn( - "Failed to convert data types for tree model statement {}.", loadTsFileTreeStatement, e); + "Failed to convert data types for load statement {}.", loadTsFileTreeStatement, e); return Optional.of( new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage())); } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 4971775fefa34..117cb3a89615e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -79,7 +79,7 @@ public abstract class AlignedTVList extends TVList { // Index relation: columnIndex(dataTypeIndex) -> arrayIndex -> elementIndex protected List> bitMaps; - // not null when constructed by queries for tree model + // not null when constructed by queries BitMap allValueColDeletedMap; private final AlignedTVList outer = this; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java index fc1e2f6ff6943..d2f11d65d1fea 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.metadata.cache; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; @@ -50,11 +51,16 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; public class DataNodeSchemaCacheTest { - DataNodeSchemaCache dataNodeSchemaCache; + private DataNodeSchemaCache dataNodeSchemaCache; private Map s1TagMap; + private boolean lastCacheEnabledDefaultValue; @Before public void setUp() throws Exception { + lastCacheEnabledDefaultValue = CommonDescriptor.getInstance().getConfig().isLastCacheEnable(); + if (!lastCacheEnabledDefaultValue) { + CommonDescriptor.getInstance().getConfig().setLastCacheEnable(true); + } dataNodeSchemaCache = DataNodeSchemaCache.getInstance(); s1TagMap = new HashMap<>(); s1TagMap.put("k1", "v1"); @@ -62,14 +68,15 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { + CommonDescriptor.getInstance().getConfig().setLastCacheEnable(lastCacheEnabledDefaultValue); dataNodeSchemaCache.cleanUp(); ClusterTemplateManager.getInstance().clear(); } @Test public void testGetSchemaEntity() throws IllegalPathException { - PartialPath device1 = new PartialPath("root.sg1.d1"); - String[] measurements = new String[3]; + final PartialPath device1 = new PartialPath("root.sg1.d1"); + final String[] measurements = new String[3]; measurements[0] = "s1"; measurements[1] = "s2"; measurements[2] = "s3"; @@ -82,12 +89,7 @@ public void testGetSchemaEntity() throws IllegalPathException { .collect( Collectors.toMap( o -> new PartialPath(o.getNodes()), - o -> - new SchemaCacheEntry( - "root.sg1", - o.getMeasurementSchema(), - o.getTagMap(), - o.isUnderAlignedEntity()))); + o -> new SchemaCacheEntry(o.getMeasurementSchema(), o.getTagMap()))); Assert.assertEquals( TSDataType.INT32, schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s1")).getTsDataType()); @@ -102,7 +104,7 @@ public void testGetSchemaEntity() throws IllegalPathException { schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s3")).getTsDataType()); Assert.assertNull(schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s3")).getTagMap()); - String[] otherMeasurements = new String[3]; + final String[] otherMeasurements = new String[3]; otherMeasurements[0] = "s3"; otherMeasurements[1] = "s4"; otherMeasurements[2] = "s5"; @@ -115,12 +117,7 @@ public void testGetSchemaEntity() throws IllegalPathException { .collect( Collectors.toMap( o -> new PartialPath(o.getNodes()), - o -> - new SchemaCacheEntry( - "root.sg1", - o.getMeasurementSchema(), - o.getTagMap(), - o.isUnderAlignedEntity()))); + o -> new SchemaCacheEntry(o.getMeasurementSchema(), o.getTagMap()))); Assert.assertEquals( TSDataType.BOOLEAN, schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s3")).getTsDataType()); @@ -135,71 +132,9 @@ public void testGetSchemaEntity() throws IllegalPathException { Assert.assertNull(schemaCacheEntryMap.get(new PartialPath("root.sg1.d1.s4")).getTagMap()); } - @Test - public void testLastCache() throws IllegalPathException { - // test no cache - PartialPath devicePath = new PartialPath("root.sg1.d1"); - PartialPath seriesPath1 = new PartialPath("root.sg1.d1.s1"); - PartialPath seriesPath2 = new PartialPath("root.sg1.d1.s2"); - PartialPath seriesPath3 = new PartialPath("root.sg1.d1.s3"); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath1)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - // test no last cache - dataNodeSchemaCache.put((ClusterSchemaTree) generateSchemaTree1()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath1)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - // put cache - long timestamp = 100; - long timestamp2 = 101; - TsPrimitiveType value = TsPrimitiveType.getByType(TSDataType.INT32, 101); - TsPrimitiveType value2 = TsPrimitiveType.getByType(TSDataType.INT32, 100); - TsPrimitiveType value3 = TsPrimitiveType.getByType(TSDataType.INT32, 99); - - // put into last cache when cache not exist - TimeValuePair timeValuePair = new TimeValuePair(timestamp, value); - dataNodeSchemaCache.updateLastCache(devicePath, "s1", timeValuePair, false, 99L); - TimeValuePair cachedTimeValuePair = dataNodeSchemaCache.getLastCache(seriesPath1); - Assert.assertNotNull(cachedTimeValuePair); - Assert.assertEquals(timestamp, cachedTimeValuePair.getTimestamp()); - Assert.assertEquals(value, cachedTimeValuePair.getValue()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - - // same time but low priority - TimeValuePair timeValuePair2 = new TimeValuePair(timestamp, value2); - dataNodeSchemaCache.updateLastCache(devicePath, "s1", timeValuePair2, false, 100L); - TimeValuePair cachedTimeValuePair2 = dataNodeSchemaCache.getLastCache(seriesPath1); - Assert.assertNotNull(cachedTimeValuePair2); - Assert.assertEquals(timestamp, cachedTimeValuePair2.getTimestamp()); - Assert.assertEquals(value, cachedTimeValuePair2.getValue()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - - // same time but high priority - dataNodeSchemaCache.updateLastCache(devicePath, "s1", timeValuePair2, true, 100L); - cachedTimeValuePair2 = dataNodeSchemaCache.getLastCache(seriesPath1); - Assert.assertNotNull(cachedTimeValuePair2); - Assert.assertEquals(timestamp, cachedTimeValuePair2.getTimestamp()); - Assert.assertEquals(value2, cachedTimeValuePair2.getValue()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - - // put into last cache when cache already exist - TimeValuePair timeValuePair3 = new TimeValuePair(timestamp2, value3); - dataNodeSchemaCache.updateLastCache(devicePath, "s1", timeValuePair3, false, 100L); - TimeValuePair cachedTimeValuePair3 = dataNodeSchemaCache.getLastCache(seriesPath1); - Assert.assertNotNull(cachedTimeValuePair3); - Assert.assertEquals(timestamp2, cachedTimeValuePair3.getTimestamp()); - Assert.assertEquals(value3, cachedTimeValuePair3.getValue()); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath2)); - Assert.assertNull(dataNodeSchemaCache.getLastCache(seriesPath3)); - } - private ISchemaTree generateSchemaTree1() throws IllegalPathException { - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); - Map s1TagMap = new HashMap<>(); + final ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + final Map s1TagMap = new HashMap<>(); s1TagMap.put("k1", "v1"); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s1"), @@ -227,7 +162,7 @@ private ISchemaTree generateSchemaTree1() throws IllegalPathException { } private ISchemaTree generateSchemaTree2() throws IllegalPathException { - ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + final ClusterSchemaTree schemaTree = new ClusterSchemaTree(); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s3"), @@ -256,58 +191,81 @@ private ISchemaTree generateSchemaTree2() throws IllegalPathException { @Test public void testUpdateLastCache() throws IllegalPathException { - String database = "root.db"; - PartialPath device = new PartialPath("root.db.d"); + final String database = "root.db"; + final PartialPath device = new PartialPath("root.db.d"); + + final String[] measurements = new String[] {"s1", "s2", "s3"}; + + final MeasurementSchema s1 = new MeasurementSchema("s1", TSDataType.INT32); + final MeasurementSchema s2 = new MeasurementSchema("s2", TSDataType.INT32); + final MeasurementSchema s3 = new MeasurementSchema("s3", TSDataType.INT32); + + final TimeValuePair tv1 = new TimeValuePair(1, new TsPrimitiveType.TsInt(1)); - String[] measurements = new String[] {"s1", "s2", "s3"}; - MeasurementSchema[] measurementSchemas = - new MeasurementSchema[] { - new MeasurementSchema("s1", TSDataType.INT32), - new MeasurementSchema("s2", TSDataType.INT32), - new MeasurementSchema("s3", TSDataType.INT32) - }; + dataNodeSchemaCache.declareLastCache( + database, new MeasurementPath(device.concatNode("s1"), s1)); + dataNodeSchemaCache.declareLastCache( + database, new MeasurementPath(device.concatNode("s3"), s3)); - dataNodeSchemaCache.updateLastCache( + // Simulate "s1" revert when the query has failed in calculation + dataNodeSchemaCache.updateLastCacheIfExists( database, device, - measurements, - measurementSchemas, - true, - index -> new TimeValuePair(1, new TsPrimitiveType.TsInt(1)), - index -> index != 1, - true, - 1L); + new String[] {"s1"}, + new TimeValuePair[] { + new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), + }, + false, + new MeasurementSchema[] {s1}); + dataNodeSchemaCache.invalidateLastCache( + database, new MeasurementPath(device.concatNode("s1"), s1)); + + // "s2" shall be null since the "null" timeValuePair has not been put + dataNodeSchemaCache.updateLastCacheIfExists( + database, + device, + new String[] {"s2"}, + new TimeValuePair[] {tv1}, + false, + new MeasurementSchema[] {s2}); + + dataNodeSchemaCache.updateLastCacheIfExists( + database, + device, + new String[] {"s3"}, + new TimeValuePair[] {tv1}, + false, + new MeasurementSchema[] {s3}); + + Assert.assertNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s1"))); + Assert.assertNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s2"))); + Assert.assertNotNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s3"))); - Assert.assertNotNull(dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s1"))); - Assert.assertNull(dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s2"))); - Assert.assertNotNull(dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s3"))); + final MeasurementSchema[] measurementSchemas = new MeasurementSchema[] {s1, s2, s3}; - dataNodeSchemaCache.updateLastCache( + dataNodeSchemaCache.updateLastCacheIfExists( database, device, measurements, - measurementSchemas, - true, - index -> new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), - index -> true, - true, - 1L); + new TimeValuePair[] { + new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), + new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), + new TimeValuePair(2, new TsPrimitiveType.TsInt(2)) + }, + false, + measurementSchemas); + Assert.assertNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s1"))); + Assert.assertNull(dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s2"))); Assert.assertEquals( new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), - dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s1"))); - Assert.assertEquals( - new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), - dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s2"))); - Assert.assertEquals( - new TimeValuePair(2, new TsPrimitiveType.TsInt(2)), - dataNodeSchemaCache.getLastCache(new PartialPath("root.db.d.s3"))); + dataNodeSchemaCache.getLastCache(new MeasurementPath("root.db.d.s3"))); } @Test public void testPut() throws Exception { - ClusterSchemaTree clusterSchemaTree = new ClusterSchemaTree(); - Template template1 = + final ClusterSchemaTree clusterSchemaTree = new ClusterSchemaTree(); + final Template template1 = new Template( "t1", Arrays.asList("s1", "s2"), @@ -315,7 +273,7 @@ public void testPut() throws Exception { Arrays.asList(TSEncoding.RLE, TSEncoding.RLE), Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY)); template1.setId(1); - Template template2 = + final Template template2 = new Template( "t2", Arrays.asList("t1", "t2", "t3"), @@ -331,22 +289,22 @@ public void testPut() throws Exception { clusterSchemaTree.appendSingleMeasurementPath( new MeasurementPath("root.sg1.d3.s1", TSDataType.FLOAT)); dataNodeSchemaCache.put(clusterSchemaTree); - ClusterSchemaTree d1Tree = - dataNodeSchemaCache.getMatchedSchemaWithTemplate(new PartialPath("root.sg1.d1")); - ClusterSchemaTree d2Tree = - dataNodeSchemaCache.getMatchedSchemaWithTemplate(new PartialPath("root.sg1.d2")); - ClusterSchemaTree d3Tree = - dataNodeSchemaCache.getMatchedSchemaWithoutTemplate(new PartialPath("root.sg1.d3.s1")); + final ClusterSchemaTree d1Tree = + dataNodeSchemaCache.getMatchedTemplateSchema(new PartialPath("root.sg1.d1")); + final ClusterSchemaTree d2Tree = + dataNodeSchemaCache.getMatchedTemplateSchema(new PartialPath("root.sg1.d2")); + final ClusterSchemaTree d3Tree = + dataNodeSchemaCache.getMatchedNormalSchema(new MeasurementPath("root.sg1.d3.s1")); List measurementPaths = d1Tree.searchMeasurementPaths(ALL_MATCH_PATTERN).left; Assert.assertEquals(2, measurementPaths.size()); - for (MeasurementPath measurementPath : measurementPaths) { + for (final MeasurementPath measurementPath : measurementPaths) { Assert.assertEquals( template1.getSchema(measurementPath.getMeasurement()), measurementPath.getMeasurementSchema()); } measurementPaths = d2Tree.searchMeasurementPaths(ALL_MATCH_PATTERN).left; Assert.assertEquals(3, measurementPaths.size()); - for (MeasurementPath measurementPath : measurementPaths) { + for (final MeasurementPath measurementPath : measurementPaths) { Assert.assertEquals( template2.getSchema(measurementPath.getMeasurement()), measurementPath.getMeasurementSchema()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/dualkeycache/DualKeyCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/dualkeycache/DualKeyCacheTest.java deleted file mode 100644 index 17ec1bcfb0ccf..0000000000000 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/dualkeycache/DualKeyCacheTest.java +++ /dev/null @@ -1,491 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.metadata.cache.dualkeycache; - -import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.SchemaCacheEntry; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheComputation; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCacheUpdating; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.lastcache.DataNodeLastCacheManager; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.TimeValuePair; -import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -@RunWith(Parameterized.class) -public class DualKeyCacheTest { - - private final String policy; - - public DualKeyCacheTest(String policy) { - this.policy = policy; - } - - @Parameterized.Parameters - public static List getTestModes() { - return Arrays.asList("FIFO", "LRU"); - } - - @Test - public void testBasicReadPut() { - DualKeyCacheBuilder dualKeyCacheBuilder = new DualKeyCacheBuilder<>(); - IDualKeyCache dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy(DualKeyCachePolicy.valueOf(policy)) - .memoryCapacity(300) - .firstKeySizeComputer(this::computeStringSize) - .secondKeySizeComputer(this::computeStringSize) - .valueSizeComputer(this::computeStringSize) - .build(); - - String[] firstKeyList = new String[] {"root.db.d1", "root.db.d2"}; - String[] secondKeyList = new String[] {"s1", "s2"}; - String[][] valueTable = - new String[][] {new String[] {"1-1", "1-2"}, new String[] {"2-1", "2-2"}}; - - for (int i = 0; i < firstKeyList.length; i++) { - for (int j = 0; j < secondKeyList.length; j++) { - dualKeyCache.put(firstKeyList[i], secondKeyList[j], valueTable[i][j]); - } - } - - int firstKeyOfMissingEntry = -1, secondKeyOfMissingEntry = -1; - for (int i = 0; i < firstKeyList.length; i++) { - for (int j = 0; j < secondKeyList.length; j++) { - String value = dualKeyCache.get(firstKeyList[i], secondKeyList[j]); - if (value == null) { - if (firstKeyOfMissingEntry == -1) { - firstKeyOfMissingEntry = i; - secondKeyOfMissingEntry = j; - } else { - Assert.fail(); - } - } else { - Assert.assertEquals(valueTable[i][j], value); - } - } - } - - Assert.assertEquals(230, dualKeyCache.stats().memoryUsage()); - Assert.assertEquals(4, dualKeyCache.stats().requestCount()); - Assert.assertEquals(3, dualKeyCache.stats().hitCount()); - - dualKeyCache.put( - firstKeyList[firstKeyOfMissingEntry], - secondKeyList[secondKeyOfMissingEntry], - valueTable[firstKeyOfMissingEntry][secondKeyOfMissingEntry]); - Assert.assertEquals(230, dualKeyCache.stats().memoryUsage()); - - for (int i = 0; i < firstKeyList.length; i++) { - int finalI = i; - dualKeyCache.compute( - new IDualKeyCacheComputation() { - @Override - public String getFirstKey() { - return firstKeyList[finalI]; - } - - @Override - public String[] getSecondKeyList() { - return secondKeyList; - } - - @Override - public void computeValue(int index, String value) { - if (value != null) { - Assert.assertEquals(valueTable[finalI][index], value); - } - } - }); - } - - Assert.assertEquals(8, dualKeyCache.stats().requestCount()); - Assert.assertEquals(6, dualKeyCache.stats().hitCount()); - } - - private int computeStringSize(String string) { - return 8 + 8 + 4 + 2 * string.length(); - } - - @Test - public void testComputeAndUpdateSize() { - DualKeyCacheBuilder dualKeyCacheBuilder = - new DualKeyCacheBuilder<>(); - IDualKeyCache dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy(DualKeyCachePolicy.valueOf(policy)) - .memoryCapacity(500) - .firstKeySizeComputer(this::computeStringSize) - .secondKeySizeComputer(this::computeStringSize) - .valueSizeComputer(SchemaCacheEntry::estimateSize) - .build(); - String firstKey = "root.db.d1"; - String[] secondKeyList = new String[] {"s1", "s2"}; - for (String s : secondKeyList) { - dualKeyCache.put( - firstKey, - s, - new SchemaCacheEntry( - "root.db", - new MeasurementSchema(s, TSDataType.INT32), - Collections.emptyMap(), - false)); - } - SchemaCacheEntry schemaCacheEntry = - new SchemaCacheEntry( - "root.db", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false); - int expectedSize = - computeStringSize("root.db.d1") - + computeStringSize("s1") * 2 - + SchemaCacheEntry.estimateSize(schemaCacheEntry) * 2; - Assert.assertEquals(expectedSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.update( - new IDualKeyCacheUpdating() { - @Override - public String getFirstKey() { - return firstKey; - } - - @Override - public String[] getSecondKeyList() { - return secondKeyList; - } - - @Override - public int updateValue(int index, SchemaCacheEntry value) { - return DataNodeLastCacheManager.updateLastCache( - value, new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - } - }); - int tmp = SchemaCacheEntry.estimateSize(schemaCacheEntry); - schemaCacheEntry.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - expectedSize += (SchemaCacheEntry.estimateSize(schemaCacheEntry) - tmp) * 2; - Assert.assertEquals(expectedSize, dualKeyCache.stats().memoryUsage()); - } - - @Test - public void testInvalidDatabase() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateCache(); - dualKeyCache.invalidate("root.db1"); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1"), "s11")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db2.d1")) - + computeStringSize("s1") * 2 - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)) - * 2; - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.evictOneEntry(); - expectSize = - PartialPath.estimateSize(new PartialPath("root.db2.d1")) - + computeStringSize("s1") - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.evictOneEntry(); - Assert.assertEquals(0, dualKeyCache.stats().memoryUsage()); - } - - @Test - public void testInvalidPathPattern1() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateCache(); - dualKeyCache.invalidate( - Arrays.asList(new PartialPath("root.db2.**"), new PartialPath("root.db1.d1.s1"))); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1"), "s11")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db1.d1")) - + PartialPath.estimateSize(new PartialPath("root.db1")) - + computeStringSize("s1") - + computeStringSize("s11") - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)) - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s11", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.evictOneEntry(); - dualKeyCache.evictOneEntry(); - Assert.assertEquals(0, dualKeyCache.stats().memoryUsage()); - } - - @Test - public void testInvalidPathPattern2() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateCache(); - dualKeyCache.invalidate( - Arrays.asList(new PartialPath("root.db1.**"), new PartialPath("root.db2.d1.*1"))); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db1"), "s11")); - Assert.assertNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db2.d1")) - + computeStringSize("s2") - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - dualKeyCache.evictOneEntry(); - Assert.assertEquals(0, dualKeyCache.stats().memoryUsage()); - } - - private IDualKeyCache generateCache() - throws IllegalPathException { - DualKeyCacheBuilder dualKeyCacheBuilder = - new DualKeyCacheBuilder<>(); - IDualKeyCache dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy(DualKeyCachePolicy.valueOf(policy)) - .memoryCapacity(2000) // actual threshold is 1600 - .firstKeySizeComputer(PartialPath::estimateSize) - .secondKeySizeComputer(this::computeStringSize) - .valueSizeComputer(SchemaCacheEntry::estimateSize) - .build(); - dualKeyCache.put( - new PartialPath("root.db1.d1"), - "s1", - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db1.d1"), - "s2", - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db1"), - "s11", - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s11", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db1.d1"), - "s2", - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db2.d1"), - "s1", - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - dualKeyCache.put( - new PartialPath("root.db2.d1"), - "s2", - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1"), "s11")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db1.d1")) * 2 - + PartialPath.estimateSize(new PartialPath("root.db1")) - + computeStringSize("s1") * 4 - + computeStringSize("s11") - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false)) - * 4 - + SchemaCacheEntry.estimateSize( - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s11", TSDataType.INT32), - Collections.emptyMap(), - false)); - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - return dualKeyCache; - } - - private IDualKeyCache generateLastCache() - throws IllegalPathException { - DualKeyCacheBuilder dualKeyCacheBuilder = - new DualKeyCacheBuilder<>(); - IDualKeyCache dualKeyCache = - dualKeyCacheBuilder - .cacheEvictionPolicy(DualKeyCachePolicy.valueOf(policy)) - .memoryCapacity(2000) // actual threshold is 1600 - .firstKeySizeComputer(PartialPath::estimateSize) - .secondKeySizeComputer(this::computeStringSize) - .valueSizeComputer(SchemaCacheEntry::estimateSize) - .build(); - SchemaCacheEntry cacheEntry1 = - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s1", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry1.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db1.d1"), "s1", cacheEntry1); - - SchemaCacheEntry cacheEntry2 = - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry2.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db1.d1"), "s2", cacheEntry2); - - SchemaCacheEntry cacheEntry3 = - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry3.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db2.d1"), "s2", cacheEntry3); - - SchemaCacheEntry cacheEntry4 = - new SchemaCacheEntry( - "root.db2", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry4.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db2.d1"), "s1", cacheEntry4); - - SchemaCacheEntry cacheEntry5 = - new SchemaCacheEntry( - "root.db1", - new MeasurementSchema("s2", TSDataType.INT32), - Collections.emptyMap(), - false); - cacheEntry5.updateLastCache(new TimeValuePair(1L, new TsPrimitiveType.TsInt(1)), true, 0L); - dualKeyCache.put(new PartialPath("root.db1"), "s2", cacheEntry5); - - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1.d1"), "s2")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db1"), "s2")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s1")); - Assert.assertNotNull(dualKeyCache.get(new PartialPath("root.db2.d1"), "s2")); - - int expectSize = - PartialPath.estimateSize(new PartialPath("root.db1.d1")) * 2 - + PartialPath.estimateSize(new PartialPath("root.db1")) - + computeStringSize("s1") * 5 - + SchemaCacheEntry.estimateSize(cacheEntry1) * 5; - Assert.assertEquals(expectSize, dualKeyCache.stats().memoryUsage()); - return dualKeyCache; - } - - @Test - public void testInvalidateSimpleTimeseriesAndDataRegion() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateLastCache(); - long memUse = dualKeyCache.stats().memoryUsage(); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db1.d1.s1")); - SchemaCacheEntry cacheEntry = dualKeyCache.get(new PartialPath("root.db1.d1"), "s1"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db1.d1.*")); - cacheEntry = dualKeyCache.get(new PartialPath("root.db1.d1"), "s2"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db2.d1.**")); - cacheEntry = dualKeyCache.get(new PartialPath("root.db2.d1"), "s2"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - dualKeyCache.invalidateDataRegionLastCache("root.db2"); - cacheEntry = dualKeyCache.get(new PartialPath("root.db2.d1"), "s1"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - cacheEntry = dualKeyCache.get(new PartialPath("root.db1"), "s2"); - // last cache container' estimateSize(): header 8b + Ilastcachevalueref 8b + lastcache's size - // invalidate operation: make Ilastcachevalueref = null. - // So the amount of change in size is estimateSize() - 8b - 8b - int size = cacheEntry.getLastCacheContainer().estimateSize() - 16; - Assert.assertEquals(memUse - size * 4, dualKeyCache.stats().memoryUsage()); - } - - @Test - public void testComplexInvalidate() throws IllegalPathException { - IDualKeyCache dualKeyCache = generateLastCache(); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db1.*.s1")); - SchemaCacheEntry cacheEntry = dualKeyCache.get(new PartialPath("root.db1.d1"), "s1"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - - dualKeyCache.invalidateLastCache(new PartialPath("root.db1.**.s2")); - cacheEntry = dualKeyCache.get(new PartialPath("root.db1.d1"), "s2"); - Assert.assertNull(cacheEntry.getLastCacheContainer().getCachedLast()); - } -} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 698ff13abe6f7..db3a6b9d79e41 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -98,6 +98,7 @@ public enum Metric { MEM("mem"), CACHE("cache"), CACHE_HIT_RATE("cache_hit"), + CACHE_ENTRIES_NUM("cache_entries_num"), QUANTITY("quantity"), LEADER_QUANTITY("leader_quantity"), SCHEMA_REGION("schema_region"),